From 01ea3ca4a4cfd9a9a92d584d8a27f432c170c05f Mon Sep 17 00:00:00 2001 From: Nicole Rappe Date: Sun, 26 Oct 2025 01:38:39 -0600 Subject: [PATCH 01/11] Bridge legacy API registration through Engine --- Data/Engine/CODE_MIGRATION_TRACKER.md | 10 ++-- Data/Engine/server.py | 80 ++++++++++++++++++++------- Data/Server/server.py | 76 +++++++++++++++++++------ 3 files changed, 124 insertions(+), 42 deletions(-) diff --git a/Data/Engine/CODE_MIGRATION_TRACKER.md b/Data/Engine/CODE_MIGRATION_TRACKER.md index 6309850..84d22c3 100644 --- a/Data/Engine/CODE_MIGRATION_TRACKER.md +++ b/Data/Engine/CODE_MIGRATION_TRACKER.md @@ -27,10 +27,10 @@ Lastly, everytime that you complete a stage, you will create a pull request name - [x] Provide fixtures that mirror the legacy SQLite schema and seed data. - [x] Assert HTTP status codes, payloads, and side effects for parity. - [x] Integrate Engine API tests into CI/local workflows. -- [ ] **Stage 5 — Bridge the legacy server to Engine APIs** - - [ ] Delegate API blueprint registration to the Engine factory from the legacy server. - - [ ] Replace legacy API routes with Engine-provided blueprints gated by a flag. - - [ ] Emit transitional logging when Engine handles requests. +- [x] **Stage 5 — Bridge the legacy server to Engine APIs** + - [x] Delegate API blueprint registration to the Engine factory from the legacy server. + - [x] Replace legacy API routes with Engine-provided blueprints gated by a flag. + - [x] Emit transitional logging when Engine handles requests. - [ ] **Stage 6 — Plan WebUI migration** - [ ] Move static/template handling into Data/Engine/services/WebUI. - [ ] Preserve TLS-aware URL generation and caching. @@ -43,5 +43,5 @@ Lastly, everytime that you complete a stage, you will create a pull request name - [ ] Update legacy server to consume Engine WebSocket registration. ## Current Status -- **Stage:** Stage 4 — Build unit and smoke tests for Engine APIs (completed) +- **Stage:** Stage 5 — Bridge the legacy server to Engine APIs (completed) - **Active Task:** Awaiting next stage instructions. diff --git a/Data/Engine/server.py b/Data/Engine/server.py index 0499b53..c8c84cf 100644 --- a/Data/Engine/server.py +++ b/Data/Engine/server.py @@ -17,7 +17,7 @@ from pathlib import Path from typing import Any, Mapping, Optional, Sequence, Tuple import eventlet -from flask import Flask +from flask import Flask, request from flask_cors import CORS from flask_socketio import SocketIO from werkzeug.middleware.proxy_fix import ProxyFix @@ -106,7 +106,46 @@ class EngineContext: api_groups: Sequence[str] -__all__ = ["EngineContext", "create_app"] +__all__ = ["EngineContext", "create_app", "register_engine_api"] + + +def _build_engine_context(settings: EngineSettings, logger: logging.Logger) -> EngineContext: + return EngineContext( + database_path=settings.database_path, + logger=logger, + scheduler=None, + tls_cert_path=settings.tls_cert_path, + tls_key_path=settings.tls_key_path, + tls_bundle_path=settings.tls_bundle_path, + config=settings.as_dict(), + api_groups=settings.api_groups, + ) + + +def _attach_transition_logging(app: Flask, context: EngineContext, logger: logging.Logger) -> None: + tracked = {group.strip().lower() for group in context.api_groups if group} + if not tracked: + tracked = {"tokens", "enrollment"} + + existing = getattr(app, "_engine_api_tracked_blueprints", set()) + if existing: + tracked.update(existing) + setattr(app, "_engine_api_tracked_blueprints", tracked) + + if getattr(app, "_engine_api_logging_installed", False): + return + + @app.before_request + def _log_engine_api_bridge() -> None: # pragma: no cover - integration behaviour exercised in higher-level tests + blueprint = (request.blueprint or "").lower() + if blueprint and blueprint in getattr(app, "_engine_api_tracked_blueprints", tracked): + logger.info( + "Engine handling API request via legacy bridge: %s %s", + request.method, + request.path, + ) + + setattr(app, "_engine_api_logging_installed", True) def create_app(config: Optional[Mapping[str, Any]] = None) -> Tuple[Flask, SocketIO, EngineContext]: @@ -115,8 +154,6 @@ def create_app(config: Optional[Mapping[str, Any]] = None) -> Tuple[Flask, Socke settings: EngineSettings = load_runtime_config(config) logger = initialise_engine_logger(settings) - database_path = settings.database_path - static_folder = settings.static_folder app = Flask(__name__, static_folder=static_folder, static_url_path="") app.wsgi_app = ProxyFix(app.wsgi_app, x_proto=1, x_host=1) @@ -141,22 +178,7 @@ def create_app(config: Optional[Mapping[str, Any]] = None) -> Tuple[Flask, Socke }, ) - tls_cert_path, tls_key_path, tls_bundle_path = ( - settings.tls_cert_path, - settings.tls_key_path, - settings.tls_bundle_path, - ) - - context = EngineContext( - database_path=database_path, - logger=logger, - scheduler=None, - tls_cert_path=tls_cert_path, - tls_key_path=tls_key_path, - tls_bundle_path=tls_bundle_path, - config=settings.as_dict(), - api_groups=settings.api_groups, - ) + context = _build_engine_context(settings, logger) from .services import API, WebSocket, WebUI # Local import to avoid circular deps during bootstrap @@ -167,3 +189,21 @@ def create_app(config: Optional[Mapping[str, Any]] = None) -> Tuple[Flask, Socke logger.debug("Engine application factory completed initialisation.") return app, socketio, context + + +def register_engine_api(app: Flask, *, config: Optional[Mapping[str, Any]] = None) -> EngineContext: + """Register Engine-managed API blueprints onto an existing Flask app.""" + + settings: EngineSettings = load_runtime_config(config) + logger = initialise_engine_logger(settings) + context = _build_engine_context(settings, logger) + + from .services import API # Local import avoids circular dependency at module import time + + API.register_api(app, context) + _attach_transition_logging(app, context, logger) + + groups_display = ", ".join(context.api_groups) if context.api_groups else "none" + logger.info("Engine API delegation activated for groups: %s", groups_display) + + return context diff --git a/Data/Server/server.py b/Data/Server/server.py index 07dcd40..67806c5 100644 --- a/Data/Server/server.py +++ b/Data/Server/server.py @@ -210,6 +210,13 @@ def _infer_server_scope(message: str, explicit: Optional[str]) -> Optional[str]: return None +def _env_flag(name: str, *, default: bool = False) -> bool: + raw = os.environ.get(name) + if raw is None: + return default + return raw.strip().lower() in {"1", "true", "yes", "on"} + + def _is_internal_request(req: Request) -> bool: """Return True if the HTTP request originated from the local server host.""" try: @@ -326,6 +333,8 @@ AUTH_RATE_LIMITER = SlidingWindowRateLimiter() ENROLLMENT_NONCE_CACHE = NonceCache() DPOP_VALIDATOR = DPoPValidator() DEVICE_AUTH_MANAGER: Optional[DeviceAuthManager] = None +ENGINE_API_ENABLED = _env_flag("BOREALIS_ENGINE_API") +ENGINE_API_GROUPS: Tuple[str, ...] = tuple() def _set_cached_github_token(token: Optional[str]) -> None: @@ -5088,24 +5097,57 @@ def init_db(): init_db() -enrollment_routes.register( - app, - db_conn_factory=_db_conn, - log=_write_service_log, - jwt_service=JWT_SERVICE, - tls_bundle_path=TLS_BUNDLE_PATH, - ip_rate_limiter=IP_RATE_LIMITER, - fp_rate_limiter=FP_RATE_LIMITER, - nonce_cache=ENROLLMENT_NONCE_CACHE, - script_signer=SCRIPT_SIGNER, -) +if ENGINE_API_ENABLED: + _engine_api_config: Dict[str, Any] = { + "DATABASE_PATH": DB_PATH, + "TLS_CERT_PATH": TLS_CERT_PATH, + "TLS_KEY_PATH": TLS_KEY_PATH, + "TLS_BUNDLE_PATH": TLS_BUNDLE_PATH, + } + api_groups_override = os.environ.get("BOREALIS_API_GROUPS") + if api_groups_override: + _engine_api_config["API_GROUPS"] = api_groups_override -token_routes.register( - app, - db_conn_factory=_db_conn, - jwt_service=JWT_SERVICE, - dpop_validator=DPOP_VALIDATOR, -) + try: + from Data.Engine.server import register_engine_api + + _engine_context = register_engine_api(app, config=_engine_api_config) + except Exception: + ENGINE_API_ENABLED = False + ENGINE_API_GROUPS = tuple() + _write_service_log( + "server", + "Engine API delegation failed; continuing with legacy API registration.", + level="ERROR", + ) + else: + ENGINE_API_GROUPS = tuple(_engine_context.api_groups) + _write_service_log( + "server", + "Engine API delegation enabled for groups: {}".format( + ", ".join(ENGINE_API_GROUPS) or "default" + ), + ) + +if not ENGINE_API_ENABLED: + enrollment_routes.register( + app, + db_conn_factory=_db_conn, + log=_write_service_log, + jwt_service=JWT_SERVICE, + tls_bundle_path=TLS_BUNDLE_PATH, + ip_rate_limiter=IP_RATE_LIMITER, + fp_rate_limiter=FP_RATE_LIMITER, + nonce_cache=ENROLLMENT_NONCE_CACHE, + script_signer=SCRIPT_SIGNER, + ) + + token_routes.register( + app, + db_conn_factory=_db_conn, + jwt_service=JWT_SERVICE, + dpop_validator=DPOP_VALIDATOR, + ) agent_routes.register( app, From 1b6d0151249e7ed22fb0739bee8cb03bdd35c691 Mon Sep 17 00:00:00 2001 From: Nicole Rappe Date: Sun, 26 Oct 2025 02:02:05 -0600 Subject: [PATCH 02/11] Fix Engine WebUI staging and logging outputs --- Borealis.ps1 | 6 +- Borealis.sh | 4 +- Data/Engine/Unit_Tests/conftest.py | 7 +- Data/Engine/config.py | 51 +++++++++++--- Data/Engine/server.py | 105 ++++++++++++++++------------- 5 files changed, 107 insertions(+), 66 deletions(-) diff --git a/Borealis.ps1 b/Borealis.ps1 index 6c1d373..8f33225 100644 --- a/Borealis.ps1 +++ b/Borealis.ps1 @@ -198,7 +198,7 @@ function Ensure-EngineWebInterface { [string]$ProjectRoot ) - $engineSource = Join-Path $ProjectRoot 'Data\Engine\web-interface' + $engineSource = Join-Path $ProjectRoot 'Engine\web-interface' $legacySource = Join-Path $ProjectRoot 'Data\Server\WebUI' if (-not (Test-Path $legacySource)) { @@ -1108,7 +1108,7 @@ switch ($choice) { $venvFolder = "Server" $dataSource = "Data" $dataDestination = "$venvFolder\Borealis" - $customUIPath = "$dataSource\Engine\web-interface" + $customUIPath = (Join-Path $scriptDir 'Engine\web-interface') $webUIDestination = "$venvFolder\web-interface" $venvPython = Join-Path $venvFolder 'Scripts\python.exe' @@ -1408,7 +1408,7 @@ switch ($choice) { Run-Step "Copy Borealis Engine WebUI Files into: $webUIDestination" { Ensure-EngineWebInterface -ProjectRoot $scriptDir - $engineWebUISource = Join-Path $engineSourceAbsolute 'web-interface' + $engineWebUISource = Join-Path $scriptDir 'Engine\web-interface' if (Test-Path $engineWebUISource) { $webUIDestinationAbsolute = Join-Path $scriptDir $webUIDestination if (Test-Path $webUIDestinationAbsolute) { diff --git a/Borealis.sh b/Borealis.sh index 1235f13..8a01512 100644 --- a/Borealis.sh +++ b/Borealis.sh @@ -331,7 +331,7 @@ PY } ensure_engine_webui_source() { - local engineSource="Data/Engine/web-interface" + local engineSource="Engine/web-interface" local legacySource="Data/Server/WebUI" if [[ -d "${engineSource}/src" && -f "${engineSource}/package.json" ]]; then return 0 @@ -351,7 +351,7 @@ ensure_engine_webui_source() { } prepare_webui() { - local customUIPath="Data/Engine/web-interface" + local customUIPath="Engine/web-interface" local webUIDestination="Server/web-interface" ensure_engine_webui_source || return 1 mkdir -p "$webUIDestination" diff --git a/Data/Engine/Unit_Tests/conftest.py b/Data/Engine/Unit_Tests/conftest.py index 7f1a01b..9256826 100644 --- a/Data/Engine/Unit_Tests/conftest.py +++ b/Data/Engine/Unit_Tests/conftest.py @@ -112,8 +112,10 @@ def engine_harness(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Iterator[ key_path.write_text("test-key", encoding="utf-8") bundle_path.write_text(bundle_contents, encoding="utf-8") - log_path = tmp_path / "logs" / "server.log" - log_path.parent.mkdir(parents=True, exist_ok=True) + logs_dir = tmp_path / "logs" + logs_dir.mkdir(parents=True, exist_ok=True) + log_path = logs_dir / "server.log" + error_log_path = logs_dir / "error.log" config = { "DATABASE_PATH": str(db_path), @@ -121,6 +123,7 @@ def engine_harness(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Iterator[ "TLS_KEY_PATH": str(key_path), "TLS_BUNDLE_PATH": str(bundle_path), "LOG_FILE": str(log_path), + "ERROR_LOG_FILE": str(error_log_path), "API_GROUPS": ("tokens", "enrollment"), } diff --git a/Data/Engine/config.py b/Data/Engine/config.py index 6c30a70..2c83265 100644 --- a/Data/Engine/config.py +++ b/Data/Engine/config.py @@ -26,8 +26,9 @@ defaults that mirror the legacy server runtime. Key environment variables are When TLS values are not provided explicitly the Engine falls back to the certificate helper shipped with the legacy server, ensuring bundling parity. -Logs are written to ``Logs/Server/server.log`` with daily rotation so the new -runtime integrates with existing operational practices. +Logs are written to ``Logs/Engine/engine.log`` with daily rotation and +errors are additionally duplicated to ``Logs/Engine/error.log`` so the +runtime integrates with the platform's logging policy. """ from __future__ import annotations @@ -48,7 +49,9 @@ except Exception: # pragma: no-cover - Engine configuration still works without ENGINE_DIR = Path(__file__).resolve().parent PROJECT_ROOT = ENGINE_DIR.parent.parent DEFAULT_DATABASE_PATH = PROJECT_ROOT / "database.db" -LOG_FILE_PATH = PROJECT_ROOT / "Logs" / "Server" / "server.log" +LOG_ROOT = PROJECT_ROOT / "Logs" / "Engine" +LOG_FILE_PATH = LOG_ROOT / "engine.log" +ERROR_LOG_FILE_PATH = LOG_ROOT / "error.log" def _ensure_parent(path: Path) -> None: @@ -61,16 +64,27 @@ def _ensure_parent(path: Path) -> None: def _resolve_static_folder() -> str: - candidates = [ - ENGINE_DIR / "web-interface" / "build", - ENGINE_DIR / "web-interface" / "dist", + candidate_roots = [ + ENGINE_DIR.parent / "Engine" / "web-interface", ENGINE_DIR / "web-interface", ] + + candidates = [] + for root in candidate_roots: + absolute_root = root.resolve() + candidates.extend( + [ + absolute_root / "build", + absolute_root / "dist", + absolute_root, + ] + ) + for candidate in candidates: - absolute = candidate.resolve() - if absolute.is_dir(): - return str(absolute) - return str(candidates[0].resolve()) + if candidate.is_dir(): + return str(candidate) + + return str(candidates[0]) def _parse_origins(raw: Optional[Any]) -> Optional[List[str]]: @@ -139,6 +153,7 @@ class EngineSettings: tls_key_path: Optional[str] tls_bundle_path: Optional[str] log_file: str + error_log_file: str api_groups: Tuple[str, ...] raw: MutableMapping[str, Any] = field(default_factory=dict) @@ -219,6 +234,9 @@ def load_runtime_config(overrides: Optional[Mapping[str, Any]] = None) -> Engine log_file = str(runtime_config.get("LOG_FILE") or LOG_FILE_PATH) _ensure_parent(Path(log_file)) + error_log_file = str(runtime_config.get("ERROR_LOG_FILE") or ERROR_LOG_FILE_PATH) + _ensure_parent(Path(error_log_file)) + api_groups = _parse_api_groups( runtime_config.get("API_GROUPS") or os.environ.get("BOREALIS_API_GROUPS") ) @@ -237,6 +255,7 @@ def load_runtime_config(overrides: Optional[Mapping[str, Any]] = None) -> Engine tls_key_path=tls_key_path if tls_key_path else None, tls_bundle_path=tls_bundle_path if tls_bundle_path else None, log_file=str(log_file), + error_log_file=str(error_log_file), api_groups=api_groups, raw=runtime_config, ) @@ -244,7 +263,7 @@ def load_runtime_config(overrides: Optional[Mapping[str, Any]] = None) -> Engine def initialise_engine_logger(settings: EngineSettings, name: str = "borealis.engine") -> logging.Logger: - """Configure the Engine logger to write to the shared server log.""" + """Configure the Engine logger to write to Engine log files.""" logger = logging.getLogger(name) if not logger.handlers: @@ -263,6 +282,16 @@ def initialise_engine_logger(settings: EngineSettings, name: str = "borealis.eng file_handler.setFormatter(formatter) logger.addHandler(file_handler) + error_handler = TimedRotatingFileHandler( + settings.error_log_file, + when="midnight", + backupCount=0, + encoding="utf-8", + ) + error_handler.setLevel(logging.ERROR) + error_handler.setFormatter(formatter) + logger.addHandler(error_handler) + logger.setLevel(logging.INFO) logger.propagate = False return logger diff --git a/Data/Engine/server.py b/Data/Engine/server.py index c8c84cf..afccc7c 100644 --- a/Data/Engine/server.py +++ b/Data/Engine/server.py @@ -4,8 +4,8 @@ Stage 1 introduced the structural skeleton for the Engine runtime. Stage 2 builds upon that foundation by centralising configuration handling and logging initialisation so the Engine mirrors the legacy server's start-up behaviour. The factory delegates configuration resolution to :mod:`Data.Engine.config` -and emits structured logs to ``Logs/Server/server.log`` to align with the -project's operational practices. +and emits structured logs to ``Logs/Engine/engine.log`` (with an accompanying +error log) to align with the project's operational practices. """ from __future__ import annotations @@ -16,63 +16,72 @@ from dataclasses import dataclass from pathlib import Path from typing import Any, Mapping, Optional, Sequence, Tuple -import eventlet +try: # pragma: no-cover - optional dependency when running without eventlet + import eventlet # type: ignore +except Exception: # pragma: no-cover - fall back to threading mode + eventlet = None # type: ignore[assignment] + logging.getLogger(__name__).warning( + "Eventlet is not available; Engine will run Socket.IO in threading mode." + ) +else: # pragma: no-cover - monkey patch only when eventlet is present + eventlet.monkey_patch(thread=False) + from flask import Flask, request from flask_cors import CORS from flask_socketio import SocketIO from werkzeug.middleware.proxy_fix import ProxyFix -# Eventlet ensures Socket.IO long-polling and WebSocket support parity with the -# legacy server. We keep thread pools enabled for compatibility with blocking -# filesystem/database operations. -eventlet.monkey_patch(thread=False) +if eventlet: + try: # pragma: no-cover - defensive import mirroring the legacy runtime. + from eventlet.wsgi import HttpProtocol # type: ignore + except Exception: # pragma: no-cover - the Engine should still operate without it. + HttpProtocol = None # type: ignore[assignment] + else: + _original_handle_one_request = HttpProtocol.handle_one_request -try: # pragma: no-cover - defensive import mirroring the legacy runtime. - from eventlet.wsgi import HttpProtocol # type: ignore -except Exception: # pragma: no-cover - the Engine should still operate without it. - HttpProtocol = None # type: ignore[assignment] -else: - _original_handle_one_request = HttpProtocol.handle_one_request + def _quiet_tls_http_mismatch(self): # type: ignore[override] + """Mirror the legacy suppression of noisy TLS handshake errors.""" - def _quiet_tls_http_mismatch(self): # type: ignore[override] - """Mirror the legacy suppression of noisy TLS handshake errors.""" + def _close_connection_quietly(): + try: + self.close_connection = True # type: ignore[attr-defined] + except Exception: + pass + try: + conn = getattr(self, "socket", None) or getattr(self, "connection", None) + if conn: + conn.close() + except Exception: + pass - def _close_connection_quietly(): try: - self.close_connection = True # type: ignore[attr-defined] - except Exception: - pass - try: - conn = getattr(self, "socket", None) or getattr(self, "connection", None) - if conn: - conn.close() - except Exception: - pass - - try: - return _original_handle_one_request(self) - except ssl.SSLError as exc: # type: ignore[arg-type] - reason = getattr(exc, "reason", "") - reason_text = str(reason).lower() if reason else "" - message = " ".join(str(arg) for arg in exc.args if arg).lower() - if ( - "http_request" in message - or reason_text == "http request" - or "unknown ca" in message - or reason_text == "unknown ca" - or "unknown_ca" in message - ): + return _original_handle_one_request(self) + except ssl.SSLError as exc: # type: ignore[arg-type] + reason = getattr(exc, "reason", "") + reason_text = str(reason).lower() if reason else "" + message = " ".join(str(arg) for arg in exc.args if arg).lower() + if ( + "http_request" in message + or reason_text == "http request" + or "unknown ca" in message + or reason_text == "unknown ca" + or "unknown_ca" in message + ): + _close_connection_quietly() + return None + raise + except ssl.SSLEOFError: + _close_connection_quietly() + return None + except ConnectionAbortedError: _close_connection_quietly() return None - raise - except ssl.SSLEOFError: - _close_connection_quietly() - return None - except ConnectionAbortedError: - _close_connection_quietly() - return None - HttpProtocol.handle_one_request = _quiet_tls_http_mismatch # type: ignore[assignment] + HttpProtocol.handle_one_request = _quiet_tls_http_mismatch # type: ignore[assignment] +else: + HttpProtocol = None # type: ignore[assignment] + +_SOCKETIO_ASYNC_MODE = "eventlet" if eventlet else "threading" # Ensure the legacy ``Modules`` package is importable when running from the @@ -171,7 +180,7 @@ def create_app(config: Optional[Mapping[str, Any]] = None) -> Tuple[Flask, Socke socketio = SocketIO( app, cors_allowed_origins="*", - async_mode="eventlet", + async_mode=_SOCKETIO_ASYNC_MODE, engineio_options={ "max_http_buffer_size": 100_000_000, "max_websocket_message_size": 100_000_000, From ccf2e636203ecd8d80cd251670fcc5580e7ebfe3 Mon Sep 17 00:00:00 2001 From: Nicole Rappe Date: Sun, 26 Oct 2025 02:09:44 -0600 Subject: [PATCH 03/11] Require Eventlet for Engine runtime --- Data/Engine/server.py | 115 ++++++++++++++++++++++-------------------- 1 file changed, 59 insertions(+), 56 deletions(-) diff --git a/Data/Engine/server.py b/Data/Engine/server.py index afccc7c..e0babcb 100644 --- a/Data/Engine/server.py +++ b/Data/Engine/server.py @@ -9,6 +9,7 @@ error log) to align with the project's operational practices. """ from __future__ import annotations +import importlib.util import logging import ssl import sys @@ -16,72 +17,74 @@ from dataclasses import dataclass from pathlib import Path from typing import Any, Mapping, Optional, Sequence, Tuple -try: # pragma: no-cover - optional dependency when running without eventlet - import eventlet # type: ignore -except Exception: # pragma: no-cover - fall back to threading mode - eventlet = None # type: ignore[assignment] - logging.getLogger(__name__).warning( - "Eventlet is not available; Engine will run Socket.IO in threading mode." - ) -else: # pragma: no-cover - monkey patch only when eventlet is present - eventlet.monkey_patch(thread=False) -from flask import Flask, request -from flask_cors import CORS -from flask_socketio import SocketIO -from werkzeug.middleware.proxy_fix import ProxyFix +def _require_dependency(module: str, friendly_name: str) -> None: + if importlib.util.find_spec(module) is None: # pragma: no cover - import check + raise RuntimeError( + f"{friendly_name} (Python module '{module}') is required for the Borealis Engine runtime. " + "Install the packaged dependencies by running Borealis.ps1 or ensure the module is present in the active environment." + ) -if eventlet: - try: # pragma: no-cover - defensive import mirroring the legacy runtime. - from eventlet.wsgi import HttpProtocol # type: ignore - except Exception: # pragma: no-cover - the Engine should still operate without it. - HttpProtocol = None # type: ignore[assignment] - else: - _original_handle_one_request = HttpProtocol.handle_one_request - def _quiet_tls_http_mismatch(self): # type: ignore[override] - """Mirror the legacy suppression of noisy TLS handshake errors.""" +_require_dependency("eventlet", "Eventlet") +_require_dependency("flask", "Flask") +_require_dependency("flask_socketio", "Flask-SocketIO") - def _close_connection_quietly(): - try: - self.close_connection = True # type: ignore[attr-defined] - except Exception: - pass - try: - conn = getattr(self, "socket", None) or getattr(self, "connection", None) - if conn: - conn.close() - except Exception: - pass +import eventlet # type: ignore # noqa: E402 # pragma: no cover - import guarded above +from eventlet import wsgi as eventlet_wsgi # type: ignore # noqa: E402 # pragma: no cover +from flask import Flask, request # noqa: E402 +from flask_cors import CORS # noqa: E402 +from flask_socketio import SocketIO # noqa: E402 +from werkzeug.middleware.proxy_fix import ProxyFix # noqa: E402 + +eventlet.monkey_patch(thread=False) # pragma: no cover - aligns with legacy runtime + +HttpProtocol = getattr(eventlet_wsgi, "HttpProtocol", None) +if HttpProtocol is not None: # pragma: no branch - attribute exists in supported versions + _original_handle_one_request = HttpProtocol.handle_one_request + + def _quiet_tls_http_mismatch(self): # type: ignore[override] + """Mirror the legacy suppression of noisy TLS handshake errors.""" + + def _close_connection_quietly(): try: - return _original_handle_one_request(self) - except ssl.SSLError as exc: # type: ignore[arg-type] - reason = getattr(exc, "reason", "") - reason_text = str(reason).lower() if reason else "" - message = " ".join(str(arg) for arg in exc.args if arg).lower() - if ( - "http_request" in message - or reason_text == "http request" - or "unknown ca" in message - or reason_text == "unknown ca" - or "unknown_ca" in message - ): - _close_connection_quietly() - return None - raise - except ssl.SSLEOFError: - _close_connection_quietly() - return None - except ConnectionAbortedError: + self.close_connection = True # type: ignore[attr-defined] + except Exception: + pass + try: + conn = getattr(self, "socket", None) or getattr(self, "connection", None) + if conn: + conn.close() + except Exception: + pass + + try: + return _original_handle_one_request(self) + except ssl.SSLError as exc: # type: ignore[arg-type] + reason = getattr(exc, "reason", "") + reason_text = str(reason).lower() if reason else "" + message = " ".join(str(arg) for arg in exc.args if arg).lower() + if ( + "http_request" in message + or reason_text == "http request" + or "unknown ca" in message + or reason_text == "unknown ca" + or "unknown_ca" in message + ): _close_connection_quietly() return None + raise + except ssl.SSLEOFError: + _close_connection_quietly() + return None + except ConnectionAbortedError: + _close_connection_quietly() + return None - HttpProtocol.handle_one_request = _quiet_tls_http_mismatch # type: ignore[assignment] -else: - HttpProtocol = None # type: ignore[assignment] + HttpProtocol.handle_one_request = _quiet_tls_http_mismatch # type: ignore[assignment] -_SOCKETIO_ASYNC_MODE = "eventlet" if eventlet else "threading" +_SOCKETIO_ASYNC_MODE = "eventlet" # Ensure the legacy ``Modules`` package is importable when running from the From 9bc72ec6565d24d68cdef6ef1efdc12dcfe769f4 Mon Sep 17 00:00:00 2001 From: Nicole Rappe Date: Sun, 26 Oct 2025 02:25:28 -0600 Subject: [PATCH 04/11] Add Engine runtime dependency manifest --- Data/Engine/requirements.txt | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 Data/Engine/requirements.txt diff --git a/Data/Engine/requirements.txt b/Data/Engine/requirements.txt new file mode 100644 index 0000000..9ecad6c --- /dev/null +++ b/Data/Engine/requirements.txt @@ -0,0 +1,9 @@ +#////////// PROJECT FILE SEPARATION LINE ////////// CODE AFTER THIS LINE ARE FROM: /Data/Engine/requirements.txt +Flask +flask-cors +flask_socketio +eventlet +cryptography +PyJWT[crypto] +pyotp +qrcode From 6f7ed47bf6d6987c001047a12d514f4c863ca7ef Mon Sep 17 00:00:00 2001 From: Nicole Rappe Date: Sun, 26 Oct 2025 02:25:33 -0600 Subject: [PATCH 05/11] Rename Engine requirements manifest --- Data/Engine/{requirements.txt => engine-requirements.txt} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename Data/Engine/{requirements.txt => engine-requirements.txt} (67%) diff --git a/Data/Engine/requirements.txt b/Data/Engine/engine-requirements.txt similarity index 67% rename from Data/Engine/requirements.txt rename to Data/Engine/engine-requirements.txt index 9ecad6c..a892616 100644 --- a/Data/Engine/requirements.txt +++ b/Data/Engine/engine-requirements.txt @@ -1,4 +1,4 @@ -#////////// PROJECT FILE SEPARATION LINE ////////// CODE AFTER THIS LINE ARE FROM: /Data/Engine/requirements.txt +#////////// PROJECT FILE SEPARATION LINE ////////// CODE AFTER THIS LINE ARE FROM: /Data/Engine/engine-requirements.txt Flask flask-cors flask_socketio From b3fdd63d2bb565d2cc4fe2fd423715e60b40bb15 Mon Sep 17 00:00:00 2001 From: Nicole Rappe Date: Sun, 26 Oct 2025 02:57:22 -0600 Subject: [PATCH 06/11] Ensure Engine can locate legacy modules --- Data/Engine/server.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Data/Engine/server.py b/Data/Engine/server.py index e0babcb..a059b23 100644 --- a/Data/Engine/server.py +++ b/Data/Engine/server.py @@ -93,6 +93,7 @@ _ENGINE_DIR = Path(__file__).resolve().parent _SEARCH_ROOTS = [ _ENGINE_DIR.parent / "Server", _ENGINE_DIR.parent.parent / "Data" / "Server", + _ENGINE_DIR.parent.parent.parent / "Data" / "Server", ] for root in _SEARCH_ROOTS: modules_dir = root / "Modules" From 3edb20d21b08c404d818cf41354254a75f75b212 Mon Sep 17 00:00:00 2001 From: Nicole Rappe Date: Sun, 26 Oct 2025 03:11:57 -0600 Subject: [PATCH 07/11] Validate Engine TLS material before starting Socket.IO --- Data/Engine/bootstrapper.py | 49 ++++++++++++++++++++++++++++++++++--- 1 file changed, 46 insertions(+), 3 deletions(-) diff --git a/Data/Engine/bootstrapper.py b/Data/Engine/bootstrapper.py index 3658ba3..e24caa8 100644 --- a/Data/Engine/bootstrapper.py +++ b/Data/Engine/bootstrapper.py @@ -9,9 +9,10 @@ legacy server defaults by binding to ``0.0.0.0:5001`` and honouring the from __future__ import annotations import os +from pathlib import Path from typing import Any, Dict -from .server import create_app +from .server import EngineContext, create_app DEFAULT_HOST = "0.0.0.0" @@ -28,6 +29,41 @@ def _build_runtime_config() -> Dict[str, Any]: } +def _prepare_tls_run_kwargs(context: EngineContext) -> Dict[str, Any]: + """Validate and return TLS arguments for the Socket.IO runner.""" + + run_kwargs: Dict[str, Any] = {} + + key_path_value = context.tls_key_path + if not key_path_value: + return run_kwargs + + key_path = Path(key_path_value) + if not key_path.is_file(): + raise RuntimeError(f"Engine TLS key file not found: {key_path}") + + cert_candidates = [] + if context.tls_bundle_path: + cert_candidates.append(context.tls_bundle_path) + if context.tls_cert_path and context.tls_cert_path not in cert_candidates: + cert_candidates.append(context.tls_cert_path) + + if not cert_candidates: + raise RuntimeError("Engine TLS certificate path not configured; ensure certificates are provisioned.") + + missing_candidates = [] + for candidate in cert_candidates: + candidate_path = Path(candidate) + if candidate_path.is_file(): + run_kwargs["certfile"] = str(candidate_path) + run_kwargs["keyfile"] = str(key_path) + return run_kwargs + missing_candidates.append(str(candidate_path)) + + checked = ", ".join(missing_candidates) + raise RuntimeError(f"Engine TLS certificate file not found. Checked: {checked}") + + def main() -> None: config = _build_runtime_config() app, socketio, context = create_app(config) @@ -36,8 +72,15 @@ def main() -> None: port = int(config.get("PORT", DEFAULT_PORT)) run_kwargs: Dict[str, Any] = {"host": host, "port": port} - if context.tls_bundle_path and context.tls_key_path: - run_kwargs.update({"certfile": context.tls_bundle_path, "keyfile": context.tls_key_path}) + try: + tls_kwargs = _prepare_tls_run_kwargs(context) + except RuntimeError as exc: + context.logger.error("TLS configuration error: %s", exc) + raise + else: + if tls_kwargs: + run_kwargs.update(tls_kwargs) + context.logger.info("Engine TLS enabled using certificate %s", tls_kwargs["certfile"]) socketio.run(app, **run_kwargs) From 021ba436f75138d7d4c6227b4c92f81e701e2e98 Mon Sep 17 00:00:00 2001 From: Nicole Rappe Date: Sun, 26 Oct 2025 03:20:57 -0600 Subject: [PATCH 08/11] Auto-provision Engine TLS assets when launching --- Data/Engine/bootstrapper.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/Data/Engine/bootstrapper.py b/Data/Engine/bootstrapper.py index e24caa8..400a1df 100644 --- a/Data/Engine/bootstrapper.py +++ b/Data/Engine/bootstrapper.py @@ -29,9 +29,37 @@ def _build_runtime_config() -> Dict[str, Any]: } +def _ensure_tls_material(context: EngineContext) -> None: + """Ensure TLS certificate material exists, updating the context if created.""" + + try: # Lazy import so Engine still starts if legacy modules are unavailable. + from Modules.crypto import certificates # type: ignore + except Exception: + return + + try: + cert_path, key_path, bundle_path = certificates.ensure_certificate() + except Exception as exc: + context.logger.error("Failed to auto-provision Engine TLS certificates: %s", exc) + return + + cert_path_str = str(cert_path) + key_path_str = str(key_path) + bundle_path_str = str(bundle_path) + + if not context.tls_cert_path or not Path(context.tls_cert_path).is_file(): + context.tls_cert_path = cert_path_str + if not context.tls_key_path or not Path(context.tls_key_path).is_file(): + context.tls_key_path = key_path_str + if not context.tls_bundle_path or not Path(context.tls_bundle_path).is_file(): + context.tls_bundle_path = bundle_path_str + + def _prepare_tls_run_kwargs(context: EngineContext) -> Dict[str, Any]: """Validate and return TLS arguments for the Socket.IO runner.""" + _ensure_tls_material(context) + run_kwargs: Dict[str, Any] = {} key_path_value = context.tls_key_path From 1fe349af853ce2e213292befc98f2cc026e6ba48 Mon Sep 17 00:00:00 2001 From: Nicole Rappe Date: Sun, 26 Oct 2025 03:37:05 -0600 Subject: [PATCH 09/11] Stage Engine web interface assets at runtime --- Data/Engine/bootstrapper.py | 63 ++++++++++++++++++++++++++++++++++++- Data/Engine/config.py | 3 +- 2 files changed, 64 insertions(+), 2 deletions(-) diff --git a/Data/Engine/bootstrapper.py b/Data/Engine/bootstrapper.py index 400a1df..a833fee 100644 --- a/Data/Engine/bootstrapper.py +++ b/Data/Engine/bootstrapper.py @@ -8,9 +8,11 @@ legacy server defaults by binding to ``0.0.0.0:5001`` and honouring the from __future__ import annotations +import logging import os +import shutil from pathlib import Path -from typing import Any, Dict +from typing import Any, Dict, Set from .server import EngineContext, create_app @@ -19,6 +21,10 @@ DEFAULT_HOST = "0.0.0.0" DEFAULT_PORT = 5001 +def _project_root() -> Path: + return Path(__file__).resolve().parents[2] + + def _build_runtime_config() -> Dict[str, Any]: return { "HOST": os.environ.get("BOREALIS_ENGINE_HOST", DEFAULT_HOST), @@ -29,6 +35,55 @@ def _build_runtime_config() -> Dict[str, Any]: } +def _stage_web_interface_assets(logger: logging.Logger) -> None: + project_root = _project_root() + engine_web_root = project_root / "Engine" / "web-interface" + legacy_source = project_root / "Data" / "Server" / "WebUI" + + package_json = engine_web_root / "package.json" + src_dir = engine_web_root / "src" + + if package_json.is_file() and src_dir.is_dir(): + return + + if not legacy_source.is_dir(): + logger.warning( + "Legacy WebUI source missing; unable to stage Engine web interface from %s", + legacy_source, + ) + return + + engine_web_root.mkdir(parents=True, exist_ok=True) + + preserved: Set[str] = {".gitignore", "README.md"} + for child in engine_web_root.iterdir(): + if child.name in preserved: + continue + if child.is_dir(): + shutil.rmtree(child, ignore_errors=True) + else: + try: + child.unlink() + except FileNotFoundError: + continue + + for item in legacy_source.iterdir(): + destination = engine_web_root / item.name + if item.is_dir(): + shutil.copytree(item, destination, dirs_exist_ok=True) + else: + shutil.copy2(item, destination) + + if not package_json.is_file() or not src_dir.is_dir(): + raise RuntimeError( + f"Failed to stage Engine web interface assets into {engine_web_root}" + ) + + logger.info( + "Engine web interface staged from %s to %s", legacy_source, engine_web_root + ) + + def _ensure_tls_material(context: EngineContext) -> None: """Ensure TLS certificate material exists, updating the context if created.""" @@ -96,6 +151,12 @@ def main() -> None: config = _build_runtime_config() app, socketio, context = create_app(config) + try: + _stage_web_interface_assets(context.logger) + except Exception as exc: + context.logger.error("Failed to stage Engine web interface: %s", exc) + raise + host = config.get("HOST", DEFAULT_HOST) port = int(config.get("PORT", DEFAULT_PORT)) diff --git a/Data/Engine/config.py b/Data/Engine/config.py index 2c83265..c743b3a 100644 --- a/Data/Engine/config.py +++ b/Data/Engine/config.py @@ -65,8 +65,9 @@ def _ensure_parent(path: Path) -> None: def _resolve_static_folder() -> str: candidate_roots = [ - ENGINE_DIR.parent / "Engine" / "web-interface", + PROJECT_ROOT / "Engine" / "web-interface", ENGINE_DIR / "web-interface", + PROJECT_ROOT / "Data" / "Server" / "web-interface", ] candidates = [] From d7a263afee91031f18394f6c3d0974542807f544 Mon Sep 17 00:00:00 2001 From: Nicole Rappe Date: Sun, 26 Oct 2025 03:41:53 -0600 Subject: [PATCH 10/11] Fix Engine web UI staging root detection --- Data/Engine/bootstrapper.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/Data/Engine/bootstrapper.py b/Data/Engine/bootstrapper.py index a833fee..b62e105 100644 --- a/Data/Engine/bootstrapper.py +++ b/Data/Engine/bootstrapper.py @@ -22,7 +22,25 @@ DEFAULT_PORT = 5001 def _project_root() -> Path: - return Path(__file__).resolve().parents[2] + """Locate the repository root regardless of runtime staging depth.""" + + current = Path(__file__).resolve().parent + + # Prefer an explicit sentinel (Borealis.ps1) so running from the staged + # Engine runtime still discovers the true project root one level higher. + for candidate in (current, *current.parents): + if (candidate / "Borealis.ps1").is_file(): + return candidate + + # Fallback: if we're inside ``/Engine/Data/Engine`` the parent of the + # ``Engine`` directory is the project root. + for candidate in current.parents: + if candidate.name.lower() == "engine" and (candidate.parent / "Borealis.ps1").is_file(): + return candidate.parent + + # Last resort mirrors the previous behaviour (two levels up) so we don't + # regress in unforeseen layouts. + return current.parents[1] def _build_runtime_config() -> Dict[str, Any]: From 016568c2483d759b5a91c17de45d6b6e57794f1c Mon Sep 17 00:00:00 2001 From: Nicole Rappe Date: Sun, 26 Oct 2025 03:50:05 -0600 Subject: [PATCH 11/11] Enforce web interface staging to always refresh --- Data/Engine/bootstrapper.py | 66 ++++++++++--------------------------- 1 file changed, 18 insertions(+), 48 deletions(-) diff --git a/Data/Engine/bootstrapper.py b/Data/Engine/bootstrapper.py index b62e105..c1e8a49 100644 --- a/Data/Engine/bootstrapper.py +++ b/Data/Engine/bootstrapper.py @@ -12,7 +12,7 @@ import logging import os import shutil from pathlib import Path -from typing import Any, Dict, Set +from typing import Any, Dict from .server import EngineContext, create_app @@ -22,25 +22,18 @@ DEFAULT_PORT = 5001 def _project_root() -> Path: - """Locate the repository root regardless of runtime staging depth.""" + """Locate the repository root by discovering the Borealis bootstrap script.""" current = Path(__file__).resolve().parent - # Prefer an explicit sentinel (Borealis.ps1) so running from the staged - # Engine runtime still discovers the true project root one level higher. for candidate in (current, *current.parents): if (candidate / "Borealis.ps1").is_file(): return candidate - # Fallback: if we're inside ``/Engine/Data/Engine`` the parent of the - # ``Engine`` directory is the project root. - for candidate in current.parents: - if candidate.name.lower() == "engine" and (candidate.parent / "Borealis.ps1").is_file(): - return candidate.parent - - # Last resort mirrors the previous behaviour (two levels up) so we don't - # regress in unforeseen layouts. - return current.parents[1] + raise RuntimeError( + "Unable to locate the Borealis project root; Borealis.ps1 was not found " + "in any parent directory." + ) def _build_runtime_config() -> Dict[str, Any]: @@ -58,43 +51,20 @@ def _stage_web_interface_assets(logger: logging.Logger) -> None: engine_web_root = project_root / "Engine" / "web-interface" legacy_source = project_root / "Data" / "Server" / "WebUI" - package_json = engine_web_root / "package.json" - src_dir = engine_web_root / "src" - - if package_json.is_file() and src_dir.is_dir(): - return - if not legacy_source.is_dir(): - logger.warning( - "Legacy WebUI source missing; unable to stage Engine web interface from %s", - legacy_source, - ) - return - - engine_web_root.mkdir(parents=True, exist_ok=True) - - preserved: Set[str] = {".gitignore", "README.md"} - for child in engine_web_root.iterdir(): - if child.name in preserved: - continue - if child.is_dir(): - shutil.rmtree(child, ignore_errors=True) - else: - try: - child.unlink() - except FileNotFoundError: - continue - - for item in legacy_source.iterdir(): - destination = engine_web_root / item.name - if item.is_dir(): - shutil.copytree(item, destination, dirs_exist_ok=True) - else: - shutil.copy2(item, destination) - - if not package_json.is_file() or not src_dir.is_dir(): raise RuntimeError( - f"Failed to stage Engine web interface assets into {engine_web_root}" + f"Engine web interface source missing: {legacy_source}" + ) + + if engine_web_root.exists(): + shutil.rmtree(engine_web_root) + + shutil.copytree(legacy_source, engine_web_root) + + index_path = engine_web_root / "index.html" + if not index_path.is_file(): + raise RuntimeError( + f"Engine web interface staging failed; missing {index_path}" ) logger.info(