From 40cab79f2186645320e1ea50372c4614890d5fd9 Mon Sep 17 00:00:00 2001 From: Nicole Rappe Date: Thu, 23 Oct 2025 01:51:27 -0600 Subject: [PATCH] Restore agent detail ingestion and device description updates --- Data/Engine/domain/devices.py | 4 + Data/Engine/interfaces/http/agent.py | 39 +++- Data/Engine/interfaces/http/devices.py | 20 +- .../sqlite/device_inventory_repository.py | 59 ++++- Data/Engine/services/devices/__init__.py | 15 +- .../devices/device_inventory_service.py | 205 +++++++++++++++++- Data/Engine/tests/test_http_agent.py | 98 +++++++++ Data/Engine/tests/test_http_sites_devices.py | 45 +++- 8 files changed, 473 insertions(+), 12 deletions(-) diff --git a/Data/Engine/domain/devices.py b/Data/Engine/domain/devices.py index 5c292c2..b369169 100644 --- a/Data/Engine/domain/devices.py +++ b/Data/Engine/domain/devices.py @@ -228,6 +228,10 @@ def assemble_device_snapshot(record: Mapping[str, Any]) -> Dict[str, Any]: "agent_guid": record.get("guid") or record.get("agent_guid") or "", "connection_type": record.get("connection_type") or "", "connection_endpoint": record.get("connection_endpoint") or "", + "ssl_key_fingerprint": record.get("ssl_key_fingerprint") or "", + "status": record.get("status") or "", + "token_version": record.get("token_version") or 0, + "key_added_at": record.get("key_added_at") or "", "created_at": record.get("created_at") or 0, } diff --git a/Data/Engine/interfaces/http/agent.py b/Data/Engine/interfaces/http/agent.py index 1d415db..811a939 100644 --- a/Data/Engine/interfaces/http/agent.py +++ b/Data/Engine/interfaces/http/agent.py @@ -11,7 +11,10 @@ from flask import Blueprint, Flask, current_app, g, jsonify, request from Data.Engine.builders.device_auth import DeviceAuthRequestBuilder from Data.Engine.domain.device_auth import DeviceAuthContext, DeviceAuthFailure from Data.Engine.services.container import EngineServiceContainer -from Data.Engine.services.devices.device_inventory_service import DeviceHeartbeatError +from Data.Engine.services.devices.device_inventory_service import ( + DeviceDetailsError, + DeviceHeartbeatError, +) AGENT_CONTEXT_HEADER = "X-Borealis-Agent-Context" @@ -110,4 +113,36 @@ def script_request() -> Any: return jsonify(response) -__all__ = ["register", "blueprint", "heartbeat", "script_request", "require_device_auth"] +@blueprint.route("/api/agent/details", methods=["POST"]) +@require_device_auth +def save_details() -> Any: + services = _services() + payload = request.get_json(force=True, silent=True) or {} + context = cast(DeviceAuthContext, g.device_auth) + + try: + services.device_inventory.save_agent_details(context=context, payload=payload) + except DeviceDetailsError as exc: + error_payload = {"error": exc.code} + if exc.code == "invalid_payload": + return jsonify(error_payload), 400 + if exc.code in {"fingerprint_mismatch", "guid_mismatch"}: + return jsonify(error_payload), 403 + if exc.code == "device_not_registered": + return jsonify(error_payload), 404 + current_app.logger.exception( + "device-details-error guid=%s code=%s", context.identity.guid.value, exc.code + ) + return jsonify(error_payload), 500 + + return jsonify({"status": "ok"}) + + +__all__ = [ + "register", + "blueprint", + "heartbeat", + "script_request", + "save_details", + "require_device_auth", +] diff --git a/Data/Engine/interfaces/http/devices.py b/Data/Engine/interfaces/http/devices.py index e618aa8..4c10c2c 100644 --- a/Data/Engine/interfaces/http/devices.py +++ b/Data/Engine/interfaces/http/devices.py @@ -5,7 +5,7 @@ from ipaddress import ip_address from flask import Blueprint, Flask, current_app, jsonify, request, session from Data.Engine.services.container import EngineServiceContainer -from Data.Engine.services.devices import RemoteDeviceError +from Data.Engine.services.devices import DeviceDescriptionError, RemoteDeviceError blueprint = Blueprint("engine_devices", __name__) @@ -64,6 +64,24 @@ def get_device_by_guid(guid: str) -> object: return jsonify(device) +@blueprint.route("/api/device/description/", methods=["POST"]) +def set_device_description(hostname: str) -> object: + payload = request.get_json(silent=True) or {} + description = payload.get("description") + try: + _inventory().update_device_description(hostname, description) + except DeviceDescriptionError as exc: + if exc.code == "invalid_hostname": + return jsonify({"error": "invalid hostname"}), 400 + if exc.code == "not_found": + return jsonify({"error": "not found"}), 404 + current_app.logger.exception( + "device-description-error host=%s code=%s", hostname, exc.code + ) + return jsonify({"error": "internal error"}), 500 + return jsonify({"status": "ok"}) + + @blueprint.route("/api/agent_devices", methods=["GET"]) def list_agent_devices() -> object: guard = _require_admin() diff --git a/Data/Engine/repositories/sqlite/device_inventory_repository.py b/Data/Engine/repositories/sqlite/device_inventory_repository.py index 8ae5767..9a50a9e 100644 --- a/Data/Engine/repositories/sqlite/device_inventory_repository.py +++ b/Data/Engine/repositories/sqlite/device_inventory_repository.py @@ -5,6 +5,7 @@ from __future__ import annotations import logging import sqlite3 import time +import uuid from contextlib import closing from typing import Any, Dict, List, Optional, Tuple @@ -158,8 +159,12 @@ class SQLiteDeviceInventoryRepository: agent_id, ansible_ee_ver, connection_type, - connection_endpoint - ) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) + connection_endpoint, + ssl_key_fingerprint, + token_version, + status, + key_added_at + ) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(hostname) DO UPDATE SET description=excluded.description, created_at=COALESCE({DEVICE_TABLE}.created_at, excluded.created_at), @@ -182,7 +187,11 @@ class SQLiteDeviceInventoryRepository: agent_id=COALESCE(NULLIF(excluded.agent_id, ''), {DEVICE_TABLE}.agent_id), ansible_ee_ver=COALESCE(NULLIF(excluded.ansible_ee_ver, ''), {DEVICE_TABLE}.ansible_ee_ver), connection_type=COALESCE(NULLIF(excluded.connection_type, ''), {DEVICE_TABLE}.connection_type), - connection_endpoint=COALESCE(NULLIF(excluded.connection_endpoint, ''), {DEVICE_TABLE}.connection_endpoint) + connection_endpoint=COALESCE(NULLIF(excluded.connection_endpoint, ''), {DEVICE_TABLE}.connection_endpoint), + ssl_key_fingerprint=COALESCE(NULLIF(excluded.ssl_key_fingerprint, ''), {DEVICE_TABLE}.ssl_key_fingerprint), + token_version=COALESCE(NULLIF(excluded.token_version, 0), {DEVICE_TABLE}.token_version), + status=COALESCE(NULLIF(excluded.status, ''), {DEVICE_TABLE}.status), + key_added_at=COALESCE(NULLIF(excluded.key_added_at, ''), {DEVICE_TABLE}.key_added_at) """ params: List[Any] = [ @@ -209,6 +218,10 @@ class SQLiteDeviceInventoryRepository: column_values.get("ansible_ee_ver"), column_values.get("connection_type"), column_values.get("connection_endpoint"), + column_values.get("ssl_key_fingerprint"), + column_values.get("token_version"), + column_values.get("status"), + column_values.get("key_added_at"), ] with closing(self._connections()) as conn: @@ -223,6 +236,42 @@ class SQLiteDeviceInventoryRepository: cur.execute(f"DELETE FROM {DEVICE_TABLE} WHERE hostname = ?", (hostname,)) conn.commit() + def record_device_fingerprint(self, guid: Optional[str], fingerprint: Optional[str], added_at: str) -> None: + normalized_guid = clean_device_str(guid) + normalized_fp = clean_device_str(fingerprint) + if not normalized_guid or not normalized_fp: + return + + with closing(self._connections()) as conn: + cur = conn.cursor() + cur.execute( + """ + INSERT OR IGNORE INTO device_keys (id, guid, ssl_key_fingerprint, added_at) + VALUES (?, ?, ?, ?) + """, + (str(uuid.uuid4()), normalized_guid, normalized_fp.lower(), added_at), + ) + cur.execute( + """ + UPDATE device_keys + SET retired_at = ? + WHERE guid = ? + AND ssl_key_fingerprint != ? + AND retired_at IS NULL + """, + (added_at, normalized_guid, normalized_fp.lower()), + ) + cur.execute( + """ + UPDATE devices + SET ssl_key_fingerprint = COALESCE(LOWER(?), ssl_key_fingerprint), + key_added_at = COALESCE(key_added_at, ?) + WHERE LOWER(guid) = LOWER(?) + """, + (normalized_fp, added_at, normalized_guid), + ) + conn.commit() + def _extract_device_columns(self, details: Dict[str, Any]) -> Dict[str, Any]: summary = details.get("summary") or {} payload: Dict[str, Any] = {} @@ -250,4 +299,8 @@ class SQLiteDeviceInventoryRepository: payload["connection_endpoint"] = clean_device_str( summary.get("connection_endpoint") or summary.get("endpoint") ) + payload["ssl_key_fingerprint"] = clean_device_str(summary.get("ssl_key_fingerprint")) + payload["token_version"] = coerce_int(summary.get("token_version")) or 0 + payload["status"] = clean_device_str(summary.get("status")) + payload["key_added_at"] = clean_device_str(summary.get("key_added_at")) return payload diff --git a/Data/Engine/services/devices/__init__.py b/Data/Engine/services/devices/__init__.py index d659909..ddef61e 100644 --- a/Data/Engine/services/devices/__init__.py +++ b/Data/Engine/services/devices/__init__.py @@ -1,4 +1,15 @@ -from .device_inventory_service import DeviceInventoryService, RemoteDeviceError +from .device_inventory_service import ( + DeviceDescriptionError, + DeviceDetailsError, + DeviceInventoryService, + RemoteDeviceError, +) from .device_view_service import DeviceViewService -__all__ = ["DeviceInventoryService", "RemoteDeviceError", "DeviceViewService"] +__all__ = [ + "DeviceInventoryService", + "RemoteDeviceError", + "DeviceViewService", + "DeviceDetailsError", + "DeviceDescriptionError", +] diff --git a/Data/Engine/services/devices/device_inventory_service.py b/Data/Engine/services/devices/device_inventory_service.py index ec146f0..9252494 100644 --- a/Data/Engine/services/devices/device_inventory_service.py +++ b/Data/Engine/services/devices/device_inventory_service.py @@ -2,19 +2,27 @@ from __future__ import annotations +import json import logging import sqlite3 import time +from datetime import datetime, timezone from collections.abc import Mapping from typing import Any, Dict, List, Optional from Data.Engine.repositories.sqlite.device_inventory_repository import ( SQLiteDeviceInventoryRepository, ) -from Data.Engine.domain.device_auth import DeviceAuthContext +from Data.Engine.domain.device_auth import DeviceAuthContext, normalize_guid from Data.Engine.domain.devices import clean_device_str, coerce_int -__all__ = ["DeviceInventoryService", "RemoteDeviceError", "DeviceHeartbeatError"] +__all__ = [ + "DeviceInventoryService", + "RemoteDeviceError", + "DeviceHeartbeatError", + "DeviceDetailsError", + "DeviceDescriptionError", +] class RemoteDeviceError(Exception): @@ -29,6 +37,18 @@ class DeviceHeartbeatError(Exception): self.code = code +class DeviceDetailsError(Exception): + def __init__(self, code: str, message: Optional[str] = None) -> None: + super().__init__(message or code) + self.code = code + + +class DeviceDescriptionError(Exception): + def __init__(self, code: str, message: Optional[str] = None) -> None: + super().__init__(message or code) + self.code = code + + class DeviceInventoryService: def __init__( self, @@ -220,7 +240,7 @@ class DeviceInventoryService: summary["hostname"] = hostname if metrics: - last_user = metrics.get("last_user") or metrics.get("username") + last_user = metrics.get("last_user") if last_user: cleaned_user = clean_device_str(last_user) if cleaned_user: @@ -300,3 +320,182 @@ class DeviceInventoryService: ) raise DeviceHeartbeatError("storage_error", "failed to persist heartbeat") from exc + # ------------------------------------------------------------------ + # Agent details + # ------------------------------------------------------------------ + @staticmethod + def _is_empty(value: Any) -> bool: + return value in (None, "", [], {}) + + @classmethod + def _deep_merge_preserve(cls, prev: Dict[str, Any], incoming: Dict[str, Any]) -> Dict[str, Any]: + merged: Dict[str, Any] = dict(prev or {}) + for key, value in (incoming or {}).items(): + if isinstance(value, Mapping): + existing = merged.get(key) + if not isinstance(existing, Mapping): + existing = {} + merged[key] = cls._deep_merge_preserve(dict(existing), dict(value)) + elif isinstance(value, list): + if value: + merged[key] = value + else: + if cls._is_empty(value): + continue + merged[key] = value + return merged + + def save_agent_details( + self, + *, + context: DeviceAuthContext, + payload: Mapping[str, Any], + ) -> None: + hostname = clean_device_str(payload.get("hostname")) + details_raw = payload.get("details") + agent_id = clean_device_str(payload.get("agent_id")) + agent_hash = clean_device_str(payload.get("agent_hash")) + + if not isinstance(details_raw, Mapping): + raise DeviceDetailsError("invalid_payload", "details object required") + + details_dict: Dict[str, Any] + try: + details_dict = json.loads(json.dumps(details_raw)) + except Exception: + details_dict = dict(details_raw) + + incoming_summary = dict(details_dict.get("summary") or {}) + if not hostname: + hostname = clean_device_str(incoming_summary.get("hostname")) + if not hostname: + raise DeviceDetailsError("invalid_payload", "hostname required") + + snapshot = self._repo.load_snapshot(hostname=hostname) + if not snapshot: + snapshot = {} + + previous_details = snapshot.get("details") + if isinstance(previous_details, Mapping): + try: + prev_details = json.loads(json.dumps(previous_details)) + except Exception: + prev_details = dict(previous_details) + else: + prev_details = {} + + prev_summary = dict(prev_details.get("summary") or {}) + + existing_guid = clean_device_str(snapshot.get("guid") or snapshot.get("summary", {}).get("agent_guid")) + normalized_existing_guid = normalize_guid(existing_guid) + auth_guid = context.identity.guid.value + + if normalized_existing_guid and normalized_existing_guid != auth_guid: + raise DeviceDetailsError("guid_mismatch", "device guid mismatch") + + fingerprint = context.identity.fingerprint.value.lower() + stored_fp = clean_device_str(snapshot.get("summary", {}).get("ssl_key_fingerprint")) + if stored_fp and stored_fp.lower() != fingerprint: + raise DeviceDetailsError("fingerprint_mismatch", "device fingerprint mismatch") + + incoming_summary.setdefault("hostname", hostname) + if agent_id and not incoming_summary.get("agent_id"): + incoming_summary["agent_id"] = agent_id + if agent_hash: + incoming_summary["agent_hash"] = agent_hash + incoming_summary["agent_guid"] = auth_guid + if fingerprint: + incoming_summary["ssl_key_fingerprint"] = fingerprint + if not incoming_summary.get("last_seen") and prev_summary.get("last_seen"): + incoming_summary["last_seen"] = prev_summary.get("last_seen") + + details_dict["summary"] = incoming_summary + merged_details = self._deep_merge_preserve(prev_details, details_dict) + merged_summary = merged_details.setdefault("summary", {}) + + if not merged_summary.get("last_user") and prev_summary.get("last_user"): + merged_summary["last_user"] = prev_summary.get("last_user") + + created_at = coerce_int(merged_summary.get("created_at")) + if created_at is None: + created_at = coerce_int(snapshot.get("created_at")) + if created_at is None: + created_at = int(time.time()) + merged_summary["created_at"] = created_at + + if fingerprint: + merged_summary["ssl_key_fingerprint"] = fingerprint + if not merged_summary.get("key_added_at"): + merged_summary["key_added_at"] = datetime.now(timezone.utc).isoformat() + if merged_summary.get("token_version") is None: + merged_summary["token_version"] = 1 + if not merged_summary.get("status") and snapshot.get("summary", {}).get("status"): + merged_summary["status"] = snapshot.get("summary", {}).get("status") + + description = clean_device_str(merged_summary.get("description")) + existing_description = snapshot.get("description") if snapshot else "" + description_to_store = description if description is not None else (existing_description or "") + + existing_hash = clean_device_str(snapshot.get("agent_hash") or snapshot.get("summary", {}).get("agent_hash")) + effective_hash = agent_hash or existing_hash + + try: + self._repo.upsert_device( + hostname, + description_to_store, + merged_details, + created_at, + agent_hash=effective_hash, + guid=auth_guid, + ) + except sqlite3.DatabaseError as exc: + raise DeviceDetailsError("storage_error", str(exc)) from exc + + added_at = merged_summary.get("key_added_at") or datetime.now(timezone.utc).isoformat() + self._repo.record_device_fingerprint(auth_guid, fingerprint, added_at) + + # ------------------------------------------------------------------ + # Description management + # ------------------------------------------------------------------ + def update_device_description(self, hostname: str, description: Optional[str]) -> None: + normalized_host = clean_device_str(hostname) + if not normalized_host: + raise DeviceDescriptionError("invalid_hostname", "invalid hostname") + + snapshot = self._repo.load_snapshot(hostname=normalized_host) + if not snapshot: + raise DeviceDescriptionError("not_found", "device not found") + + details = snapshot.get("details") + if isinstance(details, Mapping): + try: + existing = json.loads(json.dumps(details)) + except Exception: + existing = dict(details) + else: + existing = {} + + summary = dict(existing.get("summary") or {}) + summary["description"] = description or "" + existing["summary"] = summary + + created_at = coerce_int(summary.get("created_at")) + if created_at is None: + created_at = coerce_int(snapshot.get("created_at")) + if created_at is None: + created_at = int(time.time()) + + agent_hash = clean_device_str(summary.get("agent_hash") or snapshot.get("agent_hash")) + guid = clean_device_str(summary.get("agent_guid") or snapshot.get("guid")) + + try: + self._repo.upsert_device( + normalized_host, + description or (snapshot.get("description") or ""), + existing, + created_at, + agent_hash=agent_hash, + guid=guid, + ) + except sqlite3.DatabaseError as exc: + raise DeviceDescriptionError("storage_error", str(exc)) from exc diff --git a/Data/Engine/tests/test_http_agent.py b/Data/Engine/tests/test_http_agent.py index 8ca499e..885ca6c 100644 --- a/Data/Engine/tests/test_http_agent.py +++ b/Data/Engine/tests/test_http_agent.py @@ -232,3 +232,101 @@ def test_script_request_reports_status_and_signing_key(prepared_app, monkeypatch assert resp.get_json()["status"] == "quarantined" assert resp.get_json()["poll_after_ms"] == 60000 + +def test_agent_details_persists_inventory(prepared_app, monkeypatch): + client = prepared_app.test_client() + guid = "5C9D76E4-4C5A-4A5D-9B5D-1C2E3F4A5B6C" + fingerprint = "aa:bb:cc:dd" + hostname = "device-details" + _insert_device(prepared_app, guid, fingerprint, hostname) + + services = prepared_app.extensions["engine_services"] + context = _build_context(guid, fingerprint) + monkeypatch.setattr(services.device_auth, "authenticate", lambda request, path: context) + + payload = { + "hostname": hostname, + "agent_id": "AGENT-01", + "agent_hash": "hash-value", + "details": { + "summary": { + "hostname": hostname, + "device_type": "Laptop", + "last_user": "BUNNY-LAB\\nicole.rappe", + "operating_system": "Windows 11", + "description": "Primary workstation", + }, + "memory": [{"slot": "DIMM0", "capacity": 17179869184}], + "storage": [{"model": "NVMe", "size": 512}], + "network": [{"adapter": "Ethernet", "ips": ["192.168.1.50"]}], + }, + } + + resp = client.post( + "/api/agent/details", + json=payload, + headers={"Authorization": "Bearer token"}, + ) + + assert resp.status_code == 200 + assert resp.get_json() == {"status": "ok"} + + db_path = Path(prepared_app.config["ENGINE_DATABASE_PATH"]) + with sqlite3.connect(db_path) as conn: + row = conn.execute( + """ + SELECT device_type, last_user, memory, storage, network, description + FROM devices + WHERE guid = ? + """, + (guid,), + ).fetchone() + + assert row is not None + device_type, last_user, memory_json, storage_json, network_json, description = row + assert device_type == "Laptop" + assert last_user == "BUNNY-LAB\\nicole.rappe" + assert description == "Primary workstation" + assert json.loads(memory_json)[0]["capacity"] == 17179869184 + assert json.loads(storage_json)[0]["model"] == "NVMe" + assert json.loads(network_json)[0]["ips"][0] == "192.168.1.50" + + +def test_heartbeat_preserves_last_user_from_details(prepared_app, monkeypatch): + client = prepared_app.test_client() + guid = "7E8F90A1-B2C3-4D5E-8F90-A1B2C3D4E5F6" + fingerprint = "11:22:33:44" + hostname = "device-preserve" + _insert_device(prepared_app, guid, fingerprint, hostname) + + services = prepared_app.extensions["engine_services"] + context = _build_context(guid, fingerprint) + monkeypatch.setattr(services.device_auth, "authenticate", lambda request, path: context) + + client.post( + "/api/agent/details", + json={ + "hostname": hostname, + "details": { + "summary": {"hostname": hostname, "last_user": "BUNNY-LAB\\nicole.rappe"} + }, + }, + headers={"Authorization": "Bearer token"}, + ) + + client.post( + "/api/agent/heartbeat", + json={"hostname": hostname, "metrics": {"uptime": 120}}, + headers={"Authorization": "Bearer token"}, + ) + + db_path = Path(prepared_app.config["ENGINE_DATABASE_PATH"]) + with sqlite3.connect(db_path) as conn: + row = conn.execute( + "SELECT last_user FROM devices WHERE guid = ?", + (guid,), + ).fetchone() + + assert row is not None + assert row[0] == "BUNNY-LAB\\nicole.rappe" + diff --git a/Data/Engine/tests/test_http_sites_devices.py b/Data/Engine/tests/test_http_sites_devices.py index 486d82c..0925449 100644 --- a/Data/Engine/tests/test_http_sites_devices.py +++ b/Data/Engine/tests/test_http_sites_devices.py @@ -1,5 +1,6 @@ -import sqlite3 from datetime import datetime, timezone +import sqlite3 +import time import pytest @@ -106,3 +107,45 @@ def test_credentials_list_requires_admin(prepared_app): resp = client.get("/api/credentials") assert resp.status_code == 200 assert resp.get_json() == {"credentials": []} + + +def test_device_description_update(prepared_app, engine_settings): + client = prepared_app.test_client() + hostname = "device-desc" + guid = "A3D3F1E5-9B8C-4C6F-80F1-4D5E6F7A8B9C" + + now = int(time.time()) + conn = sqlite3.connect(engine_settings.database.path) + cur = conn.cursor() + cur.execute( + """ + INSERT INTO devices ( + guid, + hostname, + description, + created_at, + last_seen + ) VALUES (?, ?, '', ?, ?) + """, + (guid, hostname, now, now), + ) + conn.commit() + conn.close() + + resp = client.post( + f"/api/device/description/{hostname}", + json={"description": "Primary workstation"}, + ) + + assert resp.status_code == 200 + assert resp.get_json() == {"status": "ok"} + + conn = sqlite3.connect(engine_settings.database.path) + row = conn.execute( + "SELECT description FROM devices WHERE hostname = ?", + (hostname,), + ).fetchone() + conn.close() + + assert row is not None + assert row[0] == "Primary workstation"