From 9b5074ed75e6027614ec367ffebc44f2c94723cc Mon Sep 17 00:00:00 2001 From: Nicole Rappe Date: Sun, 2 Nov 2025 19:24:17 -0700 Subject: [PATCH] Misc Changes --- .gitignore | 14 +- .../Engine/Assemblies/DB_MIGRATION_TRACKER.md | 14 +- Data/Engine/assembly_management/databases.py | 145 ++++++++++++++++++ Data/Engine/assembly_management/sync.py | 10 +- .../services/API/assemblies/management.py | 4 +- Data/Engine/services/assemblies/service.py | 7 +- 6 files changed, 176 insertions(+), 18 deletions(-) diff --git a/.gitignore b/.gitignore index 6897328a..b9004093 100644 --- a/.gitignore +++ b/.gitignore @@ -23,4 +23,16 @@ database.db # Misc Files/Folders .vs/s __pycache__ -/Update_Staging/ \ No newline at end of file +/Update_Staging/ + +# Assembly Databases +/Data/Engine/Assemblies/community.db +/Data/Engine/Assemblies/community.db-shm +/Data/Engine/Assemblies/community.db-wal +/Data/Engine/Assemblies/official.db +/Data/Engine/Assemblies/official.db-shm +/Data/Engine/Assemblies/official.db-wal +/Data/Engine/Assemblies/user_created.db +/Data/Engine/Assemblies/user_created.db-shm +/Data/Engine/Assemblies/user_created.db-wal +/Data/Engine/Assemblies/Payloads/ \ No newline at end of file diff --git a/Data/Engine/Assemblies/DB_MIGRATION_TRACKER.md b/Data/Engine/Assemblies/DB_MIGRATION_TRACKER.md index 58e85edf..372a8cba 100644 --- a/Data/Engine/Assemblies/DB_MIGRATION_TRACKER.md +++ b/Data/Engine/Assemblies/DB_MIGRATION_TRACKER.md @@ -48,10 +48,10 @@ - `initialise_assembly_runtime()` is invoked from both `create_app` and `register_engine_api`, wiring the cache onto `EngineContext` and ensuring graceful shutdown flushing. ## 2. Update Engine services and APIs for multi-domain assemblies -[ ] Refactor existing assembly REST endpoints to read from the cache instead of filesystem JSON. -[ ] Add source metadata (`official`, `community`, `user`) to API responses. -[ ] Introduce administrative endpoints to support Dev Mode overrides: clone between domains, force writes to read-only DBs, bulk sync official DB from staging. -[ ] Provide queue status (dirty vs. persisted) in list/detail responses for UI pill rendering. +[x] Refactor existing assembly REST endpoints to read from the cache instead of filesystem JSON. +[x] Add source metadata (`official`, `community`, `user`) to API responses. +[x] Introduce administrative endpoints to support Dev Mode overrides: clone between domains, force writes to read-only DBs, bulk sync official DB from staging. +[x] Provide queue status (dirty vs. persisted) in list/detail responses for UI pill rendering. ### Details ``` 1. Inspect `Data/Engine/services/assemblies/` (create if absent) and replace filesystem access with calls into `AssemblyCache`. @@ -71,8 +71,10 @@ 6. Adjust error handling to surface concurrency/locking issues with retries and user-friendly messages. ``` -**Stage Notes (in progress)** -- Refactoring work is underway to move REST endpoints onto `AssemblyCache` while we align on assembly GUID usage and domain permissions. Pending operator testing before tasks can be completed. +**Stage Notes** +- Assembly REST stack now proxies through `AssemblyRuntimeService`, sourcing list/detail data from `AssemblyCache` with GUID-based payload resolution (`Data/Engine/services/assemblies/service.py`, `Data/Engine/services/API/assemblies/management.py`). +- Responses include `source`, `is_dirty`, and `payload_guid`, and expose the queue snapshot for dirty-pill rendering; domain guards enforce user vs Admin+Dev Mode write access. +- Added Dev Mode toggle/flush endpoints plus official-domain sync, all wired to the cache write queue and importer; verified via operator testing of the API list/update/clone paths. ## 3. Implement Dev Mode authorization and UX toggles [ ] Gate privileged writes behind Admin role + Dev Mode toggle. diff --git a/Data/Engine/assembly_management/databases.py b/Data/Engine/assembly_management/databases.py index 8e367de7..ed5929e5 100644 --- a/Data/Engine/assembly_management/databases.py +++ b/Data/Engine/assembly_management/databases.py @@ -308,6 +308,7 @@ class AssemblyDatabaseManager: def _apply_schema(self, conn: sqlite3.Connection) -> None: cur = conn.cursor() + self._migrate_legacy_schema(cur) for statement in _SCHEMA_STATEMENTS: cur.execute(statement) conn.commit() @@ -331,3 +332,147 @@ class AssemblyDatabaseManager: runtime_candidate, exc, ) + + def _migrate_legacy_schema(self, cur: sqlite3.Cursor) -> None: + """Upgrade legacy assembly/payload tables to the consolidated schema.""" + + cur.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='assemblies'") + if not cur.fetchone(): + return + + cur.execute("PRAGMA table_info('assemblies')") + legacy_columns = {row[1] for row in cur.fetchall()} + if "assembly_guid" in legacy_columns: + return # Already migrated + + self._logger.info("Migrating legacy assemblies schema to assembly_guid layout.") + cur.execute( + """ + CREATE TABLE IF NOT EXISTS assemblies_new ( + assembly_guid TEXT PRIMARY KEY, + display_name TEXT NOT NULL, + summary TEXT, + category TEXT, + assembly_kind TEXT NOT NULL, + assembly_type TEXT, + version INTEGER NOT NULL DEFAULT 1, + metadata_json TEXT, + tags_json TEXT, + checksum TEXT, + payload_type TEXT NOT NULL, + payload_file_name TEXT NOT NULL, + payload_file_extension TEXT NOT NULL, + payload_size_bytes INTEGER NOT NULL DEFAULT 0, + payload_checksum TEXT, + payload_created_at TEXT NOT NULL, + payload_updated_at TEXT NOT NULL, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL + ) + """ + ) + + cur.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='payloads'") + has_payloads = cur.fetchone() is not None + + if has_payloads: + cur.execute( + """ + INSERT INTO assemblies_new ( + assembly_guid, + display_name, + summary, + category, + assembly_kind, + assembly_type, + version, + metadata_json, + tags_json, + checksum, + payload_type, + payload_file_name, + payload_file_extension, + payload_size_bytes, + payload_checksum, + payload_created_at, + payload_updated_at, + created_at, + updated_at + ) + SELECT + a.assembly_id AS assembly_guid, + a.display_name, + a.summary, + a.category, + a.assembly_kind, + a.assembly_type, + a.version, + COALESCE(a.metadata_json, '{}') AS metadata_json, + COALESCE(a.tags_json, '{}') AS tags_json, + a.checksum, + COALESCE(p.payload_type, 'unknown') AS payload_type, + COALESCE(p.file_name, 'payload.json') AS payload_file_name, + COALESCE(p.file_extension, '.json') AS payload_file_extension, + COALESCE(p.size_bytes, 0) AS payload_size_bytes, + COALESCE(p.checksum, '') AS payload_checksum, + COALESCE(p.created_at, a.created_at) AS payload_created_at, + COALESCE(p.updated_at, a.updated_at) AS payload_updated_at, + a.created_at, + a.updated_at + FROM assemblies AS a + LEFT JOIN payloads AS p ON p.payload_guid = a.payload_guid + """ + ) + else: + cur.execute( + """ + INSERT INTO assemblies_new ( + assembly_guid, + display_name, + summary, + category, + assembly_kind, + assembly_type, + version, + metadata_json, + tags_json, + checksum, + payload_type, + payload_file_name, + payload_file_extension, + payload_size_bytes, + payload_checksum, + payload_created_at, + payload_updated_at, + created_at, + updated_at + ) + SELECT + assembly_id AS assembly_guid, + display_name, + summary, + category, + assembly_kind, + assembly_type, + version, + COALESCE(metadata_json, '{}'), + COALESCE(tags_json, '{}'), + checksum, + 'unknown' AS payload_type, + 'payload.json' AS payload_file_name, + '.json' AS payload_file_extension, + 0 AS payload_size_bytes, + '' AS payload_checksum, + created_at AS payload_created_at, + updated_at AS payload_updated_at, + created_at, + updated_at + FROM assemblies + """ + ) + + cur.execute("DROP TABLE assemblies") + if has_payloads: + cur.execute("DROP TABLE payloads") + cur.execute("ALTER TABLE assemblies_new RENAME TO assemblies") + self._logger.info("Legacy assemblies schema migration completed.") diff --git a/Data/Engine/assembly_management/sync.py b/Data/Engine/assembly_management/sync.py index 3b888726..103439dc 100644 --- a/Data/Engine/assembly_management/sync.py +++ b/Data/Engine/assembly_management/sync.py @@ -110,21 +110,21 @@ def _record_from_file( return None payload_type = _payload_type_for_kind(kind) - guid = hashlib.sha1(rel_path.encode("utf-8")).hexdigest() - descriptor = payload_manager.store_payload(payload_type, text, assembly_guid=guid, extension=".json") + assembly_guid = hashlib.sha1(rel_path.encode("utf-8")).hexdigest() + descriptor = payload_manager.store_payload(payload_type, text, assembly_guid=assembly_guid, extension=".json") file_stat = file_path.stat() timestamp = _dt.datetime.utcfromtimestamp(file_stat.st_mtime).replace(microsecond=0) descriptor.created_at = timestamp descriptor.updated_at = timestamp - assembly_id = _assembly_id_from_path(rel_path) + assembly_path = _assembly_id_from_path(rel_path) document_metadata = _metadata_from_document(kind, document, rel_path) tags = _coerce_dict(document.get("tags")) record = AssemblyRecord( - assembly_id=assembly_id, - display_name=document_metadata.get("display_name") or assembly_id.rsplit("/", 1)[-1], + assembly_guid=assembly_guid, + display_name=document_metadata.get("display_name") or assembly_path.rsplit("/", 1)[-1], summary=document_metadata.get("summary"), category=document_metadata.get("category"), assembly_kind=kind, diff --git a/Data/Engine/services/API/assemblies/management.py b/Data/Engine/services/API/assemblies/management.py index f9c9ad92..28ccf9cd 100644 --- a/Data/Engine/services/API/assemblies/management.py +++ b/Data/Engine/services/API/assemblies/management.py @@ -25,8 +25,8 @@ from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple from flask import Blueprint, jsonify, request, session from itsdangerous import BadSignature, SignatureExpired, URLSafeTimedSerializer -from Data.Engine.assembly_management.models import AssemblyDomain -from ..assemblies.service import AssemblyRuntimeService +from ....assembly_management.models import AssemblyDomain +from ...assemblies.service import AssemblyRuntimeService if TYPE_CHECKING: # pragma: no cover - typing aide from .. import EngineServiceAdapters diff --git a/Data/Engine/services/assemblies/service.py b/Data/Engine/services/assemblies/service.py index d69e98a4..8448a90b 100644 --- a/Data/Engine/services/assemblies/service.py +++ b/Data/Engine/services/assemblies/service.py @@ -17,9 +17,9 @@ import logging import uuid from typing import Any, Dict, List, Mapping, Optional -from Data.Engine.assembly_management.bootstrap import AssemblyCache -from Data.Engine.assembly_management.models import AssemblyDomain, AssemblyRecord, CachedAssembly, PayloadType -from Data.Engine.assembly_management.sync import sync_official_domain +from ...assembly_management.bootstrap import AssemblyCache +from ...assembly_management.models import AssemblyDomain, AssemblyRecord, CachedAssembly, PayloadType +from ...assembly_management.sync import sync_official_domain class AssemblyRuntimeService: @@ -329,4 +329,3 @@ def _coerce_int(value: Any, *, default: int = 0) -> int: def _utcnow() -> _dt.datetime: return _dt.datetime.utcnow().replace(microsecond=0) -