From 582960cc32904ef611a4458923cd46ec94f35e4d Mon Sep 17 00:00:00 2001 From: DungTT Date: Fri, 22 May 2026 16:47:51 +0700 Subject: [PATCH] agent --- .gitignore | 7 + .vscode/settings.json | 7 + agent/README.md | 76 +++++ agent/app/__init__.py | 2 + agent/app/api/__init__.py | 2 + agent/app/api/apps.py | 53 +++ agent/app/api/health.py | 38 +++ agent/app/api/services.py | 40 +++ agent/app/api/tasks.py | 74 ++++ agent/app/config.py | 83 +++++ agent/app/core/__init__.py | 2 + agent/app/core/checksum.py | 17 + agent/app/core/command_runner.py | 50 +++ agent/app/core/downloader.py | 42 +++ agent/app/core/installer.py | 30 ++ agent/app/core/manifest_client.py | 18 + agent/app/core/manifest_validator.py | 26 ++ agent/app/core/service_manager.py | 40 +++ agent/app/core/task_runner.py | 248 ++++++++++++++ agent/app/main.py | 37 ++ agent/app/models/__init__.py | 2 + agent/app/models/schemas.py | 196 +++++++++++ agent/app/storage/__init__.py | 2 + agent/app/storage/database.py | 102 ++++++ agent/app/storage/repository.py | 255 ++++++++++++++ agent/app/utils/__init__.py | 2 + agent/app/utils/validators.py | 55 +++ agent/packaging/DEBIAN/control | 10 + agent/packaging/DEBIAN/postinst | 21 ++ agent/packaging/DEBIAN/postrm | 6 + agent/packaging/DEBIAN/prerm | 8 + .../systemd/local-installer-agent.service | 13 + agent/requirements.txt | 4 + agent/scripts/build-deb.sh | 109 ++++++ agent/scripts/install-agent.sh | 23 ++ web-server/public/css/styles.css | 36 +- web-server/server.js | 323 ++++++++++++++++++ web-server/src/repository.js | 93 +++++ web-server/views/agent.ejs | 157 +++++++++ 39 files changed, 2307 insertions(+), 2 deletions(-) create mode 100644 .gitignore create mode 100644 .vscode/settings.json create mode 100644 agent/README.md create mode 100644 agent/app/__init__.py create mode 100644 agent/app/api/__init__.py create mode 100644 agent/app/api/apps.py create mode 100644 agent/app/api/health.py create mode 100644 agent/app/api/services.py create mode 100644 agent/app/api/tasks.py create mode 100644 agent/app/config.py create mode 100644 agent/app/core/__init__.py create mode 100644 agent/app/core/checksum.py create mode 100644 agent/app/core/command_runner.py create mode 100644 agent/app/core/downloader.py create mode 100644 agent/app/core/installer.py create mode 100644 agent/app/core/manifest_client.py create mode 100644 agent/app/core/manifest_validator.py create mode 100644 agent/app/core/service_manager.py create mode 100644 agent/app/core/task_runner.py create mode 100644 agent/app/main.py create mode 100644 agent/app/models/__init__.py create mode 100644 agent/app/models/schemas.py create mode 100644 agent/app/storage/__init__.py create mode 100644 agent/app/storage/database.py create mode 100644 agent/app/storage/repository.py create mode 100644 agent/app/utils/__init__.py create mode 100644 agent/app/utils/validators.py create mode 100644 agent/packaging/DEBIAN/control create mode 100644 agent/packaging/DEBIAN/postinst create mode 100644 agent/packaging/DEBIAN/postrm create mode 100644 agent/packaging/DEBIAN/prerm create mode 100644 agent/packaging/systemd/local-installer-agent.service create mode 100644 agent/requirements.txt create mode 100644 agent/scripts/build-deb.sh create mode 100644 agent/scripts/install-agent.sh create mode 100644 web-server/views/agent.ejs diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1242aef --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +__pycache__/ +*.py[cod] + +agent/.venv/ +agent/build/ + +web-server/uploads/ diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..cf4a57f --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,7 @@ +{ + "python.defaultInterpreterPath": "${workspaceFolder}/agent/.venv/Scripts/python.exe", + "python.analysis.extraPaths": [ + "${workspaceFolder}/agent" + ], + "python.terminal.activateEnvironment": true +} diff --git a/agent/README.md b/agent/README.md new file mode 100644 index 0000000..a04d0d4 --- /dev/null +++ b/agent/README.md @@ -0,0 +1,76 @@ +# Local Installer Agent + +FastAPI service that runs on each Linux client and listens on `127.0.0.1:5010`. + +It accepts install, update, remove, task, log, installed-app, and service-control requests from `robot.installer`. It stores state in local SQLite and installs trusted `.deb` components downloaded from `robot.package`. + +## Development + +```bash +cd agent +python3 -m venv .venv +source .venv/bin/activate +pip install -r requirements.txt +uvicorn app.main:app --host 127.0.0.1 --port 5010 +``` + +## Install script URL + +Client machines should keep using the stable installer command: + +```bash +curl -fsSL https://robot.package/install-agent.sh | sudo bash +``` + +The web server resolves `https://robot.package/packages/agent/latest.deb?arch=` +to the newest uploaded `local-installer-agent__.deb`, so updating the Agent does not require editing the install command. + +Agent packages are uploaded from the web server Admin page at `/agent`. The default storage folder is +`web-server/uploads/packages/agent`, and it can be changed with `AGENT_PACKAGE_DIR`. + +## Important API + +```text +GET /health +GET /system-info +GET /apps/installed +POST /apps/install +POST /apps/update +POST /apps/remove +GET /tasks/{taskId} +GET /tasks/{taskId}/logs +GET /tasks/{taskId}/components +POST /services/start +POST /services/stop +POST /services/restart +GET /services/{serviceName}/status +``` + +`POST /apps/install` supports both: + +```json +{ + "appId": "robot-suite", + "version": "1.0.0" +} +``` + +and a direct single `.deb` payload: + +```json +{ + "appId": "robot-web-app", + "appName": "Robot Web App", + "packageName": "robot-web-app", + "serviceName": "robot-web-app.service", + "version": "1.0.0", + "downloadUrl": "https://robot.package/packages/robot-web-app_1.0.0_amd64.deb", + "checksum": "sha256_hash_here" +} +``` + +For manifest mode, the Agent fetches: + +```text +{ROBOT_PACKAGE_BASE_URL}/api/apps/{appId}/versions/{version}/manifest +``` diff --git a/agent/app/__init__.py b/agent/app/__init__.py new file mode 100644 index 0000000..1bff3a4 --- /dev/null +++ b/agent/app/__init__.py @@ -0,0 +1,2 @@ +"""Local Installer Agent package.""" + diff --git a/agent/app/api/__init__.py b/agent/app/api/__init__.py new file mode 100644 index 0000000..2c30878 --- /dev/null +++ b/agent/app/api/__init__.py @@ -0,0 +1,2 @@ +"""API routers for the Local Installer Agent.""" + diff --git a/agent/app/api/apps.py b/agent/app/api/apps.py new file mode 100644 index 0000000..d057a6c --- /dev/null +++ b/agent/app/api/apps.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +import uuid + +from fastapi import APIRouter, BackgroundTasks, HTTPException + +from app.core.task_runner import TaskRunner +from app.models.schemas import InstallRequest, RemoveRequest, UpdateRequest +from app.storage.repository import Repository + + +router = APIRouter(prefix="/apps", tags=["apps"]) +repository = Repository() +task_runner = TaskRunner(repository) + + +def _task_id(prefix: str) -> str: + return f"task_{prefix}_{uuid.uuid4().hex[:12]}" + + +@router.get("/installed") +def installed_apps() -> list[dict]: + return repository.list_installed_apps() + + +@router.post("/install") +def install_app(request: InstallRequest, background_tasks: BackgroundTasks) -> dict[str, str]: + if not request.version: + raise HTTPException(status_code=400, detail="version is required") + task_id = _task_id("install") + repository.create_task(task_id, "install", request.app_id, request.app_name) + background_tasks.add_task(task_runner.run_install, task_id, request, "install") + return {"taskId": task_id, "status": "queued"} + + +@router.post("/update") +def update_app(request: UpdateRequest, background_tasks: BackgroundTasks) -> dict[str, str]: + if not request.version: + raise HTTPException(status_code=400, detail="version or targetVersion is required") + task_id = _task_id("update") + repository.create_task(task_id, "update", request.app_id, request.app_name) + background_tasks.add_task(task_runner.run_install, task_id, request, "update") + return {"taskId": task_id, "status": "queued"} + + +@router.post("/remove") +def remove_app(request: RemoveRequest, background_tasks: BackgroundTasks) -> dict[str, str]: + if not request.package_name and not repository.list_installed_components(request.app_id): + raise HTTPException(status_code=400, detail="packageName is required when app is not tracked locally") + task_id = _task_id("remove") + repository.create_task(task_id, "remove", request.app_id, request.app_id) + background_tasks.add_task(task_runner.run_remove, task_id, request) + return {"taskId": task_id, "status": "queued"} diff --git a/agent/app/api/health.py b/agent/app/api/health.py new file mode 100644 index 0000000..0ed5441 --- /dev/null +++ b/agent/app/api/health.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +import platform +import shutil +import socket + +from fastapi import APIRouter + +from app.config import settings + + +router = APIRouter() + + +@router.get("/health") +def health() -> dict[str, str]: + return { + "status": "online", + "agentVersion": settings.agent_version, + "hostname": socket.gethostname(), + "os": platform.platform(), + "architecture": platform.machine(), + } + + +@router.get("/system-info") +def system_info() -> dict[str, str]: + disk = shutil.disk_usage("/") + memory_total = "unknown" + return { + "hostname": socket.gethostname(), + "os": platform.platform(), + "kernel": platform.release(), + "architecture": platform.machine(), + "diskFree": f"{disk.free // (1024 ** 3)}GB", + "memoryTotal": memory_total, + } + diff --git a/agent/app/api/services.py b/agent/app/api/services.py new file mode 100644 index 0000000..9702037 --- /dev/null +++ b/agent/app/api/services.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +from fastapi import APIRouter + +from app.core.command_runner import CommandRunner +from app.core.service_manager import ServiceManager +from app.models.schemas import ServiceRequest +from app.storage.repository import Repository + + +router = APIRouter(prefix="/services", tags=["services"]) + + +def _manager() -> ServiceManager: + return ServiceManager(CommandRunner(Repository())) + + +@router.post("/start") +def start_service(request: ServiceRequest) -> dict[str, str]: + _manager().start_service(request.service_name) + return {"serviceName": request.service_name, "status": "started"} + + +@router.post("/stop") +def stop_service(request: ServiceRequest) -> dict[str, str]: + _manager().stop_service(request.service_name) + return {"serviceName": request.service_name, "status": "stopped"} + + +@router.post("/restart") +def restart_service(request: ServiceRequest) -> dict[str, str]: + _manager().restart_service(request.service_name) + return {"serviceName": request.service_name, "status": "restarted"} + + +@router.get("/{service_name}/status") +def service_status(service_name: str) -> dict[str, object]: + request = ServiceRequest(serviceName=service_name) + return _manager().get_service_status(request.service_name) + diff --git a/agent/app/api/tasks.py b/agent/app/api/tasks.py new file mode 100644 index 0000000..fd4b573 --- /dev/null +++ b/agent/app/api/tasks.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +from fastapi import APIRouter, HTTPException + +from app.storage.repository import Repository + + +router = APIRouter(prefix="/tasks", tags=["tasks"]) +repository = Repository() + + +def _task_response(row: dict) -> dict: + return { + "taskId": row["id"], + "type": row["type"], + "appId": row["app_id"], + "appName": row["app_name"], + "status": row["status"], + "progress": row["progress"], + "currentStep": row["current_step"], + "currentComponentId": row["current_component_id"], + "errorMessage": row["error_message"], + "createdAt": row["created_at"], + "startedAt": row["started_at"], + "finishedAt": row["finished_at"], + } + + +@router.get("/{task_id}") +def get_task(task_id: str) -> dict: + task = repository.get_task(task_id) + if not task: + raise HTTPException(status_code=404, detail="Task not found") + return _task_response(task) + + +@router.get("/{task_id}/logs") +def get_task_logs(task_id: str) -> dict: + if not repository.get_task(task_id): + raise HTTPException(status_code=404, detail="Task not found") + return { + "taskId": task_id, + "logs": [ + { + "time": item["timestamp"], + "level": item["level"], + "message": item["message"], + } + for item in repository.get_task_logs(task_id) + ], + } + + +@router.get("/{task_id}/components") +def get_task_components(task_id: str) -> dict: + if not repository.get_task(task_id): + raise HTTPException(status_code=404, detail="Task not found") + return { + "taskId": task_id, + "components": [ + { + "componentId": item["component_id"], + "type": item["type"], + "status": item["status"], + "progress": item["progress"], + "currentStep": item["current_step"], + "errorMessage": item["error_message"], + "startedAt": item["started_at"], + "finishedAt": item["finished_at"], + } + for item in repository.get_task_components(task_id) + ], + } + diff --git a/agent/app/config.py b/agent/app/config.py new file mode 100644 index 0000000..36bd8c2 --- /dev/null +++ b/agent/app/config.py @@ -0,0 +1,83 @@ +from __future__ import annotations + +import os +from dataclasses import dataclass +from functools import lru_cache +from pathlib import Path +from urllib.parse import urlparse + + +def _csv(value: str | None, fallback: list[str]) -> list[str]: + if not value: + return fallback + return [item.strip() for item in value.split(",") if item.strip()] + + +def _default_allowed_download_hosts(base_url: str) -> list[str]: + parsed = urlparse(base_url) + if parsed.hostname: + return [parsed.hostname] + return ["robot.package"] + + +@dataclass(frozen=True) +class Settings: + agent_version: str + host: str + port: int + robot_package_base_url: str + allowed_origins: list[str] + allowed_download_hosts: list[str] + allowed_docker_registries: list[str] + cache_dir: Path + app_dir: Path + log_dir: Path + db_path: Path + allow_remove: bool + allow_purge: bool + allow_docker: bool + allow_docker_compose: bool + command_timeout_seconds: int + + +def _bool(name: str, default: bool) -> bool: + value = os.getenv(name) + if value is None: + return default + return value.strip().lower() in {"1", "true", "yes", "on"} + + +@lru_cache(maxsize=1) +def get_settings() -> Settings: + robot_package_base_url = os.getenv("ROBOT_PACKAGE_BASE_URL", "https://robot.package").rstrip("/") + return Settings( + agent_version=os.getenv("AGENT_VERSION", "0.1.0"), + host=os.getenv("AGENT_HOST", "127.0.0.1"), + port=int(os.getenv("AGENT_PORT", "5010")), + robot_package_base_url=robot_package_base_url, + allowed_origins=_csv( + os.getenv("ALLOWED_ORIGINS"), + ["https://robot.installer", "http://localhost:3000", "http://localhost:5173"], + ), + allowed_download_hosts=_csv( + os.getenv("ALLOWED_DOWNLOAD_HOSTS"), + _default_allowed_download_hosts(robot_package_base_url), + ), + allowed_docker_registries=_csv( + os.getenv("ALLOWED_DOCKER_REGISTRIES"), + ["registry.robot.package"], + ), + cache_dir=Path(os.getenv("CACHE_DIR", "/var/cache/local-installer-agent/packages")), + app_dir=Path(os.getenv("APP_DIR", "/opt/robot-apps")), + log_dir=Path(os.getenv("LOG_DIR", "/var/log/local-installer-agent")), + db_path=Path(os.getenv("DB_PATH", "/var/lib/local-installer-agent/agent.db")), + allow_remove=_bool("ALLOW_REMOVE", True), + allow_purge=_bool("ALLOW_PURGE", False), + allow_docker=_bool("ALLOW_DOCKER", False), + allow_docker_compose=_bool("ALLOW_DOCKER_COMPOSE", False), + command_timeout_seconds=int(os.getenv("COMMAND_TIMEOUT_SECONDS", "900")), + ) + + +settings = get_settings() + diff --git a/agent/app/core/__init__.py b/agent/app/core/__init__.py new file mode 100644 index 0000000..76a342a --- /dev/null +++ b/agent/app/core/__init__.py @@ -0,0 +1,2 @@ +"""Core installers and task orchestration.""" + diff --git a/agent/app/core/checksum.py b/agent/app/core/checksum.py new file mode 100644 index 0000000..20778de --- /dev/null +++ b/agent/app/core/checksum.py @@ -0,0 +1,17 @@ +from __future__ import annotations + +import hashlib +from pathlib import Path + + +def sha256_file(file_path: Path) -> str: + digest = hashlib.sha256() + with file_path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def verify_sha256(file_path: Path, expected: str) -> bool: + return sha256_file(file_path).lower() == expected.lower() + diff --git a/agent/app/core/command_runner.py b/agent/app/core/command_runner.py new file mode 100644 index 0000000..8c18147 --- /dev/null +++ b/agent/app/core/command_runner.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +import subprocess + +from app.config import settings +from app.storage.repository import Repository + + +class CommandError(RuntimeError): + def __init__(self, command: list[str], returncode: int, stdout: str, stderr: str) -> None: + super().__init__(f"Command failed with exit code {returncode}: {' '.join(command)}") + self.command = command + self.returncode = returncode + self.stdout = stdout + self.stderr = stderr + + +class CommandRunner: + def __init__(self, repository: Repository, task_id: str | None = None) -> None: + self.repository = repository + self.task_id = task_id + + def run(self, command: list[str], timeout: int | None = None) -> subprocess.CompletedProcess[str]: + if self.task_id: + self.repository.add_log(self.task_id, "debug", f"Running command: {' '.join(command)}") + + try: + result = subprocess.run( + command, + check=False, + capture_output=True, + text=True, + timeout=timeout or settings.command_timeout_seconds, + ) + except subprocess.TimeoutExpired as error: + if self.task_id: + self.repository.add_log(self.task_id, "error", f"Command timed out: {' '.join(command)}") + raise CommandError(command, 124, error.stdout or "", error.stderr or "") from error + + if self.task_id: + for line in result.stdout.splitlines(): + self.repository.add_log(self.task_id, "debug", line) + for line in result.stderr.splitlines(): + self.repository.add_log(self.task_id, "warning", line) + + if result.returncode != 0: + raise CommandError(command, result.returncode, result.stdout, result.stderr) + + return result + diff --git a/agent/app/core/downloader.py b/agent/app/core/downloader.py new file mode 100644 index 0000000..4bf8cd8 --- /dev/null +++ b/agent/app/core/downloader.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +import re +from pathlib import Path +from urllib.parse import unquote, urlparse + +import httpx + +from app.config import settings +from app.storage.repository import Repository +from app.utils.validators import validate_url_host + + +SAFE_FILE_RE = re.compile(r"[^a-zA-Z0-9._+-]+") + + +def _safe_file_name(url: str) -> str: + parsed = urlparse(url) + name = Path(unquote(parsed.path)).name or "package.deb" + return SAFE_FILE_RE.sub("-", name).strip("-") or "package.deb" + + +class Downloader: + def __init__(self, repository: Repository, task_id: str) -> None: + self.repository = repository + self.task_id = task_id + + def download(self, url: str) -> Path: + validate_url_host(url, settings.allowed_download_hosts) + settings.cache_dir.mkdir(parents=True, exist_ok=True) + destination = settings.cache_dir / _safe_file_name(url) + + self.repository.add_log(self.task_id, "info", f"Downloading {url}") + with httpx.stream("GET", url, follow_redirects=True, timeout=120) as response: + response.raise_for_status() + with destination.open("wb") as handle: + for chunk in response.iter_bytes(): + handle.write(chunk) + + self.repository.add_log(self.task_id, "info", f"Downloaded to {destination}") + return destination + diff --git a/agent/app/core/installer.py b/agent/app/core/installer.py new file mode 100644 index 0000000..132b4e9 --- /dev/null +++ b/agent/app/core/installer.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +from pathlib import Path + +from app.core.command_runner import CommandRunner + + +class DebInstaller: + def __init__(self, command_runner: CommandRunner) -> None: + self.command_runner = command_runner + + def install_deb(self, file_path: Path) -> None: + self.command_runner.run(["apt", "install", "-y", str(file_path)]) + + def remove_package(self, package_name: str, purge: bool = False) -> None: + action = "purge" if purge else "remove" + self.command_runner.run(["apt", action, "-y", package_name]) + + def get_package_version(self, package_name: str) -> str | None: + result = self.command_runner.run(["dpkg-query", "-W", "-f=${Version}", package_name]) + version = result.stdout.strip() + return version or None + + def check_package_installed(self, package_name: str) -> bool: + try: + self.get_package_version(package_name) + return True + except Exception: + return False + diff --git a/agent/app/core/manifest_client.py b/agent/app/core/manifest_client.py new file mode 100644 index 0000000..93a12ba --- /dev/null +++ b/agent/app/core/manifest_client.py @@ -0,0 +1,18 @@ +from __future__ import annotations + +from urllib.parse import quote + +import httpx + +from app.config import settings + + +class ManifestClient: + def fetch_manifest(self, app_id: str, version: str) -> dict: + app_id_part = quote(app_id, safe="") + version_part = quote(version, safe="") + url = f"{settings.robot_package_base_url}/api/apps/{app_id_part}/versions/{version_part}/manifest" + response = httpx.get(url, follow_redirects=True, timeout=30) + response.raise_for_status() + return response.json() + diff --git a/agent/app/core/manifest_validator.py b/agent/app/core/manifest_validator.py new file mode 100644 index 0000000..310af2c --- /dev/null +++ b/agent/app/core/manifest_validator.py @@ -0,0 +1,26 @@ +from __future__ import annotations + +from app.config import settings +from app.models.schemas import AppManifest, DebComponent +from app.utils.validators import validate_url_host + + +class ManifestValidator: + def validate(self, payload: dict) -> dict: + manifest = AppManifest.model_validate(payload).model_dump(by_alias=True) + components = [] + for raw_component in manifest["components"]: + component_type = raw_component.get("type") + if component_type == "deb": + component = DebComponent.model_validate(raw_component).model_dump(by_alias=True) + validate_url_host(component["downloadUrl"], settings.allowed_download_hosts) + components.append(component) + elif component_type == "docker" and not settings.allow_docker: + raise ValueError("Docker components are not enabled on this Agent") + elif component_type == "docker_compose" and not settings.allow_docker_compose: + raise ValueError("Docker Compose components are not enabled on this Agent") + else: + raise ValueError(f"Unsupported component type: {component_type}") + manifest["components"] = sorted(components, key=lambda item: item.get("installOrder", 10)) + return manifest + diff --git a/agent/app/core/service_manager.py b/agent/app/core/service_manager.py new file mode 100644 index 0000000..5aea8ee --- /dev/null +++ b/agent/app/core/service_manager.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +from app.core.command_runner import CommandRunner + + +class ServiceManager: + def __init__(self, command_runner: CommandRunner) -> None: + self.command_runner = command_runner + + def enable_service(self, service_name: str) -> None: + self.command_runner.run(["systemctl", "enable", service_name]) + + def disable_service(self, service_name: str) -> None: + self.command_runner.run(["systemctl", "disable", service_name]) + + def start_service(self, service_name: str) -> None: + self.command_runner.run(["systemctl", "start", service_name]) + + def stop_service(self, service_name: str) -> None: + self.command_runner.run(["systemctl", "stop", service_name]) + + def restart_service(self, service_name: str) -> None: + self.command_runner.run(["systemctl", "restart", service_name]) + + def get_service_status(self, service_name: str) -> dict[str, object]: + active = self._query(["systemctl", "is-active", service_name]) == "active" + enabled = self._query(["systemctl", "is-enabled", service_name]) == "enabled" + return { + "serviceName": service_name, + "active": active, + "enabled": enabled, + "status": "running" if active else "stopped", + } + + def _query(self, command: list[str]) -> str: + try: + return self.command_runner.run(command).stdout.strip() + except Exception: + return "unknown" + diff --git a/agent/app/core/task_runner.py b/agent/app/core/task_runner.py new file mode 100644 index 0000000..7807c09 --- /dev/null +++ b/agent/app/core/task_runner.py @@ -0,0 +1,248 @@ +from __future__ import annotations + +import hashlib +import os +import traceback +from typing import Any + +from app.config import settings +from app.core.checksum import verify_sha256 +from app.core.command_runner import CommandRunner +from app.core.downloader import Downloader +from app.core.installer import DebInstaller +from app.core.manifest_client import ManifestClient +from app.core.manifest_validator import ManifestValidator +from app.core.service_manager import ServiceManager +from app.models.schemas import InstallRequest, RemoveRequest, UpdateRequest +from app.storage.repository import Repository, utc_now + + +class TaskRunner: + def __init__(self, repository: Repository) -> None: + self.repository = repository + self.manifest_client = ManifestClient() + self.manifest_validator = ManifestValidator() + + def run_install(self, task_id: str, request: InstallRequest | UpdateRequest, task_type: str = "install") -> None: + try: + self._mark_started(task_id, f"starting {task_type}") + self._require_root_if_available() + manifest = self._resolve_manifest(request) + self.repository.add_log(task_id, "info", f"Installing {manifest['appId']} {manifest['version']}") + self._install_manifest(task_id, manifest) + manifest_hash = hashlib.sha256( + self.repository.export_manifest_hash(manifest).encode("utf-8") + ).hexdigest() + self.repository.upsert_installed_app( + manifest["appId"], + manifest["appName"], + manifest["version"], + manifest_hash, + ) + self.repository.update_task( + task_id, + status="success", + progress=100, + current_step="completed", + finished_at=utc_now(), + ) + self.repository.add_log(task_id, "info", f"Task {task_id} completed") + except Exception as error: + self._fail_task(task_id, error) + + def run_remove(self, task_id: str, request: RemoveRequest) -> None: + try: + if not settings.allow_remove: + raise ValueError("Remove is disabled on this Agent") + if request.purge and not settings.allow_purge: + raise ValueError("Purge is disabled on this Agent") + self._mark_started(task_id, "starting remove") + self._require_root_if_available() + + components = self.repository.list_installed_components(request.app_id) + if not components and request.package_name: + components = [ + { + "component_id": request.package_name, + "type": "deb", + "install_order": 10, + "package_name": request.package_name, + "service_name": request.service_name, + } + ] + if not components: + raise ValueError("No installed components found for this app") + + command_runner = CommandRunner(self.repository, task_id) + installer = DebInstaller(command_runner) + services = ServiceManager(command_runner) + + ordered = sorted(components, key=lambda item: item["install_order"], reverse=True) + total = len(ordered) + for index, component in enumerate(ordered, start=1): + progress = int((index - 1) / total * 80) + 10 + component_id = component["component_id"] + self.repository.update_task( + task_id, + progress=progress, + current_step=f"removing {component_id}", + current_component_id=component_id, + ) + service_name = component.get("service_name") + if service_name: + self.repository.add_log(task_id, "info", f"Stopping service {service_name}") + services.stop_service(service_name) + services.disable_service(service_name) + + package_name = component.get("package_name") + if component["type"] == "deb" and package_name: + self.repository.add_log(task_id, "info", f"Removing package {package_name}") + installer.remove_package(package_name, purge=request.purge) + else: + raise ValueError(f"Unsupported installed component type: {component['type']}") + + self.repository.delete_installed_app(request.app_id) + self.repository.update_task( + task_id, + status="success", + progress=100, + current_step="completed", + finished_at=utc_now(), + ) + except Exception as error: + self._fail_task(task_id, error) + + def _resolve_manifest(self, request: InstallRequest | UpdateRequest) -> dict[str, Any]: + if request.download_url: + digest = request.sha256 or request.checksum + return self.manifest_validator.validate( + { + "schemaVersion": "1.0", + "appId": request.app_id, + "appName": request.app_name or request.app_id, + "version": request.version, + "architecture": "amd64", + "components": [ + { + "componentId": request.package_name, + "type": "deb", + "installOrder": 10, + "required": True, + "packageName": request.package_name, + "version": request.version, + "downloadUrl": request.download_url, + "sha256": digest, + "serviceName": request.service_name, + } + ], + } + ) + + payload = self.manifest_client.fetch_manifest(request.app_id, request.version) + return self.manifest_validator.validate(payload) + + def _install_manifest(self, task_id: str, manifest: dict[str, Any]) -> None: + components = manifest["components"] + for component in components: + self.repository.create_task_component( + task_id, + manifest["appId"], + component["componentId"], + component["type"], + component.get("installOrder", 10), + ) + + total = len(components) + if total == 0: + raise ValueError("Manifest has no installable components") + + for index, component in enumerate(components, start=1): + base_progress = int((index - 1) / total * 80) + 10 + component_id = component["componentId"] + self.repository.update_task( + task_id, + progress=base_progress, + current_step=f"installing {component_id}", + current_component_id=component_id, + ) + self.repository.update_task_component( + task_id, + component_id, + status="running", + progress=5, + current_step="downloading", + started_at=utc_now(), + ) + + if component["type"] != "deb": + raise ValueError(f"Unsupported component type in MVP: {component['type']}") + + self._install_deb_component(task_id, manifest["appId"], component) + + self.repository.update_task_component( + task_id, + component_id, + status="success", + progress=100, + current_step="completed", + finished_at=utc_now(), + ) + + def _install_deb_component(self, task_id: str, app_id: str, component: dict[str, Any]) -> None: + component_id = component["componentId"] + downloader = Downloader(self.repository, task_id) + command_runner = CommandRunner(self.repository, task_id) + installer = DebInstaller(command_runner) + services = ServiceManager(command_runner) + + package_path = downloader.download(component["downloadUrl"]) + self.repository.update_task_component(task_id, component_id, progress=35, current_step="verifying checksum") + if not verify_sha256(package_path, component["sha256"]): + raise ValueError(f"Checksum mismatch for {component_id}") + self.repository.add_log(task_id, "info", f"Checksum verified for {component_id}") + + self.repository.update_task_component(task_id, component_id, progress=60, current_step="installing package") + installer.install_deb(package_path) + + self.repository.update_task_component(task_id, component_id, progress=75, current_step="verifying package") + installed_version = installer.get_package_version(component["packageName"]) + self.repository.add_log( + task_id, + "info", + f"Package {component['packageName']} installed with version {installed_version}", + ) + + service_name = component.get("serviceName") + if service_name: + self.repository.update_task_component(task_id, component_id, progress=90, current_step="starting service") + services.enable_service(service_name) + services.start_service(service_name) + + self.repository.upsert_installed_component(app_id, component) + + def _mark_started(self, task_id: str, step: str) -> None: + self.repository.update_task( + task_id, + status="running", + progress=5, + current_step=step, + started_at=utc_now(), + ) + self.repository.add_log(task_id, "info", step) + + def _fail_task(self, task_id: str, error: Exception) -> None: + self.repository.update_task( + task_id, + status="failed", + current_step="failed", + error_message=str(error), + finished_at=utc_now(), + ) + self.repository.add_log(task_id, "error", str(error)) + self.repository.add_log(task_id, "debug", traceback.format_exc()) + + def _require_root_if_available(self) -> None: + geteuid = getattr(os, "geteuid", None) + if callable(geteuid) and geteuid() != 0: + raise PermissionError("Agent must run as root to call apt and systemctl") + diff --git a/agent/app/main.py b/agent/app/main.py new file mode 100644 index 0000000..4c418c6 --- /dev/null +++ b/agent/app/main.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +from contextlib import asynccontextmanager + +from fastapi import FastAPI +from fastapi.middleware.cors import CORSMiddleware + +from app.api import apps, health, services, tasks +from app.config import settings +from app.storage.database import initialize_database + + +@asynccontextmanager +async def lifespan(app: FastAPI): + initialize_database() + yield + + +app = FastAPI( + title="Local Installer Agent", + version=settings.agent_version, + lifespan=lifespan, +) + +app.add_middleware( + CORSMiddleware, + allow_origins=settings.allowed_origins, + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) + +app.include_router(health.router) +app.include_router(apps.router) +app.include_router(tasks.router) +app.include_router(services.router) + diff --git a/agent/app/models/__init__.py b/agent/app/models/__init__.py new file mode 100644 index 0000000..fd97084 --- /dev/null +++ b/agent/app/models/__init__.py @@ -0,0 +1,2 @@ +"""Pydantic models for API contracts.""" + diff --git a/agent/app/models/schemas.py b/agent/app/models/schemas.py new file mode 100644 index 0000000..7ad290e --- /dev/null +++ b/agent/app/models/schemas.py @@ -0,0 +1,196 @@ +from __future__ import annotations + +from typing import Any, Literal + +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator + +from app.utils.validators import ( + validate_app_id, + validate_package_name, + validate_service_name, + validate_sha256, + validate_version, +) + + +TaskType = Literal["install", "update", "remove"] +TaskStatus = Literal["queued", "running", "success", "failed", "cancelled"] +ComponentType = Literal["deb", "docker", "docker_compose"] + + +class CamelModel(BaseModel): + model_config = ConfigDict(populate_by_name=True, extra="forbid") + + +class HealthResponse(CamelModel): + status: str + agent_version: str = Field(alias="agentVersion") + hostname: str + os: str + architecture: str + + +class SystemInfoResponse(CamelModel): + hostname: str + os: str + kernel: str + architecture: str + disk_free: str = Field(alias="diskFree") + memory_total: str = Field(alias="memoryTotal") + + +class InstallRequest(CamelModel): + app_id: str = Field(alias="appId") + version: str | None = None + app_name: str | None = Field(default=None, alias="appName") + package_name: str | None = Field(default=None, alias="packageName") + service_name: str | None = Field(default=None, alias="serviceName") + download_url: str | None = Field(default=None, alias="downloadUrl") + checksum: str | None = None + sha256: str | None = None + + @field_validator("app_id") + @classmethod + def _app_id(cls, value: str) -> str: + return validate_app_id(value) + + @field_validator("version") + @classmethod + def _version(cls, value: str | None) -> str | None: + return validate_version(value) if value else None + + @field_validator("package_name") + @classmethod + def _package_name(cls, value: str | None) -> str | None: + return validate_package_name(value) if value else None + + @field_validator("service_name") + @classmethod + def _service_name(cls, value: str | None) -> str | None: + return validate_service_name(value) + + @model_validator(mode="after") + def _direct_package_fields(self) -> "InstallRequest": + digest = self.sha256 or self.checksum + if digest: + normalized = validate_sha256(digest) + self.sha256 = normalized + self.checksum = normalized + + has_direct_package = any([self.download_url, self.package_name, self.checksum, self.sha256]) + if has_direct_package and not all([self.download_url, self.package_name, self.checksum or self.sha256]): + raise ValueError("direct install requires packageName, downloadUrl, and checksum/sha256") + return self + + +class UpdateRequest(InstallRequest): + current_version: str | None = Field(default=None, alias="currentVersion") + target_version: str | None = Field(default=None, alias="targetVersion") + + @model_validator(mode="after") + def _normalize_target(self) -> "UpdateRequest": + if self.target_version: + self.target_version = validate_version(self.target_version) + self.version = self.target_version + return self + + +class RemoveRequest(CamelModel): + app_id: str = Field(alias="appId") + package_name: str | None = Field(default=None, alias="packageName") + service_name: str | None = Field(default=None, alias="serviceName") + purge: bool = False + + @field_validator("app_id") + @classmethod + def _app_id(cls, value: str) -> str: + return validate_app_id(value) + + @field_validator("package_name") + @classmethod + def _package_name(cls, value: str | None) -> str | None: + return validate_package_name(value) if value else None + + @field_validator("service_name") + @classmethod + def _service_name(cls, value: str | None) -> str | None: + return validate_service_name(value) + + +class ServiceRequest(CamelModel): + service_name: str = Field(alias="serviceName") + + @field_validator("service_name") + @classmethod + def _service_name(cls, value: str) -> str: + return validate_service_name(value) or value + + +class TaskQueuedResponse(CamelModel): + task_id: str = Field(alias="taskId") + status: str + + +class DebComponent(CamelModel): + component_id: str = Field(alias="componentId") + type: Literal["deb"] = "deb" + install_order: int = Field(default=10, alias="installOrder") + required: bool = True + package_name: str = Field(alias="packageName") + version: str + download_url: str = Field(alias="downloadUrl") + sha256: str + service_name: str | None = Field(default=None, alias="serviceName") + + @field_validator("component_id") + @classmethod + def _component_id(cls, value: str) -> str: + return validate_app_id(value) + + @field_validator("package_name") + @classmethod + def _package_name(cls, value: str) -> str: + return validate_package_name(value) + + @field_validator("version") + @classmethod + def _version(cls, value: str) -> str: + return validate_version(value) + + @field_validator("sha256") + @classmethod + def _sha256(cls, value: str) -> str: + return validate_sha256(value) + + @field_validator("service_name") + @classmethod + def _service_name(cls, value: str | None) -> str | None: + return validate_service_name(value) + + +class RawComponent(CamelModel): + component_id: str = Field(alias="componentId") + type: ComponentType + install_order: int = Field(default=10, alias="installOrder") + required: bool = True + payload: dict[str, Any] = Field(default_factory=dict) + + +class AppManifest(CamelModel): + schema_version: str = Field(default="1.0", alias="schemaVersion") + app_id: str = Field(alias="appId") + app_name: str = Field(alias="appName") + version: str + architecture: str = "amd64" + components: list[dict[str, Any]] + signature: str | None = None + + @field_validator("app_id") + @classmethod + def _app_id(cls, value: str) -> str: + return validate_app_id(value) + + @field_validator("version") + @classmethod + def _version(cls, value: str) -> str: + return validate_version(value) diff --git a/agent/app/storage/__init__.py b/agent/app/storage/__init__.py new file mode 100644 index 0000000..c5c4f3f --- /dev/null +++ b/agent/app/storage/__init__.py @@ -0,0 +1,2 @@ +"""SQLite storage layer.""" + diff --git a/agent/app/storage/database.py b/agent/app/storage/database.py new file mode 100644 index 0000000..b3b5c2f --- /dev/null +++ b/agent/app/storage/database.py @@ -0,0 +1,102 @@ +from __future__ import annotations + +import sqlite3 +from pathlib import Path + +from app.config import settings + + +SCHEMA = """ +PRAGMA journal_mode = WAL; +PRAGMA foreign_keys = ON; + +CREATE TABLE IF NOT EXISTS tasks ( + id TEXT PRIMARY KEY, + type TEXT NOT NULL, + app_id TEXT, + app_name TEXT, + status TEXT NOT NULL, + progress INTEGER DEFAULT 0, + current_step TEXT, + current_component_id TEXT, + error_message TEXT, + created_at TEXT NOT NULL, + started_at TEXT, + finished_at TEXT +); + +CREATE TABLE IF NOT EXISTS task_logs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + task_id TEXT NOT NULL, + timestamp TEXT NOT NULL, + level TEXT NOT NULL, + message TEXT NOT NULL, + FOREIGN KEY (task_id) REFERENCES tasks(id) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS installed_apps ( + app_id TEXT PRIMARY KEY, + app_name TEXT NOT NULL, + version TEXT NOT NULL, + manifest_hash TEXT, + status TEXT NOT NULL, + installed_at TEXT NOT NULL, + updated_at TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS installed_components ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + app_id TEXT NOT NULL, + component_id TEXT NOT NULL, + type TEXT NOT NULL, + install_order INTEGER NOT NULL, + status TEXT NOT NULL, + package_name TEXT, + package_version TEXT, + service_name TEXT, + docker_image TEXT, + docker_digest TEXT, + container_name TEXT, + compose_project_name TEXT, + installed_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + UNIQUE(app_id, component_id) +); + +CREATE TABLE IF NOT EXISTS task_components ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + task_id TEXT NOT NULL, + app_id TEXT NOT NULL, + component_id TEXT NOT NULL, + type TEXT NOT NULL, + install_order INTEGER NOT NULL, + status TEXT NOT NULL, + progress INTEGER DEFAULT 0, + current_step TEXT, + error_message TEXT, + started_at TEXT, + finished_at TEXT, + FOREIGN KEY (task_id) REFERENCES tasks(id) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS agent_config ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL +); +""" + + +def get_connection() -> sqlite3.Connection: + connection = sqlite3.connect(settings.db_path, timeout=30) + connection.row_factory = sqlite3.Row + return connection + + +def initialize_database() -> None: + db_path = Path(settings.db_path) + db_path.parent.mkdir(parents=True, exist_ok=True) + settings.cache_dir.mkdir(parents=True, exist_ok=True) + settings.log_dir.mkdir(parents=True, exist_ok=True) + with get_connection() as connection: + connection.executescript(SCHEMA) + diff --git a/agent/app/storage/repository.py b/agent/app/storage/repository.py new file mode 100644 index 0000000..88fdb3e --- /dev/null +++ b/agent/app/storage/repository.py @@ -0,0 +1,255 @@ +from __future__ import annotations + +import json +from datetime import datetime, timezone +from typing import Any + +from app.storage.database import get_connection + + +def utc_now() -> str: + return datetime.now(timezone.utc).isoformat(timespec="seconds").replace("+00:00", "Z") + + +def row_to_dict(row: Any) -> dict[str, Any] | None: + if row is None: + return None + return dict(row) + + +class Repository: + def create_task(self, task_id: str, task_type: str, app_id: str, app_name: str | None) -> None: + now = utc_now() + with get_connection() as connection: + connection.execute( + """ + INSERT INTO tasks (id, type, app_id, app_name, status, progress, current_step, created_at) + VALUES (?, ?, ?, ?, 'queued', 0, 'queued', ?) + """, + (task_id, task_type, app_id, app_name, now), + ) + connection.execute( + "INSERT INTO task_logs (task_id, timestamp, level, message) VALUES (?, ?, 'info', ?)", + (task_id, now, f"Task {task_id} queued"), + ) + + def update_task( + self, + task_id: str, + *, + status: str | None = None, + progress: int | None = None, + current_step: str | None = None, + current_component_id: str | None = None, + error_message: str | None = None, + started_at: str | None = None, + finished_at: str | None = None, + ) -> None: + fields: list[str] = [] + values: list[Any] = [] + for key, value in { + "status": status, + "progress": progress, + "current_step": current_step, + "current_component_id": current_component_id, + "error_message": error_message, + "started_at": started_at, + "finished_at": finished_at, + }.items(): + if value is not None: + fields.append(f"{key} = ?") + values.append(value) + if not fields: + return + values.append(task_id) + with get_connection() as connection: + connection.execute(f"UPDATE tasks SET {', '.join(fields)} WHERE id = ?", values) + + def add_log(self, task_id: str, level: str, message: str) -> None: + with get_connection() as connection: + connection.execute( + "INSERT INTO task_logs (task_id, timestamp, level, message) VALUES (?, ?, ?, ?)", + (task_id, utc_now(), level, message), + ) + + def get_task(self, task_id: str) -> dict[str, Any] | None: + with get_connection() as connection: + row = connection.execute("SELECT * FROM tasks WHERE id = ?", (task_id,)).fetchone() + return row_to_dict(row) + + def get_task_logs(self, task_id: str) -> list[dict[str, Any]]: + with get_connection() as connection: + rows = connection.execute( + """ + SELECT timestamp, level, message + FROM task_logs + WHERE task_id = ? + ORDER BY id ASC + """, + (task_id,), + ).fetchall() + return [dict(row) for row in rows] + + def create_task_component( + self, + task_id: str, + app_id: str, + component_id: str, + component_type: str, + install_order: int, + ) -> None: + with get_connection() as connection: + connection.execute( + """ + INSERT INTO task_components ( + task_id, app_id, component_id, type, install_order, status, progress, current_step + ) + VALUES (?, ?, ?, ?, ?, 'queued', 0, 'queued') + """, + (task_id, app_id, component_id, component_type, install_order), + ) + + def update_task_component( + self, + task_id: str, + component_id: str, + *, + status: str | None = None, + progress: int | None = None, + current_step: str | None = None, + error_message: str | None = None, + started_at: str | None = None, + finished_at: str | None = None, + ) -> None: + fields: list[str] = [] + values: list[Any] = [] + for key, value in { + "status": status, + "progress": progress, + "current_step": current_step, + "error_message": error_message, + "started_at": started_at, + "finished_at": finished_at, + }.items(): + if value is not None: + fields.append(f"{key} = ?") + values.append(value) + if not fields: + return + values.extend([task_id, component_id]) + with get_connection() as connection: + connection.execute( + f"UPDATE task_components SET {', '.join(fields)} WHERE task_id = ? AND component_id = ?", + values, + ) + + def get_task_components(self, task_id: str) -> list[dict[str, Any]]: + with get_connection() as connection: + rows = connection.execute( + """ + SELECT component_id, type, status, progress, current_step, error_message, started_at, finished_at + FROM task_components + WHERE task_id = ? + ORDER BY install_order ASC, id ASC + """, + (task_id,), + ).fetchall() + return [dict(row) for row in rows] + + def list_installed_apps(self) -> list[dict[str, Any]]: + with get_connection() as connection: + rows = connection.execute( + """ + SELECT app_id, app_name, version, manifest_hash, status, installed_at, updated_at + FROM installed_apps + ORDER BY app_name ASC + """ + ).fetchall() + return [dict(row) for row in rows] + + def upsert_installed_app( + self, + app_id: str, + app_name: str, + version: str, + manifest_hash: str | None, + status: str = "installed", + ) -> None: + now = utc_now() + with get_connection() as connection: + connection.execute( + """ + INSERT INTO installed_apps (app_id, app_name, version, manifest_hash, status, installed_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(app_id) DO UPDATE SET + app_name = excluded.app_name, + version = excluded.version, + manifest_hash = excluded.manifest_hash, + status = excluded.status, + updated_at = excluded.updated_at + """, + (app_id, app_name, version, manifest_hash, status, now, now), + ) + + def delete_installed_app(self, app_id: str) -> None: + with get_connection() as connection: + connection.execute("DELETE FROM installed_components WHERE app_id = ?", (app_id,)) + connection.execute("DELETE FROM installed_apps WHERE app_id = ?", (app_id,)) + + def upsert_installed_component(self, app_id: str, component: dict[str, Any]) -> None: + now = utc_now() + with get_connection() as connection: + connection.execute( + """ + INSERT INTO installed_components ( + app_id, component_id, type, install_order, status, package_name, package_version, + service_name, docker_image, docker_digest, container_name, compose_project_name, + installed_at, updated_at + ) + VALUES (?, ?, ?, ?, 'installed', ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(app_id, component_id) DO UPDATE SET + type = excluded.type, + install_order = excluded.install_order, + status = excluded.status, + package_name = excluded.package_name, + package_version = excluded.package_version, + service_name = excluded.service_name, + docker_image = excluded.docker_image, + docker_digest = excluded.docker_digest, + container_name = excluded.container_name, + compose_project_name = excluded.compose_project_name, + updated_at = excluded.updated_at + """, + ( + app_id, + component["componentId"], + component["type"], + component.get("installOrder", 10), + component.get("packageName"), + component.get("version"), + component.get("serviceName"), + component.get("image"), + component.get("digest"), + component.get("containerName"), + component.get("projectName"), + now, + now, + ), + ) + + def list_installed_components(self, app_id: str) -> list[dict[str, Any]]: + with get_connection() as connection: + rows = connection.execute( + """ + SELECT * + FROM installed_components + WHERE app_id = ? + ORDER BY install_order ASC, id ASC + """, + (app_id,), + ).fetchall() + return [dict(row) for row in rows] + + def export_manifest_hash(self, manifest: dict[str, Any]) -> str: + return json.dumps(manifest, sort_keys=True, separators=(",", ":")) + diff --git a/agent/app/utils/__init__.py b/agent/app/utils/__init__.py new file mode 100644 index 0000000..33dafe6 --- /dev/null +++ b/agent/app/utils/__init__.py @@ -0,0 +1,2 @@ +"""Utility helpers.""" + diff --git a/agent/app/utils/validators.py b/agent/app/utils/validators.py new file mode 100644 index 0000000..ab5e847 --- /dev/null +++ b/agent/app/utils/validators.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +import re +from urllib.parse import urlparse + + +APP_ID_RE = re.compile(r"^[a-zA-Z0-9._+-]+$") +PACKAGE_NAME_RE = re.compile(r"^[a-zA-Z0-9._+-]+$") +SERVICE_NAME_RE = re.compile(r"^[a-zA-Z0-9._@+-]+\.service$") +VERSION_RE = re.compile(r"^[a-zA-Z0-9._:+~=-]+$") +SHA256_RE = re.compile(r"^[0-9a-fA-F]{64}$") + + +def validate_app_id(value: str) -> str: + if not value or not APP_ID_RE.fullmatch(value): + raise ValueError("appId contains invalid characters") + return value + + +def validate_package_name(value: str) -> str: + if not value or not PACKAGE_NAME_RE.fullmatch(value): + raise ValueError("packageName contains invalid characters") + return value + + +def validate_service_name(value: str | None) -> str | None: + if value is None or value == "": + return None + if not SERVICE_NAME_RE.fullmatch(value): + raise ValueError("serviceName must be a systemd .service name") + return value + + +def validate_version(value: str) -> str: + if not value or not VERSION_RE.fullmatch(value): + raise ValueError("version contains invalid characters") + return value + + +def validate_sha256(value: str) -> str: + if not value or not SHA256_RE.fullmatch(value): + raise ValueError("sha256/checksum must be a 64 character hex digest") + return value.lower() + + +def validate_url_host(url: str, allowed_hosts: list[str]) -> str: + parsed = urlparse(url) + if parsed.scheme not in {"http", "https"}: + raise ValueError("download URL must use http or https") + if not parsed.hostname: + raise ValueError("download URL is missing a host") + if parsed.hostname not in set(allowed_hosts): + raise ValueError(f"download host is not allowed: {parsed.hostname}") + return url + diff --git a/agent/packaging/DEBIAN/control b/agent/packaging/DEBIAN/control new file mode 100644 index 0000000..5cf1599 --- /dev/null +++ b/agent/packaging/DEBIAN/control @@ -0,0 +1,10 @@ +Package: local-installer-agent +Version: 0.1.0 +Section: utils +Priority: optional +Architecture: amd64 +Maintainer: Robot Team +Depends: python3, python3-venv, python3-pip, curl +Description: Local Installer Agent for robot.installer + A local background service that installs, updates, and removes trusted .deb apps + from robot.package on the user's Linux machine. diff --git a/agent/packaging/DEBIAN/postinst b/agent/packaging/DEBIAN/postinst new file mode 100644 index 0000000..b4482ea --- /dev/null +++ b/agent/packaging/DEBIAN/postinst @@ -0,0 +1,21 @@ +#!/bin/bash +set -e + +mkdir -p /var/lib/local-installer-agent +mkdir -p /var/log/local-installer-agent +mkdir -p /var/cache/local-installer-agent/packages + +cd /opt/local-installer-agent + +if [ ! -d "venv" ]; then + python3 -m venv venv +fi + +./venv/bin/pip install --upgrade pip +./venv/bin/pip install -r requirements.txt + +systemctl daemon-reload +systemctl enable local-installer-agent +systemctl restart local-installer-agent + +exit 0 diff --git a/agent/packaging/DEBIAN/postrm b/agent/packaging/DEBIAN/postrm new file mode 100644 index 0000000..c252664 --- /dev/null +++ b/agent/packaging/DEBIAN/postrm @@ -0,0 +1,6 @@ +#!/bin/bash +set -e + +systemctl daemon-reload + +exit 0 diff --git a/agent/packaging/DEBIAN/prerm b/agent/packaging/DEBIAN/prerm new file mode 100644 index 0000000..edd84d5 --- /dev/null +++ b/agent/packaging/DEBIAN/prerm @@ -0,0 +1,8 @@ +#!/bin/bash +set -e + +if systemctl is-active --quiet local-installer-agent; then + systemctl stop local-installer-agent +fi + +exit 0 diff --git a/agent/packaging/systemd/local-installer-agent.service b/agent/packaging/systemd/local-installer-agent.service new file mode 100644 index 0000000..d870d60 --- /dev/null +++ b/agent/packaging/systemd/local-installer-agent.service @@ -0,0 +1,13 @@ +[Unit] +Description=Local Installer Agent +After=network.target + +[Service] +WorkingDirectory=/opt/local-installer-agent +ExecStart=/opt/local-installer-agent/venv/bin/python -m uvicorn app.main:app --host 127.0.0.1 --port 5010 +Restart=always +User=root +EnvironmentFile=/etc/local-installer-agent/agent.env + +[Install] +WantedBy=multi-user.target diff --git a/agent/requirements.txt b/agent/requirements.txt new file mode 100644 index 0000000..cc20873 --- /dev/null +++ b/agent/requirements.txt @@ -0,0 +1,4 @@ +fastapi>=0.115,<1.0 +uvicorn[standard]>=0.30,<1.0 +pydantic>=2,<3 +httpx>=0.27,<1.0 diff --git a/agent/scripts/build-deb.sh b/agent/scripts/build-deb.sh new file mode 100644 index 0000000..5000911 --- /dev/null +++ b/agent/scripts/build-deb.sh @@ -0,0 +1,109 @@ +#!/usr/bin/env bash +set -euo pipefail + +PKG_NAME="local-installer-agent" +ARCH="${ARCH:-amd64}" +PUBLISH_DIR="${AGENT_PUBLISH_DIR:-../web-server/uploads/packages/agent}" + +if [ -z "${BUILD_ROOT:-}" ]; then + if [[ "$(pwd -P)" == /mnt/* ]]; then + BUILD_ROOT="/tmp/${PKG_NAME}-build" + else + BUILD_ROOT="build" + fi +fi + +next_patch_version() { + local latest="" + + if [ -d "${PUBLISH_DIR}" ]; then + latest="$( + for package_path in "${PUBLISH_DIR}/${PKG_NAME}_"*"_${ARCH}.deb"; do + [ -e "${package_path}" ] || continue + package_file="$(basename "${package_path}")" + package_version="${package_file#${PKG_NAME}_}" + package_version="${package_version%_${ARCH}.deb}" + printf '%s\n' "${package_version}" + done | sort -V | tail -n 1 + )" + fi + + if [ -z "${latest}" ]; then + latest="$( + sed -nE 's/^Version:[[:space:]]*([^[:space:]]+).*/\1/p' packaging/DEBIAN/control | + head -n 1 + )" + fi + + if [[ "${latest}" =~ ^([0-9]+)\.([0-9]+)\.([0-9]+)$ ]]; then + printf '%s.%s.%s\n' "${BASH_REMATCH[1]}" "${BASH_REMATCH[2]}" "$((BASH_REMATCH[3] + 1))" + return + fi + + printf '0.1.0\n' +} + +VERSION="${VERSION:-$(next_patch_version)}" +BUILD_DIR="${BUILD_ROOT}/${PKG_NAME}_${VERSION}_${ARCH}" + +rm -rf "${BUILD_ROOT}" + +mkdir -p "${BUILD_DIR}/opt/local-installer-agent" +mkdir -p "${BUILD_DIR}/etc/local-installer-agent" +mkdir -p "${BUILD_DIR}/etc/systemd/system" +mkdir -p "${BUILD_DIR}/DEBIAN" + +cp -r app "${BUILD_DIR}/opt/local-installer-agent/" +cp requirements.txt "${BUILD_DIR}/opt/local-installer-agent/" + +find "${BUILD_DIR}/opt/local-installer-agent/app" -type d -name "__pycache__" -prune -exec rm -rf {} + +find "${BUILD_DIR}/opt/local-installer-agent/app" -type f \( -name "*.pyc" -o -name "*.pyo" \) -delete +find "${BUILD_DIR}/opt/local-installer-agent" -type d -exec chmod 755 {} + +find "${BUILD_DIR}/opt/local-installer-agent" -type f -exec chmod 644 {} + + +cp packaging/systemd/local-installer-agent.service \ + "${BUILD_DIR}/etc/systemd/system/local-installer-agent.service" + +cp packaging/DEBIAN/control "${BUILD_DIR}/DEBIAN/control" +cp packaging/DEBIAN/postinst "${BUILD_DIR}/DEBIAN/postinst" +cp packaging/DEBIAN/prerm "${BUILD_DIR}/DEBIAN/prerm" +cp packaging/DEBIAN/postrm "${BUILD_DIR}/DEBIAN/postrm" + +chmod 755 "${BUILD_DIR}/DEBIAN/postinst" +chmod 755 "${BUILD_DIR}/DEBIAN/prerm" +chmod 755 "${BUILD_DIR}/DEBIAN/postrm" +chmod 644 "${BUILD_DIR}/DEBIAN/control" +chmod 644 "${BUILD_DIR}/etc/systemd/system/local-installer-agent.service" + +sed -i -E "s/^Version:.*/Version: ${VERSION}/" "${BUILD_DIR}/DEBIAN/control" +sed -i -E "s/^Architecture:.*/Architecture: ${ARCH}/" "${BUILD_DIR}/DEBIAN/control" + +cat > "${BUILD_DIR}/etc/local-installer-agent/agent.env" < { + const arch = normalizeAgentArch(req.query.arch); + const latestPackage = await findLatestAgentPackage(arch); + + if (latestPackage) { + res.type('application/vnd.debian.binary-package'); + res.setHeader('Content-Disposition', `attachment; filename="${latestPackage.fileName}"`); + res.setHeader('X-Agent-Version', latestPackage.version); + res.sendFile(latestPackage.filePath); + return; + } + + const latestPackageRecord = await findLatestAgentPackageRecord(arch); + if (latestPackageRecord) { + res.setHeader('X-Agent-Version', latestPackageRecord.version); + res.redirect(latestPackageRecord.filePath); + return; + } + + res.status(404).type('text/plain').send( + `No Local Installer Agent package found${arch ? ` for ${arch}` : ''}. ` + + 'Upload local-installer-agent__.deb to web-server/uploads/packages/agent, ' + + 'or upload it as package code local-installer-agent.' + ); +})); +app.use('/uploads/packages', express.static(uploadDir)); +app.use('/packages/agent', express.static(agentPackageDir)); app.use(loadCurrentUser); function helpers() { @@ -281,6 +337,139 @@ function getBaseUrl(req) { return `${protocol}://${req.get('host')}`; } +function normalizeAgentArch(value) { + const arch = String(value || '').trim().toLowerCase(); + return /^[a-z0-9][a-z0-9._-]*$/.test(arch) ? arch : ''; +} + +function isValidAgentVersion(value) { + return /^[a-zA-Z0-9][a-zA-Z0-9._+~=-]*$/.test(String(value || '').trim()); +} + +function compareAgentPackages(first, second) { + const versionCompare = agentVersionCollator.compare(first.version, second.version); + + if (versionCompare !== 0) return versionCompare; + return first.mtimeMs - second.mtimeMs; +} + +function formatBytes(bytes) { + const value = Number(bytes || 0); + if (value < 1024) return `${value} B`; + + const units = ['KB', 'MB', 'GB']; + let size = value / 1024; + let unitIndex = 0; + + while (size >= 1024 && unitIndex < units.length - 1) { + size /= 1024; + unitIndex += 1; + } + + return `${size.toFixed(size >= 10 ? 1 : 2)} ${units[unitIndex]}`; +} + +function formatLocalDateTime(value) { + return new Intl.DateTimeFormat('vi-VN', { + year: 'numeric', + month: '2-digit', + day: '2-digit', + hour: '2-digit', + minute: '2-digit' + }).format(value); +} + +function getAgentPackageDownloadPath(fileName) { + return `/packages/agent/${encodeURIComponent(fileName)}`; +} + +async function getAgentPackageFromEntry(entry) { + const match = /^local-installer-agent_([^/\\]+)_([^/\\]+)\.deb$/.exec(entry.name); + if (!match) return null; + + const [, version, packageArch] = match; + const filePath = path.join(agentPackageDir, entry.name); + const stat = await fsp.stat(filePath); + + return { + fileName: entry.name, + filePath, + version, + arch: packageArch, + size: stat.size, + sizeLabel: formatBytes(stat.size), + uploadedAt: formatLocalDateTime(stat.mtime), + mtimeMs: stat.mtimeMs, + downloadPath: getAgentPackageDownloadPath(entry.name) + }; +} + +async function listAgentPackages(arch = '') { + let dirEntries; + try { + dirEntries = await fsp.readdir(agentPackageDir, { withFileTypes: true }); + } catch (error) { + if (error.code === 'ENOENT') return null; + throw error; + } + + const packages = (await Promise.all( + dirEntries + .filter((entry) => entry.isFile()) + .map(getAgentPackageFromEntry) + )) + .filter(Boolean) + .filter((packageItem) => !arch || packageItem.arch.toLowerCase() === arch); + + const latestByArch = new Map(); + packages + .sort(compareAgentPackages); + + packages.forEach((packageItem) => { + latestByArch.set(packageItem.arch.toLowerCase(), packageItem.fileName); + }); + + return packages + .sort((first, second) => compareAgentPackages(second, first)) + .map((packageItem) => ({ + ...packageItem, + isLatestForArch: latestByArch.get(packageItem.arch.toLowerCase()) === packageItem.fileName + })); +} + +async function findLatestAgentPackage(arch = '') { + const packages = await listAgentPackages(arch); + const sortedPackages = packages + .slice() + .sort(compareAgentPackages); + + return sortedPackages[sortedPackages.length - 1] || null; +} + +function packageVersionMatchesArch(version, arch) { + if (!arch) return true; + return String(version.filePath || '').toLowerCase().includes(`_${arch}.deb`); +} + +async function findLatestAgentPackageRecord(arch = '') { + try { + const packageItem = await repository.getPackageById('local-installer-agent'); + const latestVersion = packageItem?.versions + ?.filter((version) => version.filePath && packageVersionMatchesArch(version, arch)) + ?.[0]; + + if (!latestVersion) return null; + + return { + filePath: latestVersion.filePath, + version: latestVersion.version + }; + } catch (error) { + console.warn('Cannot read local-installer-agent package from database:', error.message); + return null; + } +} + async function loadCurrentUser(req, res, next) { try { const cookies = parseCookies(req.headers.cookie); @@ -652,9 +841,143 @@ app.post('/logout', (req, res) => { redirectWithNotice(res, '/login', 'success', 'Đã đăng xuất.'); }); +app.get('/api/apps', asyncRoute(async (req, res) => { + const applications = await repository.listApplications(); + res.json({ + apps: applications + .filter((application) => application.status === 'Released') + .map((application) => ({ + appId: application.code, + appName: application.name, + version: application.version, + status: application.status, + packageCount: application.packageCount + })) + }); +})); + +app.get('/api/apps/:appCode', asyncRoute(async (req, res) => { + const application = await repository.getApplicationById(req.params.appCode); + if (!application || application.status !== 'Released') { + res.status(404).json({ error: 'Application not found' }); + return; + } + + res.json({ + appId: application.code, + appName: application.name, + version: application.version, + status: application.status, + packageCount: application.packageCount, + packages: application.packages + }); +})); + +app.get('/api/apps/:appCode/versions/:version/manifest', asyncRoute(async (req, res) => { + const manifest = await repository.getApplicationManifest( + req.params.appCode, + req.params.version, + getBaseUrl(req) + ); + + if (!manifest) { + res.status(404).json({ error: 'Application manifest not found' }); + return; + } + + res.json(manifest); +})); + +app.get('/install-agent.sh', (req, res) => { + const baseUrl = getBaseUrl(req); + const agentUrl = `${baseUrl}/packages/agent/latest.deb`; + res.type('text/x-shellscript').send(`#!/usr/bin/env bash +set -euo pipefail + +ARCH="$(dpkg --print-architecture)" +AGENT_URL="${agentUrl}?arch=$ARCH" +TMP_DEB="/tmp/local-installer-agent.deb" + +echo "Downloading Local Installer Agent..." +curl -fL "$AGENT_URL" -o "$TMP_DEB" + +echo "Installing Local Installer Agent..." +apt install -y "$TMP_DEB" + +echo "Starting Local Installer Agent..." +systemctl enable local-installer-agent +systemctl restart local-installer-agent + +echo "Checking Agent..." +curl -fsSL http://127.0.0.1:5010/health + +echo "" +echo "Local Installer Agent installed successfully." +`); +}); + app.use(requireAuthenticated); app.use('/uploads', express.static(path.join(__dirname, 'uploads'))); +app.get('/agent', requireAdmin, asyncRoute(async (req, res) => { + const pageData = await repository.getPageData(req.currentUser); + const agentPackages = await listAgentPackages(); + const preferredArch = normalizeAgentArch(req.query.arch) || 'amd64'; + const latestAgentPackage = await findLatestAgentPackage(preferredArch); + const baseUrl = getBaseUrl(req); + + res.render('agent', viewModel(req, 'agent', 'Agent packages', pageData, { + agentPackages, + latestAgentPackage, + agentPackageDir, + preferredArch, + installCommand: `curl -fsSL ${baseUrl}/install-agent.sh | sudo bash`, + latestAgentUrl: `${baseUrl}/packages/agent/latest.deb?arch=${preferredArch}` + })); +})); + +app.post('/agent/packages', requireAdmin, agentUpload.single('agentFile'), asyncRoute(async (req, res) => { + const version = String(req.body.version || '').trim(); + const arch = normalizeAgentArch(req.body.arch); + + try { + if (!req.file || !version || !arch) { + await removeUploadedFile(req.file); + redirectWithNotice(res, '/agent', 'warning', 'Vui lòng chọn file .deb, nhập version và architecture.'); + return; + } + + if (!isValidAgentVersion(version)) { + await removeUploadedFile(req.file); + redirectWithNotice(res, '/agent', 'warning', 'Version chỉ nên chứa chữ, số, dấu chấm, gạch ngang hoặc gạch dưới.'); + return; + } + + if (path.extname(req.file.originalname).toLowerCase() !== '.deb') { + await removeUploadedFile(req.file); + redirectWithNotice(res, '/agent', 'warning', 'Agent package phải là file .deb.'); + return; + } + + const targetFileName = `local-installer-agent_${version}_${arch}.deb`; + const targetPath = path.join(agentPackageDir, targetFileName); + const isUpdate = fs.existsSync(targetPath); + + await fsp.mkdir(agentPackageDir, { recursive: true }); + await fsp.rename(req.file.path, targetPath); + + redirectWithNotice( + res, + '/agent', + 'success', + isUpdate ? 'Đã cập nhật Agent package.' : 'Đã upload Agent package mới.' + ); + } catch (error) { + await removeUploadedFile(req.file); + throw error; + } +})); + app.post('/profile', asyncRoute(async (req, res) => { const returnTo = sanitizeReturnTo(req.body.returnTo); const fullName = String(req.body.fullName || '').trim(); diff --git a/web-server/src/repository.js b/web-server/src/repository.js index 07b5658..d252707 100644 --- a/web-server/src/repository.js +++ b/web-server/src/repository.js @@ -262,6 +262,16 @@ function mapApplicationPackageRow(row) { }; } +function toAbsoluteUrl(baseUrl, filePath) { + if (!filePath) return ''; + if (/^https?:\/\//i.test(filePath)) return filePath; + + const normalizedBaseUrl = String(baseUrl || '').replace(/\/+$/, ''); + const normalizedPath = String(filePath).startsWith('/') ? filePath : `/${filePath}`; + + return `${normalizedBaseUrl}${normalizedPath}`; +} + async function getUserById(id) { const pool = await getPool(); const result = await pool.request() @@ -806,6 +816,88 @@ async function getApplicationById(id) { return application; } +async function getApplicationManifest(appCode, version, baseUrl) { + const pool = await getPool(); + const appResult = await pool.request() + .input('AppCode', sql.NVarChar(100), String(appCode || '').trim()) + .input('AppVersion', sql.NVarChar(50), String(version || '').trim()) + .query(` + SELECT TOP (1) Id, AppCode, AppName, AppVersion + FROM dbo.Applications + WHERE AppCode = @AppCode + AND AppVersion = @AppVersion + AND Status = N'Released'; + `); + + const appRow = appResult.recordset[0]; + if (!appRow) return null; + + const componentResult = await pool.request() + .input('ApplicationId', sql.UniqueIdentifier, appRow.Id) + .query(` + SELECT + ap.Id, + p.PackageCode, + p.PackageName, + p.PackageType, + COALESCE(selected_version.Version, latest_version.Version) AS Version, + COALESCE(selected_version.FilePath, latest_version.FilePath) AS FilePath, + COALESCE(selected_version.DockerImage, latest_version.DockerImage) AS DockerImage, + COALESCE(selected_version.FileChecksumSha256, latest_version.FileChecksumSha256) AS FileChecksumSha256, + ROW_NUMBER() OVER (ORDER BY ap.AddedAt ASC, p.PackageCode ASC) * 10 AS InstallOrder + FROM dbo.ApplicationPackages AS ap + INNER JOIN dbo.Packages AS p + ON p.Id = ap.PackageId + LEFT JOIN dbo.PackageVersions AS selected_version + ON selected_version.Id = ap.SelectedVersionId + OUTER APPLY ( + SELECT TOP (1) latest.* + FROM dbo.PackageVersions AS latest + WHERE latest.PackageId = p.Id + AND latest.IsLatest = 1 + ORDER BY latest.ReleaseDate DESC, latest.UploadedAt DESC + ) AS latest_version + WHERE ap.ApplicationId = @ApplicationId + ORDER BY ap.AddedAt ASC, p.PackageCode ASC; + `); + + const components = componentResult.recordset.map((row) => { + const installOrder = Number(row.InstallOrder || 10); + + if (row.PackageType === 'docker') { + return { + componentId: row.PackageCode, + type: 'docker', + installOrder, + required: true, + image: row.DockerImage || '', + tag: row.Version || 'latest', + containerName: row.PackageCode + }; + } + + return { + componentId: row.PackageCode, + type: 'deb', + installOrder, + required: true, + packageName: row.PackageCode, + version: row.Version || '', + downloadUrl: toAbsoluteUrl(baseUrl, row.FilePath), + sha256: row.FileChecksumSha256 || '' + }; + }); + + return { + schemaVersion: '1.0', + appId: appRow.AppCode, + appName: appRow.AppName, + version: appRow.AppVersion, + architecture: 'amd64', + components + }; +} + async function getStats() { const pool = await getPool(); const result = await pool.request().query(` @@ -1216,6 +1308,7 @@ module.exports = { getPageData, listPackages, listApplications, + getApplicationManifest, getPackageById, getApplicationById, createPackageWithVersion, diff --git a/web-server/views/agent.ejs b/web-server/views/agent.ejs new file mode 100644 index 0000000..6260770 --- /dev/null +++ b/web-server/views/agent.ejs @@ -0,0 +1,157 @@ +<%- include('partials/page-start') %> + +
+
+
+

Agent

+

Quản lý Local Installer Agent package dùng cho máy client Linux.

+
+
+ Admin + <%= agentPackages.length %> packages +
+
+ +
+
+
+
+

Upload / Update

+ <% if (latestAgentPackage) { %> +

Latest <%= preferredArch %>: <%= latestAgentPackage.version %>

+ <% } else { %> +

Chưa có Agent package cho <%= preferredArch %>.

+ <% } %> +
+
+ +
+
+ + +
+ Agent .deb +
+ +
+ upload_file + Chọn file Agent package + Server sẽ lưu thành local-installer-agent_<version>_<arch>.deb + +
+ +
+
+
+ +
+ +
+
+ +
+ + + +
+
+ +
+
+ +
+ +
+ + + + + + + + + + + + + + <% if (agentPackages.length === 0) { %> + + + + <% } %> + <% agentPackages.forEach((item) => { %> + + + + + + + + + + <% }) %> + +
VersionArchFileSizeUploadedStatusActions
Chưa có Agent package nào.
<%= item.version %><%= item.arch %> + <%= item.fileName %> + <%= item.downloadPath %> + <%= item.sizeLabel %><%= item.uploadedAt %> + <% if (item.isLatestForArch) { %> + Latest + <% } else { %> + Stored + <% } %> + +
+ + download + +
+
+
+
+ Showing 1-<%= agentPackages.length %> of <%= agentPackages.length %> +
+ + Page 1 / 1 + +
+
+
+
+
+ +<%- include('partials/page-end') %>