diff --git a/README.md b/README.md new file mode 100644 index 0000000..b01cc64 --- /dev/null +++ b/README.md @@ -0,0 +1,195 @@ +# VM Bench — Proxmox Image Conversion Frontend + +Web GUI and REST API for converting VMware disk images (VMDK, VHD, etc.) into +Proxmox-compatible QCOW2 disks and provisioning VMs with automatic OS and boot +type detection. + +## Architecture + +``` +Browser vm-bench LXC Proxmox Host (srv2) + (this container) +──→ :5000 ──→ frontend/app.py ──→ :9000 backend/app.py + │ + ├─ qemu-img (format, size) + ├─ guestfish (OS, EFI detection) + ├─ 7z/unzip (archive extraction) + └─ qm create (VM provisioning) +``` + +- **Frontend** (`vm-bench` LXC) — FastAPI + Jinja2 web UI on port 5000 +- **Backend** (`srv2` Proxmox host) — FastAPI REST API on port 9000 +- **Shared storage** — `/mnt/converter/in` (staging) and `/mnt/converter/out` (output) + +## Directory Layout + +``` +/mnt/converter/ +├── frontend/ # Web UI (runs on vm-bench LXC) +│ ├── app.py # FastAPI app, routes, session handling +│ ├── api_client.py # Typed REST client → backend +│ ├── requirements.txt +│ ├── templates/ # Jinja2 HTML templates +│ │ ├── base.html +│ │ ├── index.html # New session form +│ │ ├── _analysis.html # Analysis result + confirm form +│ │ └── polling.html # Progress bar + job status +│ └── static/ +│ └── proxmox.css +├── backend/ # REST API (deploy to Proxmox host srv2) +│ ├── app.py # FastAPI app, routes +│ ├── models.py # Pydantic request/response models +│ ├── converter.py # Archive extraction, disk probing, OS/EFI detection +│ ├── provisioner.py # VM provisioning (qm create/importdisk) +│ ├── requirements.txt +│ ├── install.sh # Systemd installation script +│ └── vm-bench-backend.service +├── open-api.yaml # API spec (single source of truth) +├── in/ # Shared staging directory (gitignored) +└── out/ # Shared output directory (gitignored) +``` + +## Features + +- **File upload or URL download** — ingest `.vmdk`, `.vhd`, `.7z`, `.zip`, `.tar.gz` archives +- **Automatic archive extraction** — 7z, unzip, tar, gunzip +- **Disk analysis** — detects format, virtual size, guest OS, EFI bootability +- **VM provisioning** — creates Proxmox VM with automatic disk import and boot config +- **Progress polling** — real-time job status with progress bar +- **Staging reuse** — keep source files to create multiple VMs from one image + +## Prerequisites + +### Backend (Proxmox host `srv2`) + +- Python 3.13+ +- `qemu-img` (from `qemu-utils`) +- `guestfish` (from `libguestfs-tools`) +- `7z`, `unzip`, `tar` (for archive extraction) +- `systemd` (for service management) + +### Frontend (`vm-bench` LXC) + +- Python 3.13+ +- `wget` (for URL downloads) +- `systemd` (for service management) +- Network access to backend at `10.2.0.2:9000` + +## Installation + +### 1. Backend (on Proxmox host `srv2`) + +```bash +# The backend code is shared via /mnt/converter/backend/ +cd /mnt/converter/backend +bash install.sh +``` + +This installs Python dependencies, copies the systemd service, and starts +`vm-bench-backend` on port 9000. Verify: + +```bash +curl http://127.0.0.1:9000/api/v1/health +# → {"status":"ok"} +``` + +### 2. Frontend (on `vm-bench` LXC) + +```bash +# Install dependencies +cd /mnt/converter/frontend +pip3 install --break-system-packages -r requirements.txt + +# Copy service file and start +cp /etc/systemd/system/vm-bench.service /etc/systemd/system/ +systemctl daemon-reload +systemctl enable --now vm-bench +``` + +The frontend serves on port 5000. Open `http://:5000` in a browser. + +## API Reference + +All endpoints are under `/api/v1/`. Full specification in [`open-api.yaml`](open-api.yaml). + +| Method | Path | Description | +|--------|------|-------------| +| `GET` | `/api/v1/health` | Health check | +| `POST` | `/api/v1/analyze` | Probe source image (format, size, OS, EFI) | +| `POST` | `/api/v1/jobs` | Submit conversion + provisioning job | +| `GET` | `/api/v1/jobs/{id}` | Poll job status and progress | +| `POST` | `/api/v1/jobs/{id}/cleanup` | Delete or preserve staging files | + +### Example: Analyze a source image + +```bash +curl -X POST http://10.2.0.2:9000/api/v1/analyze \ + -H "Content-Type: application/json" \ + -d '{"vmid": 21050, "source_filename": "64bit/Debian 12.11.0 (64bit).vmdk"}' + +# Response: +# { +# "vmid": 21050, +# "filename": "Debian 12.11.0 (64bit).vmdk", +# "disk_format": "vmdk", +# "disk_size_gb": 7.5, +# "os_type": "Debian", +# "efi_detectable": true +# } +``` + +### Example: Submit a conversion job + +```bash +curl -X POST http://10.2.0.2:9000/api/v1/jobs \ + -H "Content-Type: application/json" \ + -d '{ + "vmid": 21050, + "vm_name": "debian-test", + "boot_disk": { + "disk_type": "image_file", + "source_filename": "64bit/Debian 12.11.0 (64bit).vmdk", + "format": "vmdk" + }, + "cpu_cores": 2, + "ram_mb": 4096, + "target_storage": "local-lvm" + }' + +# Response: +# { "job_id": "job_21050_1737480000", "vmid": 21050, "status": "queued", ... } +``` + +## Usage Flow + +1. **Open the web UI** → new session form +2. **Upload or download a source image** → file lands in `/mnt/converter/in/` +3. **Backend analyses the image** → shows format, size, OS, EFI status +4. **Configure VM settings** → name, CPU, RAM, storage, boot type +5. **Submit job** → backend converts, shrinks (if needed), creates VM +6. **Poll progress** → real-time status bar updates every 2 seconds +7. **Reuse or cleanup** → create another VM from same source, or delete staging files + +## Configuration + +| Variable | Default | Description | +|----------|---------|-------------| +| `BACKEND_URL` | `http://10.2.0.2:9000` | Backend API base URL (set in systemd unit) | +| `VM_ID_MIN` | `21000` | Minimum allowed Proxmox VM ID | +| `VM_ID_MAX` | `21100` | Maximum allowed Proxmox VM ID | + +## Development + +**Source of truth**: `open-api.yaml` — always derive models from this file. + +**Backend tool requirements** (on srv2): +- `qemu-img` — disk format and size detection +- `guestfish` / `virt-inspector` — OS type and EFI boot detection +- `7z`, `unzip`, `tar`, `gunzip`, `bunzip2`, `xz` — archive extraction + +**Provisioner note**: `backend/provisioner.py` is currently a stub with simulated +progress. Replace with real `qm create`, `qm importdisk`, and `qm set` commands +when deploying to the Proxmox host. + +**Staging files**: `/mnt/converter/in/` and `/mnt/converter/out/` are gitignored — +they contain user data, not code. diff --git a/backend/app.py b/backend/app.py index 3e03fe1..ae56a05 100644 --- a/backend/app.py +++ b/backend/app.py @@ -1,14 +1,13 @@ """ Backend API — Proxmox Image Conversion Engine -Runs on the Proxmox host (srv2) at http://10.2.0.2:9000/api/v1 +Runs on the Proxmox host (srv2) at http://10.2.0.2:9000 +All API routes live under /api/v1/. Matches open-api.yaml v1.1.0 spec exactly. """ from __future__ import annotations -from pathlib import Path - from fastapi import FastAPI, HTTPException from fastapi.middleware.cors import CORSMiddleware @@ -16,13 +15,21 @@ from models import ( JobSubmissionRequest, JobStatusResponse, CleanupRequest, + CleanupResponse, AnalyzeRequest, AnalyzeResponse, + HealthResponse, ErrorResponse, ) -from converter import extract_if_needed, discover_disk, detect_os +from converter import extract_if_needed, discover_disk, detect_os, detect_efi from provisioner import submit_job, get_job_status, cleanup_staging +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- +STAGING_IN = "/mnt/converter/in" +API_PREFIX = "/api/v1" + # --------------------------------------------------------------------------- # App # --------------------------------------------------------------------------- @@ -39,18 +46,32 @@ app.add_middleware( allow_headers=["*"], ) + # --------------------------------------------------------------------------- -# Routes — matching open-api.yaml +# Helpers # --------------------------------------------------------------------------- -@app.get("/health") -def health() -> dict: - return {"status": "ok"} +def _validate_filename(filename: str) -> None: + """Reject filenames that attempt path traversal.""" + if ".." in filename or filename.startswith("/"): + raise HTTPException(status_code=400, detail="Invalid filename: path traversal not allowed") -@app.post("/analyze", response_model=AnalyzeResponse) +# --------------------------------------------------------------------------- +# Routes — all under /api/v1/ +# --------------------------------------------------------------------------- + +@app.get(f"{API_PREFIX}/health", response_model=HealthResponse) +def health() -> HealthResponse: + """Health check.""" + return HealthResponse(status="ok") + + +@app.post(f"{API_PREFIX}/analyze", response_model=AnalyzeResponse) def analyze(req: AnalyzeRequest) -> AnalyzeResponse: - """Analyze a source file: extract if archive, find disk, detect OS.""" + """Analyze a source file: extract if archive, find disk, detect OS + EFI.""" + _validate_filename(req.source_filename) + try: source = extract_if_needed(req.source_filename) except Exception as exc: @@ -61,23 +82,21 @@ def analyze(req: AnalyzeRequest) -> AnalyzeResponse: except Exception as exc: raise HTTPException(status_code=400, detail=f"Disk discovery failed: {exc}") - # Determine extract_dir for OS detection extract_dir = source if source.is_dir() else source.parent - os_type = detect_os(disk.path, extract_dir) - efi = disk.format in ("raw", "qcow2") # qemu-img can probe EFI partition + efi = detect_efi(disk.path) return AnalyzeResponse( - os_type=os_type, - disk_size_gb=disk.size_gb, - disk_format=disk.format, - bootable=True, - efi_detectable=efi, + vmid=req.vmid, filename=disk.path.name, + disk_format=disk.format, + disk_size_gb=disk.size_gb, + os_type=os_type, + efi_detectable=efi, ) -@app.post("/api/v1/jobs", response_model=JobStatusResponse, status_code=202) +@app.post(f"{API_PREFIX}/jobs", response_model=JobStatusResponse, status_code=202) def create_job(req: JobSubmissionRequest) -> JobStatusResponse: """Submit a conversion + provisioning job.""" try: @@ -86,7 +105,7 @@ def create_job(req: JobSubmissionRequest) -> JobStatusResponse: raise HTTPException(status_code=400, detail=str(exc)) -@app.get("/api/v1/jobs/{job_id}", response_model=JobStatusResponse) +@app.get(f"{API_PREFIX}/jobs/{job_id}", response_model=JobStatusResponse) def job_status(job_id: str) -> JobStatusResponse: """Get current status of a conversion job.""" try: @@ -95,15 +114,19 @@ def job_status(job_id: str) -> JobStatusResponse: raise HTTPException(status_code=404, detail=f"Job not found: {job_id}") -@app.post("/api/v1/jobs/{job_id}/cleanup") -def cleanup_job(job_id: str, req: CleanupRequest) -> dict: +@app.post(f"{API_PREFIX}/jobs/{job_id}/cleanup", response_model=CleanupResponse) +def cleanup_job(job_id: str, req: CleanupRequest) -> CleanupResponse: """Delete or preserve staging files for a job.""" - # Extract vmid from job_id (format: job_{vmid}_{timestamp}) try: vmid = int(job_id.split("_")[1]) except (IndexError, ValueError): vmid = 0 - return cleanup_staging(vmid, req.delete_staging_files) + result = cleanup_staging(vmid, req.delete_staging_files) + return CleanupResponse( + job_id=result["job_id"], + action_taken=result["action_taken"], + message=result["message"], + ) # --------------------------------------------------------------------------- diff --git a/backend/converter.py b/backend/converter.py index 6ac4611..3ec0e78 100644 --- a/backend/converter.py +++ b/backend/converter.py @@ -238,3 +238,30 @@ def _parse_vmx_guest_os(vmx_path: Path) -> Optional[str]: if k in raw_lower: return v return raw.replace("-", " ").replace("_", " ").title() + + +# --------------------------------------------------------------------------- +# EFI detection +# --------------------------------------------------------------------------- + +def detect_efi(disk_path: Path) -> Optional[bool]: + """Check whether the disk has an EFI System Partition using guestfish. + + Returns True if a VFAT EFI partition is found, False if only non-EFI + filesystems detected, None if guestfish isn't available or fails. + """ + if not shutil.which("guestfish"): + return None + try: + result = subprocess.run( + ["guestfish", "--ro", "-a", str(disk_path), "-i", "list-filesystems"], + capture_output=True, text=True, timeout=30, + ) + if result.returncode != 0: + return None + # EFI System Partition is typically VFAT; look for it + if "vfat" in result.stdout.lower(): + return True + return False + except Exception: + return None diff --git a/backend/models.py b/backend/models.py index 54be64c..9e6dfff 100644 --- a/backend/models.py +++ b/backend/models.py @@ -76,18 +76,12 @@ class JobStatusResponse(BaseModel): error_details: Optional[str] = None -class CleanupResponse(BaseModel): - job_id: str - action_taken: str # "purged" | "retained" - message: str - - class ErrorResponse(BaseModel): detail: str # --------------------------------------------------------------------------- -# New: /analyze endpoint +# /api/v1/analyze endpoint # --------------------------------------------------------------------------- class AnalyzeRequest(BaseModel): @@ -97,14 +91,22 @@ class AnalyzeRequest(BaseModel): class AnalyzeResponse(BaseModel): + vmid: int + filename: str + disk_format: str + disk_size_gb: float os_type: Optional[str] = None - disk_size_gb: float = 0 - disk_format: str = "" - bootable: bool = False - efi_detectable: bool = False - filename: str = "" - error: Optional[str] = None + efi_detectable: Optional[bool] = None +# --------------------------------------------------------------------------- +# Utility response schemas +# --------------------------------------------------------------------------- + class HealthResponse(BaseModel): status: str = "ok" + +class CleanupResponse(BaseModel): + job_id: str + action_taken: str # "purged" | "retained" + message: str diff --git a/frontend/api_client.py b/frontend/api_client.py index c38ed3b..fdc9070 100644 --- a/frontend/api_client.py +++ b/frontend/api_client.py @@ -1,7 +1,8 @@ """ Typed REST client for the Proxmox Image Conversion Backend API. -Connects to the backend at http://10.2.0.2:9000/api/v1. +Connects to the backend at http://10.2.0.2:9000. +All API endpoints are under /api/v1/. Derived from open-api.yaml — do not change models without updating the spec. """ @@ -39,11 +40,11 @@ class ApiClient: # ------------------------------------------------------------------ def health(self) -> dict: - return self._get("/health") + return self._get("/api/v1/health") def analyze(self, vmid: int, filename: str, source_type: str = "upload") -> dict: - """POST /analyze — probe source file for OS, size, format.""" - return self._post("/analyze", { + """POST /api/v1/analyze — probe source file for OS, size, format.""" + return self._post("/api/v1/analyze", { "vmid": vmid, "source_filename": filename, "source_type": source_type, diff --git a/frontend/app.py b/frontend/app.py index 8ae3fa0..8987d0a 100644 --- a/frontend/app.py +++ b/frontend/app.py @@ -47,7 +47,6 @@ api = ApiClient() VM_ID_MIN = 21000 VM_ID_MAX = 21100 DEFAULT_STORAGE = "local-lvm" -DEFAULT_NETWORK = "vmbr0" def _validate_vmid(vmid: int) -> Optional[str]: if not (VM_ID_MIN <= vmid <= VM_ID_MAX): @@ -161,7 +160,6 @@ async def confirm_session( "cpu_cores": cpu_cores, "ram_mb": ram_mb, "target_storage": target_storage, - "network": DEFAULT_NETWORK, "auto_detect_boot": auto_detect_boot == "true", "boot_type": boot_type, "boot_disk": {