Claus Lohmar
509eb97b57
Large-file handling: - Set TMPDIR=/mnt/converter/in in service to spool uploads to shared storage instead of 24 GB LXC rootfs (critical for >24GB) - Chunked upload streaming (8 MiB) with progress logging every 1 GiB - Pre-flight disk space check via Content-Length header - Clean up partial files on upload/download failure - Download timeout extended to 7200s (2 hours) for 88 GB images - Switched wget from --show-progress to --progress=dot:giga (compact output, won't fill memory on large transfers) - uvicorn --timeout-keep-alive 300 on both frontend and backend VM name: - Added vm_name field to initial session form (step 1) - Falls back to auto-generated 'os_type-vmid' if left blank - Pre-filled & editable in confirm form (step 2)
282 lines
11 KiB
Python
282 lines
11 KiB
Python
"""
|
|
VM Bench — Proxmox Image Conversion Frontend
|
|
|
|
FastAPI web application serving as the user GUI.
|
|
Runs in the vm-bench LXC on port 5000.
|
|
Communicates with the backend at http://10.2.0.2:9000.
|
|
|
|
Routes:
|
|
GET / New session form
|
|
POST /session/start Upload + analyse source image
|
|
POST /session/confirm Submit conversion job
|
|
GET /session/status/{id} Poll job status (JSON)
|
|
POST /session/cleanup/{id} Clean up / reuse staging files
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import uuid
|
|
from pathlib import Path
|
|
from typing import Optional
|
|
|
|
from fastapi import FastAPI, Form, Request, UploadFile, File
|
|
from fastapi.responses import HTMLResponse, JSONResponse
|
|
from fastapi.staticfiles import StaticFiles
|
|
|
|
from jinja2 import Environment, FileSystemLoader
|
|
|
|
from api_client import ApiClient, ApiError, BACKEND_URL
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# App setup
|
|
# ---------------------------------------------------------------------------
|
|
app = FastAPI(title="VM Bench — Frontend", version="1.0.0")
|
|
|
|
BASE = Path(__file__).parent
|
|
app.mount("/static", StaticFiles(directory=str(BASE / "static")), name="static")
|
|
|
|
_jinja = Environment(loader=FileSystemLoader(str(BASE / "templates")), autoescape=True)
|
|
STAGING = Path("/mnt/converter/in")
|
|
|
|
api = ApiClient()
|
|
|
|
logger = logging.getLogger("vm-bench")
|
|
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
|
|
|
# Config
|
|
VM_ID_MIN = 21000
|
|
VM_ID_MAX = 21100
|
|
DEFAULT_STORAGE = "local-lvm"
|
|
MIN_FREE_DISK_GB = 2 # keep 2 GB headroom
|
|
DOWNLOAD_TIMEOUT = 7200 # 2 hours for very large files
|
|
UPLOAD_PROGRESS_INTERVAL = 1024**3 # log every 1 GiB during upload
|
|
|
|
def _validate_vmid(vmid: int) -> Optional[str]:
|
|
if not (VM_ID_MIN <= vmid <= VM_ID_MAX):
|
|
return f"VM ID must be between {VM_ID_MIN} and {VM_ID_MAX}."
|
|
return None
|
|
|
|
def _check_disk_space(path: Path, needed_gb: int) -> Optional[str]:
|
|
"""Return an error message if <path> has less than <needed_gb> + headroom free."""
|
|
usage = shutil.disk_usage(path.parent if path.is_file() or not path.exists() else path)
|
|
free_gb = usage.free / (1024**3)
|
|
required = needed_gb + MIN_FREE_DISK_GB
|
|
if free_gb < required:
|
|
return (
|
|
f"Insufficient disk space on {path.parent}: "
|
|
f"{free_gb:.1f} GB free, {required:.1f} GB needed "
|
|
f"({needed_gb} GB file + {MIN_FREE_DISK_GB} GB headroom). "
|
|
f"Free up space or use a smaller file."
|
|
)
|
|
return None
|
|
|
|
def render(name: str, status: int = 200, **ctx) -> HTMLResponse:
|
|
tpl = _jinja.get_template(name)
|
|
return HTMLResponse(tpl.render(**ctx), status_code=status)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Routes
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@app.get("/", response_class=HTMLResponse)
|
|
async def index(request: Request, vmid: Optional[int] = None, source: Optional[str] = None):
|
|
"""Landing page — new session form. Optionally pre-fills vmid + source for reuse."""
|
|
return render("index.html", request=request, backend_url=BACKEND_URL,
|
|
prefill_vmid=vmid or "", prefill_source=source or "",
|
|
prefill_vmname="")
|
|
|
|
|
|
@app.post("/session/start", response_class=HTMLResponse)
|
|
async def start_session(
|
|
request: Request,
|
|
vmid: int = Form(...),
|
|
vm_name: str = Form(""),
|
|
source_type: str = Form("upload"),
|
|
source_file: Optional[UploadFile] = File(None),
|
|
source_url: Optional[str] = Form(None),
|
|
):
|
|
"""Upload or download source file, then call backend /analyze."""
|
|
filename = None
|
|
error = None
|
|
|
|
# Validate VM ID range
|
|
err = _validate_vmid(vmid)
|
|
if err:
|
|
return render("_analysis.html", request=request, error=err)
|
|
|
|
# --- Get the file into /mnt/converter/in/ ---
|
|
if source_type == "upload":
|
|
if not source_file or not source_file.filename:
|
|
error = "No file uploaded."
|
|
else:
|
|
filename = source_file.filename
|
|
dest = STAGING / filename
|
|
|
|
# Estimate size from Content-Length header (may be None)
|
|
content_length = request.headers.get("content-length")
|
|
if content_length:
|
|
estimated_gb = int(content_length) / (1024**3)
|
|
err = _check_disk_space(dest, estimated_gb)
|
|
if err:
|
|
return render("_analysis.html", request=request, error=err)
|
|
|
|
try:
|
|
logger.info("Receiving upload: %s (%s bytes)", filename, content_length or "unknown")
|
|
with dest.open("wb") as f:
|
|
written = 0
|
|
while True:
|
|
chunk = source_file.file.read(8 * 1024 * 1024) # 8 MiB chunks
|
|
if not chunk:
|
|
break
|
|
f.write(chunk)
|
|
written += len(chunk)
|
|
if written % UPLOAD_PROGRESS_INTERVAL < len(chunk):
|
|
logger.info("Upload progress: %s — %.1f GiB written", filename, written / (1024**3))
|
|
logger.info("Upload complete: %s (%.1f GiB)", filename, written / (1024**3))
|
|
except OSError as exc:
|
|
# Clean up partial file on disk-full or other IO error
|
|
if dest.exists():
|
|
dest.unlink(missing_ok=True)
|
|
error = f"Upload failed (disk full?): {exc}"
|
|
except Exception as exc:
|
|
if dest.exists():
|
|
dest.unlink(missing_ok=True)
|
|
error = f"Upload failed: {exc}"
|
|
else:
|
|
url = (source_url or "").strip()
|
|
if not url:
|
|
error = "No URL provided."
|
|
else:
|
|
filename = Path(url).name or f"download_{uuid.uuid4().hex[:8]}"
|
|
dest = STAGING / filename
|
|
|
|
# Warn if staging is tight, but don't block (don't know file size)
|
|
usage = shutil.disk_usage(STAGING)
|
|
free_gb = usage.free / (1024**3)
|
|
if free_gb < 50:
|
|
logger.warning("Low disk space on %s: %.1f GB free — download may fail for large files",
|
|
STAGING, free_gb)
|
|
|
|
logger.info("Starting download: %s → %s", url, dest)
|
|
try:
|
|
# --progress=dot:giga prints one dot per 64 KiB downloaded, minimal output
|
|
# Redirection to stderr keeps stdout clean for error capture
|
|
result = subprocess.run(
|
|
["wget", "--progress=dot:giga", "-O", str(dest), url],
|
|
capture_output=True, text=True, timeout=DOWNLOAD_TIMEOUT,
|
|
)
|
|
if result.returncode != 0:
|
|
error = f"Download failed: {result.stderr.strip()[:300]}"
|
|
else:
|
|
file_size = dest.stat().st_size / (1024**3) if dest.exists() else 0
|
|
logger.info("Download complete: %s (%.1f GiB)", filename, file_size)
|
|
except subprocess.TimeoutExpired:
|
|
if dest.exists():
|
|
dest.unlink(missing_ok=True)
|
|
error = (
|
|
f"Download timed out after {DOWNLOAD_TIMEOUT // 3600} hours. "
|
|
f"The file may be too large for your network speed."
|
|
)
|
|
except Exception as exc:
|
|
if dest.exists():
|
|
dest.unlink(missing_ok=True)
|
|
error = f"Download failed: {exc}"
|
|
|
|
if error:
|
|
return render("_analysis.html", request=request, error=error)
|
|
|
|
# --- Call backend /analyze ---
|
|
try:
|
|
analysis = api.analyze(vmid=vmid, filename=filename)
|
|
except ApiError as exc:
|
|
return render("_analysis.html", request=request,
|
|
error=f"Backend analysis failed: {exc.detail}")
|
|
|
|
vm_name = vm_name.strip() if vm_name else ""
|
|
if not vm_name:
|
|
vm_name = (analysis.get("os_type") or "vm") + f"-{vmid}"
|
|
|
|
return render("_analysis.html", request=request,
|
|
vmid=vmid, source_filename=filename,
|
|
vm_name=vm_name, analysis=analysis)
|
|
|
|
|
|
@app.post("/session/confirm", response_class=HTMLResponse)
|
|
async def confirm_session(
|
|
request: Request,
|
|
vmid: int = Form(...),
|
|
source_filename: str = Form(...),
|
|
disk_format: str = Form(...),
|
|
vm_name: str = Form(...),
|
|
cpu_cores: int = Form(2),
|
|
ram_mb: int = Form(4096),
|
|
target_storage: str = Form("local-lvm"),
|
|
target_disk_size_gb: Optional[int] = Form(None),
|
|
auto_detect_boot: str = Form("true"),
|
|
boot_type: str = Form("uefi"),
|
|
):
|
|
"""Build the job payload and submit to the backend."""
|
|
# Validate VM ID range
|
|
err = _validate_vmid(vmid)
|
|
if err:
|
|
return render("_analysis.html", request=request, error=err)
|
|
|
|
payload = {
|
|
"vmid": vmid,
|
|
"vm_name": vm_name,
|
|
"cpu_cores": cpu_cores,
|
|
"ram_mb": ram_mb,
|
|
"target_storage": target_storage,
|
|
"auto_detect_boot": auto_detect_boot == "true",
|
|
"boot_type": boot_type,
|
|
"boot_disk": {
|
|
"disk_type": "image_file",
|
|
"source_filename": source_filename,
|
|
"format": disk_format,
|
|
},
|
|
}
|
|
if target_disk_size_gb is not None:
|
|
payload["target_disk_size_gb"] = target_disk_size_gb
|
|
|
|
try:
|
|
job = api.create_job(payload)
|
|
except ApiError as exc:
|
|
return render("_analysis.html", request=request,
|
|
error=f"Job submission failed: {exc.detail}")
|
|
|
|
return render("polling.html", request=request,
|
|
job_id=job["job_id"], vmid=vmid,
|
|
vm_name=vm_name, source_filename=source_filename)
|
|
|
|
|
|
@app.get("/session/status/{job_id}")
|
|
async def session_status(job_id: str):
|
|
"""Poll backend for job status (returns JSON for AJAX polling)."""
|
|
try:
|
|
return api.get_job(job_id)
|
|
except ApiError as exc:
|
|
return JSONResponse({"error": exc.detail, "status": "error"}, status_code=502)
|
|
|
|
|
|
@app.post("/session/cleanup/{job_id}")
|
|
async def session_cleanup(job_id: str, request: Request):
|
|
"""Forward cleanup request to backend."""
|
|
try:
|
|
body = await request.json()
|
|
delete = body.get("delete_staging_files", False)
|
|
return api.cleanup_job(job_id, delete)
|
|
except ApiError as exc:
|
|
return JSONResponse({"error": exc.detail}, status_code=502)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Run
|
|
# ---------------------------------------------------------------------------
|
|
if __name__ == "__main__":
|
|
import uvicorn
|
|
uvicorn.run(app, host="0.0.0.0", port=5000)
|