fix: large-file support + user-defined VM name
Large-file handling: - Set TMPDIR=/mnt/converter/in in service to spool uploads to shared storage instead of 24 GB LXC rootfs (critical for >24GB) - Chunked upload streaming (8 MiB) with progress logging every 1 GiB - Pre-flight disk space check via Content-Length header - Clean up partial files on upload/download failure - Download timeout extended to 7200s (2 hours) for 88 GB images - Switched wget from --show-progress to --progress=dot:giga (compact output, won't fill memory on large transfers) - uvicorn --timeout-keep-alive 300 on both frontend and backend VM name: - Added vm_name field to initial session form (step 1) - Falls back to auto-generated 'os_type-vmid' if left blank - Pre-filled & editable in confirm form (step 2)
This commit is contained in:
parent
4b50b185fa
commit
509eb97b57
4 changed files with 89 additions and 8 deletions
|
|
@ -7,7 +7,7 @@ Type=simple
|
|||
User=root
|
||||
WorkingDirectory=/mnt/converter/backend
|
||||
Environment=PATH=/usr/local/bin:/usr/bin:/bin
|
||||
ExecStart=/usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 9000
|
||||
ExecStart=/usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 9000 --timeout-keep-alive 300
|
||||
Restart=always
|
||||
RestartSec=3
|
||||
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ Routes:
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
|
|
@ -43,16 +44,36 @@ STAGING = Path("/mnt/converter/in")
|
|||
|
||||
api = ApiClient()
|
||||
|
||||
logger = logging.getLogger("vm-bench")
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
||||
|
||||
# Config
|
||||
VM_ID_MIN = 21000
|
||||
VM_ID_MAX = 21100
|
||||
DEFAULT_STORAGE = "local-lvm"
|
||||
MIN_FREE_DISK_GB = 2 # keep 2 GB headroom
|
||||
DOWNLOAD_TIMEOUT = 7200 # 2 hours for very large files
|
||||
UPLOAD_PROGRESS_INTERVAL = 1024**3 # log every 1 GiB during upload
|
||||
|
||||
def _validate_vmid(vmid: int) -> Optional[str]:
|
||||
if not (VM_ID_MIN <= vmid <= VM_ID_MAX):
|
||||
return f"VM ID must be between {VM_ID_MIN} and {VM_ID_MAX}."
|
||||
return None
|
||||
|
||||
def _check_disk_space(path: Path, needed_gb: int) -> Optional[str]:
|
||||
"""Return an error message if <path> has less than <needed_gb> + headroom free."""
|
||||
usage = shutil.disk_usage(path.parent if path.is_file() or not path.exists() else path)
|
||||
free_gb = usage.free / (1024**3)
|
||||
required = needed_gb + MIN_FREE_DISK_GB
|
||||
if free_gb < required:
|
||||
return (
|
||||
f"Insufficient disk space on {path.parent}: "
|
||||
f"{free_gb:.1f} GB free, {required:.1f} GB needed "
|
||||
f"({needed_gb} GB file + {MIN_FREE_DISK_GB} GB headroom). "
|
||||
f"Free up space or use a smaller file."
|
||||
)
|
||||
return None
|
||||
|
||||
def render(name: str, status: int = 200, **ctx) -> HTMLResponse:
|
||||
tpl = _jinja.get_template(name)
|
||||
return HTMLResponse(tpl.render(**ctx), status_code=status)
|
||||
|
|
@ -66,13 +87,15 @@ def render(name: str, status: int = 200, **ctx) -> HTMLResponse:
|
|||
async def index(request: Request, vmid: Optional[int] = None, source: Optional[str] = None):
|
||||
"""Landing page — new session form. Optionally pre-fills vmid + source for reuse."""
|
||||
return render("index.html", request=request, backend_url=BACKEND_URL,
|
||||
prefill_vmid=vmid or "", prefill_source=source or "")
|
||||
prefill_vmid=vmid or "", prefill_source=source or "",
|
||||
prefill_vmname="")
|
||||
|
||||
|
||||
@app.post("/session/start", response_class=HTMLResponse)
|
||||
async def start_session(
|
||||
request: Request,
|
||||
vmid: int = Form(...),
|
||||
vm_name: str = Form(""),
|
||||
source_type: str = Form("upload"),
|
||||
source_file: Optional[UploadFile] = File(None),
|
||||
source_url: Optional[str] = Form(None),
|
||||
|
|
@ -93,10 +116,36 @@ async def start_session(
|
|||
else:
|
||||
filename = source_file.filename
|
||||
dest = STAGING / filename
|
||||
|
||||
# Estimate size from Content-Length header (may be None)
|
||||
content_length = request.headers.get("content-length")
|
||||
if content_length:
|
||||
estimated_gb = int(content_length) / (1024**3)
|
||||
err = _check_disk_space(dest, estimated_gb)
|
||||
if err:
|
||||
return render("_analysis.html", request=request, error=err)
|
||||
|
||||
try:
|
||||
logger.info("Receiving upload: %s (%s bytes)", filename, content_length or "unknown")
|
||||
with dest.open("wb") as f:
|
||||
shutil.copyfileobj(source_file.file, f)
|
||||
written = 0
|
||||
while True:
|
||||
chunk = source_file.file.read(8 * 1024 * 1024) # 8 MiB chunks
|
||||
if not chunk:
|
||||
break
|
||||
f.write(chunk)
|
||||
written += len(chunk)
|
||||
if written % UPLOAD_PROGRESS_INTERVAL < len(chunk):
|
||||
logger.info("Upload progress: %s — %.1f GiB written", filename, written / (1024**3))
|
||||
logger.info("Upload complete: %s (%.1f GiB)", filename, written / (1024**3))
|
||||
except OSError as exc:
|
||||
# Clean up partial file on disk-full or other IO error
|
||||
if dest.exists():
|
||||
dest.unlink(missing_ok=True)
|
||||
error = f"Upload failed (disk full?): {exc}"
|
||||
except Exception as exc:
|
||||
if dest.exists():
|
||||
dest.unlink(missing_ok=True)
|
||||
error = f"Upload failed: {exc}"
|
||||
else:
|
||||
url = (source_url or "").strip()
|
||||
|
|
@ -105,16 +154,37 @@ async def start_session(
|
|||
else:
|
||||
filename = Path(url).name or f"download_{uuid.uuid4().hex[:8]}"
|
||||
dest = STAGING / filename
|
||||
|
||||
# Warn if staging is tight, but don't block (don't know file size)
|
||||
usage = shutil.disk_usage(STAGING)
|
||||
free_gb = usage.free / (1024**3)
|
||||
if free_gb < 50:
|
||||
logger.warning("Low disk space on %s: %.1f GB free — download may fail for large files",
|
||||
STAGING, free_gb)
|
||||
|
||||
logger.info("Starting download: %s → %s", url, dest)
|
||||
try:
|
||||
# --progress=dot:giga prints one dot per 64 KiB downloaded, minimal output
|
||||
# Redirection to stderr keeps stdout clean for error capture
|
||||
result = subprocess.run(
|
||||
["wget", "-q", "--show-progress", "-O", str(dest), url],
|
||||
capture_output=True, text=True, timeout=1800,
|
||||
["wget", "--progress=dot:giga", "-O", str(dest), url],
|
||||
capture_output=True, text=True, timeout=DOWNLOAD_TIMEOUT,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
error = f"Download failed: {result.stderr.strip()[:300]}"
|
||||
else:
|
||||
file_size = dest.stat().st_size / (1024**3) if dest.exists() else 0
|
||||
logger.info("Download complete: %s (%.1f GiB)", filename, file_size)
|
||||
except subprocess.TimeoutExpired:
|
||||
error = "Download timed out (30 min limit)."
|
||||
if dest.exists():
|
||||
dest.unlink(missing_ok=True)
|
||||
error = (
|
||||
f"Download timed out after {DOWNLOAD_TIMEOUT // 3600} hours. "
|
||||
f"The file may be too large for your network speed."
|
||||
)
|
||||
except Exception as exc:
|
||||
if dest.exists():
|
||||
dest.unlink(missing_ok=True)
|
||||
error = f"Download failed: {exc}"
|
||||
|
||||
if error:
|
||||
|
|
@ -127,7 +197,9 @@ async def start_session(
|
|||
return render("_analysis.html", request=request,
|
||||
error=f"Backend analysis failed: {exc.detail}")
|
||||
|
||||
vm_name = (analysis.get("os_type") or "vm") + f"-{vmid}"
|
||||
vm_name = vm_name.strip() if vm_name else ""
|
||||
if not vm_name:
|
||||
vm_name = (analysis.get("os_type") or "vm") + f"-{vmid}"
|
||||
|
||||
return render("_analysis.html", request=request,
|
||||
vmid=vmid, source_filename=filename,
|
||||
|
|
|
|||
|
|
@ -6,6 +6,13 @@
|
|||
<h2>New Conversion Session</h2>
|
||||
|
||||
<form id="session-form" onsubmit="startSession(event)">
|
||||
<div class="form-group">
|
||||
<label for="vm-name">VM Name *</label>
|
||||
<input type="text" id="vm-name" name="vm_name" required
|
||||
placeholder="e.g. debian-test" value="{{ prefill_vmname or '' }}">
|
||||
<span class="hint">Display name for the VM on the Proxmox host.</span>
|
||||
</div>
|
||||
|
||||
<div class="form-group">
|
||||
<label for="vmid">VM ID *</label>
|
||||
<input type="number" id="vmid" name="vmid" required
|
||||
|
|
@ -59,6 +66,7 @@ async function startSession(e) {
|
|||
|
||||
const formData = new FormData();
|
||||
formData.append('vmid', document.getElementById('vmid').value);
|
||||
formData.append('vm_name', document.getElementById('vm-name').value);
|
||||
|
||||
const sourceType = document.getElementById('source-type').value;
|
||||
formData.append('source_type', sourceType);
|
||||
|
|
|
|||
|
|
@ -8,7 +8,8 @@ User=root
|
|||
WorkingDirectory=/mnt/converter/frontend
|
||||
Environment=PATH=/usr/local/bin:/usr/bin:/bin
|
||||
Environment=BACKEND_URL=http://10.2.0.2:9000
|
||||
ExecStart=/usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 5000
|
||||
Environment=TMPDIR=/mnt/converter/in
|
||||
ExecStart=/usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 5000 --timeout-keep-alive 300
|
||||
Restart=always
|
||||
RestartSec=3
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue