fix: large-file support + user-defined VM name
Large-file handling: - Set TMPDIR=/mnt/converter/in in service to spool uploads to shared storage instead of 24 GB LXC rootfs (critical for >24GB) - Chunked upload streaming (8 MiB) with progress logging every 1 GiB - Pre-flight disk space check via Content-Length header - Clean up partial files on upload/download failure - Download timeout extended to 7200s (2 hours) for 88 GB images - Switched wget from --show-progress to --progress=dot:giga (compact output, won't fill memory on large transfers) - uvicorn --timeout-keep-alive 300 on both frontend and backend VM name: - Added vm_name field to initial session form (step 1) - Falls back to auto-generated 'os_type-vmid' if left blank - Pre-filled & editable in confirm form (step 2)
This commit is contained in:
parent
4b50b185fa
commit
509eb97b57
4 changed files with 89 additions and 8 deletions
|
|
@ -7,7 +7,7 @@ Type=simple
|
||||||
User=root
|
User=root
|
||||||
WorkingDirectory=/mnt/converter/backend
|
WorkingDirectory=/mnt/converter/backend
|
||||||
Environment=PATH=/usr/local/bin:/usr/bin:/bin
|
Environment=PATH=/usr/local/bin:/usr/bin:/bin
|
||||||
ExecStart=/usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 9000
|
ExecStart=/usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 9000 --timeout-keep-alive 300
|
||||||
Restart=always
|
Restart=always
|
||||||
RestartSec=3
|
RestartSec=3
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -15,6 +15,7 @@ Routes:
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
import os
|
import os
|
||||||
import shutil
|
import shutil
|
||||||
import subprocess
|
import subprocess
|
||||||
|
|
@ -43,16 +44,36 @@ STAGING = Path("/mnt/converter/in")
|
||||||
|
|
||||||
api = ApiClient()
|
api = ApiClient()
|
||||||
|
|
||||||
|
logger = logging.getLogger("vm-bench")
|
||||||
|
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
||||||
|
|
||||||
# Config
|
# Config
|
||||||
VM_ID_MIN = 21000
|
VM_ID_MIN = 21000
|
||||||
VM_ID_MAX = 21100
|
VM_ID_MAX = 21100
|
||||||
DEFAULT_STORAGE = "local-lvm"
|
DEFAULT_STORAGE = "local-lvm"
|
||||||
|
MIN_FREE_DISK_GB = 2 # keep 2 GB headroom
|
||||||
|
DOWNLOAD_TIMEOUT = 7200 # 2 hours for very large files
|
||||||
|
UPLOAD_PROGRESS_INTERVAL = 1024**3 # log every 1 GiB during upload
|
||||||
|
|
||||||
def _validate_vmid(vmid: int) -> Optional[str]:
|
def _validate_vmid(vmid: int) -> Optional[str]:
|
||||||
if not (VM_ID_MIN <= vmid <= VM_ID_MAX):
|
if not (VM_ID_MIN <= vmid <= VM_ID_MAX):
|
||||||
return f"VM ID must be between {VM_ID_MIN} and {VM_ID_MAX}."
|
return f"VM ID must be between {VM_ID_MIN} and {VM_ID_MAX}."
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
def _check_disk_space(path: Path, needed_gb: int) -> Optional[str]:
|
||||||
|
"""Return an error message if <path> has less than <needed_gb> + headroom free."""
|
||||||
|
usage = shutil.disk_usage(path.parent if path.is_file() or not path.exists() else path)
|
||||||
|
free_gb = usage.free / (1024**3)
|
||||||
|
required = needed_gb + MIN_FREE_DISK_GB
|
||||||
|
if free_gb < required:
|
||||||
|
return (
|
||||||
|
f"Insufficient disk space on {path.parent}: "
|
||||||
|
f"{free_gb:.1f} GB free, {required:.1f} GB needed "
|
||||||
|
f"({needed_gb} GB file + {MIN_FREE_DISK_GB} GB headroom). "
|
||||||
|
f"Free up space or use a smaller file."
|
||||||
|
)
|
||||||
|
return None
|
||||||
|
|
||||||
def render(name: str, status: int = 200, **ctx) -> HTMLResponse:
|
def render(name: str, status: int = 200, **ctx) -> HTMLResponse:
|
||||||
tpl = _jinja.get_template(name)
|
tpl = _jinja.get_template(name)
|
||||||
return HTMLResponse(tpl.render(**ctx), status_code=status)
|
return HTMLResponse(tpl.render(**ctx), status_code=status)
|
||||||
|
|
@ -66,13 +87,15 @@ def render(name: str, status: int = 200, **ctx) -> HTMLResponse:
|
||||||
async def index(request: Request, vmid: Optional[int] = None, source: Optional[str] = None):
|
async def index(request: Request, vmid: Optional[int] = None, source: Optional[str] = None):
|
||||||
"""Landing page — new session form. Optionally pre-fills vmid + source for reuse."""
|
"""Landing page — new session form. Optionally pre-fills vmid + source for reuse."""
|
||||||
return render("index.html", request=request, backend_url=BACKEND_URL,
|
return render("index.html", request=request, backend_url=BACKEND_URL,
|
||||||
prefill_vmid=vmid or "", prefill_source=source or "")
|
prefill_vmid=vmid or "", prefill_source=source or "",
|
||||||
|
prefill_vmname="")
|
||||||
|
|
||||||
|
|
||||||
@app.post("/session/start", response_class=HTMLResponse)
|
@app.post("/session/start", response_class=HTMLResponse)
|
||||||
async def start_session(
|
async def start_session(
|
||||||
request: Request,
|
request: Request,
|
||||||
vmid: int = Form(...),
|
vmid: int = Form(...),
|
||||||
|
vm_name: str = Form(""),
|
||||||
source_type: str = Form("upload"),
|
source_type: str = Form("upload"),
|
||||||
source_file: Optional[UploadFile] = File(None),
|
source_file: Optional[UploadFile] = File(None),
|
||||||
source_url: Optional[str] = Form(None),
|
source_url: Optional[str] = Form(None),
|
||||||
|
|
@ -93,10 +116,36 @@ async def start_session(
|
||||||
else:
|
else:
|
||||||
filename = source_file.filename
|
filename = source_file.filename
|
||||||
dest = STAGING / filename
|
dest = STAGING / filename
|
||||||
|
|
||||||
|
# Estimate size from Content-Length header (may be None)
|
||||||
|
content_length = request.headers.get("content-length")
|
||||||
|
if content_length:
|
||||||
|
estimated_gb = int(content_length) / (1024**3)
|
||||||
|
err = _check_disk_space(dest, estimated_gb)
|
||||||
|
if err:
|
||||||
|
return render("_analysis.html", request=request, error=err)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
logger.info("Receiving upload: %s (%s bytes)", filename, content_length or "unknown")
|
||||||
with dest.open("wb") as f:
|
with dest.open("wb") as f:
|
||||||
shutil.copyfileobj(source_file.file, f)
|
written = 0
|
||||||
|
while True:
|
||||||
|
chunk = source_file.file.read(8 * 1024 * 1024) # 8 MiB chunks
|
||||||
|
if not chunk:
|
||||||
|
break
|
||||||
|
f.write(chunk)
|
||||||
|
written += len(chunk)
|
||||||
|
if written % UPLOAD_PROGRESS_INTERVAL < len(chunk):
|
||||||
|
logger.info("Upload progress: %s — %.1f GiB written", filename, written / (1024**3))
|
||||||
|
logger.info("Upload complete: %s (%.1f GiB)", filename, written / (1024**3))
|
||||||
|
except OSError as exc:
|
||||||
|
# Clean up partial file on disk-full or other IO error
|
||||||
|
if dest.exists():
|
||||||
|
dest.unlink(missing_ok=True)
|
||||||
|
error = f"Upload failed (disk full?): {exc}"
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
|
if dest.exists():
|
||||||
|
dest.unlink(missing_ok=True)
|
||||||
error = f"Upload failed: {exc}"
|
error = f"Upload failed: {exc}"
|
||||||
else:
|
else:
|
||||||
url = (source_url or "").strip()
|
url = (source_url or "").strip()
|
||||||
|
|
@ -105,16 +154,37 @@ async def start_session(
|
||||||
else:
|
else:
|
||||||
filename = Path(url).name or f"download_{uuid.uuid4().hex[:8]}"
|
filename = Path(url).name or f"download_{uuid.uuid4().hex[:8]}"
|
||||||
dest = STAGING / filename
|
dest = STAGING / filename
|
||||||
|
|
||||||
|
# Warn if staging is tight, but don't block (don't know file size)
|
||||||
|
usage = shutil.disk_usage(STAGING)
|
||||||
|
free_gb = usage.free / (1024**3)
|
||||||
|
if free_gb < 50:
|
||||||
|
logger.warning("Low disk space on %s: %.1f GB free — download may fail for large files",
|
||||||
|
STAGING, free_gb)
|
||||||
|
|
||||||
|
logger.info("Starting download: %s → %s", url, dest)
|
||||||
try:
|
try:
|
||||||
|
# --progress=dot:giga prints one dot per 64 KiB downloaded, minimal output
|
||||||
|
# Redirection to stderr keeps stdout clean for error capture
|
||||||
result = subprocess.run(
|
result = subprocess.run(
|
||||||
["wget", "-q", "--show-progress", "-O", str(dest), url],
|
["wget", "--progress=dot:giga", "-O", str(dest), url],
|
||||||
capture_output=True, text=True, timeout=1800,
|
capture_output=True, text=True, timeout=DOWNLOAD_TIMEOUT,
|
||||||
)
|
)
|
||||||
if result.returncode != 0:
|
if result.returncode != 0:
|
||||||
error = f"Download failed: {result.stderr.strip()[:300]}"
|
error = f"Download failed: {result.stderr.strip()[:300]}"
|
||||||
|
else:
|
||||||
|
file_size = dest.stat().st_size / (1024**3) if dest.exists() else 0
|
||||||
|
logger.info("Download complete: %s (%.1f GiB)", filename, file_size)
|
||||||
except subprocess.TimeoutExpired:
|
except subprocess.TimeoutExpired:
|
||||||
error = "Download timed out (30 min limit)."
|
if dest.exists():
|
||||||
|
dest.unlink(missing_ok=True)
|
||||||
|
error = (
|
||||||
|
f"Download timed out after {DOWNLOAD_TIMEOUT // 3600} hours. "
|
||||||
|
f"The file may be too large for your network speed."
|
||||||
|
)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
|
if dest.exists():
|
||||||
|
dest.unlink(missing_ok=True)
|
||||||
error = f"Download failed: {exc}"
|
error = f"Download failed: {exc}"
|
||||||
|
|
||||||
if error:
|
if error:
|
||||||
|
|
@ -127,7 +197,9 @@ async def start_session(
|
||||||
return render("_analysis.html", request=request,
|
return render("_analysis.html", request=request,
|
||||||
error=f"Backend analysis failed: {exc.detail}")
|
error=f"Backend analysis failed: {exc.detail}")
|
||||||
|
|
||||||
vm_name = (analysis.get("os_type") or "vm") + f"-{vmid}"
|
vm_name = vm_name.strip() if vm_name else ""
|
||||||
|
if not vm_name:
|
||||||
|
vm_name = (analysis.get("os_type") or "vm") + f"-{vmid}"
|
||||||
|
|
||||||
return render("_analysis.html", request=request,
|
return render("_analysis.html", request=request,
|
||||||
vmid=vmid, source_filename=filename,
|
vmid=vmid, source_filename=filename,
|
||||||
|
|
|
||||||
|
|
@ -6,6 +6,13 @@
|
||||||
<h2>New Conversion Session</h2>
|
<h2>New Conversion Session</h2>
|
||||||
|
|
||||||
<form id="session-form" onsubmit="startSession(event)">
|
<form id="session-form" onsubmit="startSession(event)">
|
||||||
|
<div class="form-group">
|
||||||
|
<label for="vm-name">VM Name *</label>
|
||||||
|
<input type="text" id="vm-name" name="vm_name" required
|
||||||
|
placeholder="e.g. debian-test" value="{{ prefill_vmname or '' }}">
|
||||||
|
<span class="hint">Display name for the VM on the Proxmox host.</span>
|
||||||
|
</div>
|
||||||
|
|
||||||
<div class="form-group">
|
<div class="form-group">
|
||||||
<label for="vmid">VM ID *</label>
|
<label for="vmid">VM ID *</label>
|
||||||
<input type="number" id="vmid" name="vmid" required
|
<input type="number" id="vmid" name="vmid" required
|
||||||
|
|
@ -59,6 +66,7 @@ async function startSession(e) {
|
||||||
|
|
||||||
const formData = new FormData();
|
const formData = new FormData();
|
||||||
formData.append('vmid', document.getElementById('vmid').value);
|
formData.append('vmid', document.getElementById('vmid').value);
|
||||||
|
formData.append('vm_name', document.getElementById('vm-name').value);
|
||||||
|
|
||||||
const sourceType = document.getElementById('source-type').value;
|
const sourceType = document.getElementById('source-type').value;
|
||||||
formData.append('source_type', sourceType);
|
formData.append('source_type', sourceType);
|
||||||
|
|
|
||||||
|
|
@ -8,7 +8,8 @@ User=root
|
||||||
WorkingDirectory=/mnt/converter/frontend
|
WorkingDirectory=/mnt/converter/frontend
|
||||||
Environment=PATH=/usr/local/bin:/usr/bin:/bin
|
Environment=PATH=/usr/local/bin:/usr/bin:/bin
|
||||||
Environment=BACKEND_URL=http://10.2.0.2:9000
|
Environment=BACKEND_URL=http://10.2.0.2:9000
|
||||||
ExecStart=/usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 5000
|
Environment=TMPDIR=/mnt/converter/in
|
||||||
|
ExecStart=/usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 5000 --timeout-keep-alive 300
|
||||||
Restart=always
|
Restart=always
|
||||||
RestartSec=3
|
RestartSec=3
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue