fix: raw streaming upload endpoint — bypasses multipart parser for large files

Starlette's multipart parser (pure Python) is too slow for >1 GB uploads —
boundary scanning over gigabytes blocks the event loop indefinitely.

New /session/upload-raw endpoint:
- Receives raw binary body via request.stream() — no parsing overhead
- Metadata (filename, vmid, vm_name) passed in HTTP headers
- Async chunked write directly to staging — true zero-copy streaming
- Same progress logging as before

Frontend now sends File object directly via xhr.send(file)
instead of FormData — eliminates multipart encoding on the client too.
This commit is contained in:
Claus Lohmar 2026-07-22 09:02:05 +00:00
parent 837e674195
commit f3477dc23b
2 changed files with 63 additions and 4 deletions

View file

@ -28,7 +28,7 @@ from typing import Optional
import requests as http_requests
from fastapi import FastAPI, Form, Request, UploadFile, File
from fastapi.responses import HTMLResponse, JSONResponse
from fastapi.responses import HTMLResponse, JSONResponse, StreamingResponse
from fastapi.staticfiles import StaticFiles
from jinja2 import Environment, FileSystemLoader
@ -195,6 +195,57 @@ def session_upload(
"file_size_gb": file_size_gb,
})
@app.post("/session/upload-raw")
async def upload_raw(request: Request):
"""Raw streaming upload — bypasses multipart parsing for large files."""
filename = request.headers.get("X-Filename", "upload.bin")
vmid_str = request.headers.get("X-VMID", "")
vm_name = request.headers.get("X-VM-Name", "")
try:
vmid = int(vmid_str)
except ValueError:
return JSONResponse({"phase": "error", "error": "Invalid VM ID."}, status_code=400)
err = _validate_vmid(vmid)
if err:
return JSONResponse({"phase": "error", "error": err}, status_code=400)
dest = STAGING / filename
content_length = request.headers.get("content-length")
if content_length:
estimated_gb = int(content_length) / (1024**3)
err = _check_disk_space(dest, estimated_gb)
if err:
return JSONResponse({"phase": "error", "error": err}, status_code=400)
logger.info("Raw upload: %s (%s bytes)", filename, content_length or "unknown")
written = 0
try:
with dest.open("wb") as f:
async for chunk in request.stream():
f.write(chunk)
written += len(chunk)
if written % UPLOAD_PROGRESS_INTERVAL < len(chunk):
logger.info("Upload progress: %s%.1f GiB", filename, written / (1024**3))
file_size_gb = round(written / (1024**3), 1)
logger.info("Raw upload complete: %s (%.1f GiB)", filename, file_size_gb)
except Exception as exc:
if dest.exists():
dest.unlink(missing_ok=True)
logger.exception("Raw upload failed: %s", filename)
return JSONResponse({"phase": "error", "error": f"Upload failed: {exc}"}, status_code=500)
return JSONResponse({
"phase": "staged",
"filename": filename,
"vmid": vmid,
"vm_name": vm_name,
"file_size_gb": file_size_gb,
})
# ── Download ────────────────────────────────────────────────────
url = (source_url or "").strip()
if not url:

View file

@ -139,8 +139,16 @@ async function startSession(e) {
async function handleUpload(formData) {
return new Promise((resolve) => {
const xhr = new XMLHttpRequest();
xhr.open('POST', '/session/upload');
xhr.timeout = 7200000; // 2 hour timeout for very large uploads
const file = formData.get('source_file');
const vmid = formData.get('vmid');
const vmName = formData.get('vm_name');
// Raw streaming upload — bypasses multipart parsing
xhr.open('POST', '/session/upload-raw');
xhr.timeout = 7200000;
xhr.setRequestHeader('X-Filename', encodeURIComponent(file.name));
xhr.setRequestHeader('X-VMID', vmid);
xhr.setRequestHeader('X-VM-Name', vmName);
xhr.upload.addEventListener('progress', (e) => {
if (e.lengthComputable) {
@ -173,7 +181,7 @@ async function handleUpload(formData) {
resolve();
});
xhr.addEventListener('error', () => { setPhase('Upload failed', 0, false); showError('Network error.'); resolve(); });
xhr.send(formData);
xhr.send(file); // raw binary — no multipart overhead
});
}