fix: raw streaming upload endpoint — bypasses multipart parser for large files

Starlette's multipart parser (pure Python) is too slow for >1 GB uploads —
boundary scanning over gigabytes blocks the event loop indefinitely.

New /session/upload-raw endpoint:
- Receives raw binary body via request.stream() — no parsing overhead
- Metadata (filename, vmid, vm_name) passed in HTTP headers
- Async chunked write directly to staging — true zero-copy streaming
- Same progress logging as before

Frontend now sends File object directly via xhr.send(file)
instead of FormData — eliminates multipart encoding on the client too.
This commit is contained in:
Claus Lohmar 2026-07-22 09:02:05 +00:00
parent 837e674195
commit f3477dc23b
2 changed files with 63 additions and 4 deletions

View file

@ -28,7 +28,7 @@ from typing import Optional
import requests as http_requests import requests as http_requests
from fastapi import FastAPI, Form, Request, UploadFile, File from fastapi import FastAPI, Form, Request, UploadFile, File
from fastapi.responses import HTMLResponse, JSONResponse from fastapi.responses import HTMLResponse, JSONResponse, StreamingResponse
from fastapi.staticfiles import StaticFiles from fastapi.staticfiles import StaticFiles
from jinja2 import Environment, FileSystemLoader from jinja2 import Environment, FileSystemLoader
@ -195,6 +195,57 @@ def session_upload(
"file_size_gb": file_size_gb, "file_size_gb": file_size_gb,
}) })
@app.post("/session/upload-raw")
async def upload_raw(request: Request):
"""Raw streaming upload — bypasses multipart parsing for large files."""
filename = request.headers.get("X-Filename", "upload.bin")
vmid_str = request.headers.get("X-VMID", "")
vm_name = request.headers.get("X-VM-Name", "")
try:
vmid = int(vmid_str)
except ValueError:
return JSONResponse({"phase": "error", "error": "Invalid VM ID."}, status_code=400)
err = _validate_vmid(vmid)
if err:
return JSONResponse({"phase": "error", "error": err}, status_code=400)
dest = STAGING / filename
content_length = request.headers.get("content-length")
if content_length:
estimated_gb = int(content_length) / (1024**3)
err = _check_disk_space(dest, estimated_gb)
if err:
return JSONResponse({"phase": "error", "error": err}, status_code=400)
logger.info("Raw upload: %s (%s bytes)", filename, content_length or "unknown")
written = 0
try:
with dest.open("wb") as f:
async for chunk in request.stream():
f.write(chunk)
written += len(chunk)
if written % UPLOAD_PROGRESS_INTERVAL < len(chunk):
logger.info("Upload progress: %s%.1f GiB", filename, written / (1024**3))
file_size_gb = round(written / (1024**3), 1)
logger.info("Raw upload complete: %s (%.1f GiB)", filename, file_size_gb)
except Exception as exc:
if dest.exists():
dest.unlink(missing_ok=True)
logger.exception("Raw upload failed: %s", filename)
return JSONResponse({"phase": "error", "error": f"Upload failed: {exc}"}, status_code=500)
return JSONResponse({
"phase": "staged",
"filename": filename,
"vmid": vmid,
"vm_name": vm_name,
"file_size_gb": file_size_gb,
})
# ── Download ──────────────────────────────────────────────────── # ── Download ────────────────────────────────────────────────────
url = (source_url or "").strip() url = (source_url or "").strip()
if not url: if not url:

View file

@ -139,8 +139,16 @@ async function startSession(e) {
async function handleUpload(formData) { async function handleUpload(formData) {
return new Promise((resolve) => { return new Promise((resolve) => {
const xhr = new XMLHttpRequest(); const xhr = new XMLHttpRequest();
xhr.open('POST', '/session/upload'); const file = formData.get('source_file');
xhr.timeout = 7200000; // 2 hour timeout for very large uploads const vmid = formData.get('vmid');
const vmName = formData.get('vm_name');
// Raw streaming upload — bypasses multipart parsing
xhr.open('POST', '/session/upload-raw');
xhr.timeout = 7200000;
xhr.setRequestHeader('X-Filename', encodeURIComponent(file.name));
xhr.setRequestHeader('X-VMID', vmid);
xhr.setRequestHeader('X-VM-Name', vmName);
xhr.upload.addEventListener('progress', (e) => { xhr.upload.addEventListener('progress', (e) => {
if (e.lengthComputable) { if (e.lengthComputable) {
@ -173,7 +181,7 @@ async function handleUpload(formData) {
resolve(); resolve();
}); });
xhr.addEventListener('error', () => { setPhase('Upload failed', 0, false); showError('Network error.'); resolve(); }); xhr.addEventListener('error', () => { setPhase('Upload failed', 0, false); showError('Network error.'); resolve(); });
xhr.send(formData); xhr.send(file); // raw binary — no multipart overhead
}); });
} }