fix: raw streaming upload endpoint — bypasses multipart parser for large files
Starlette's multipart parser (pure Python) is too slow for >1 GB uploads — boundary scanning over gigabytes blocks the event loop indefinitely. New /session/upload-raw endpoint: - Receives raw binary body via request.stream() — no parsing overhead - Metadata (filename, vmid, vm_name) passed in HTTP headers - Async chunked write directly to staging — true zero-copy streaming - Same progress logging as before Frontend now sends File object directly via xhr.send(file) instead of FormData — eliminates multipart encoding on the client too.
This commit is contained in:
parent
837e674195
commit
f3477dc23b
2 changed files with 63 additions and 4 deletions
|
|
@ -28,7 +28,7 @@ from typing import Optional
|
|||
import requests as http_requests
|
||||
|
||||
from fastapi import FastAPI, Form, Request, UploadFile, File
|
||||
from fastapi.responses import HTMLResponse, JSONResponse
|
||||
from fastapi.responses import HTMLResponse, JSONResponse, StreamingResponse
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
|
||||
from jinja2 import Environment, FileSystemLoader
|
||||
|
|
@ -195,6 +195,57 @@ def session_upload(
|
|||
"file_size_gb": file_size_gb,
|
||||
})
|
||||
|
||||
|
||||
@app.post("/session/upload-raw")
|
||||
async def upload_raw(request: Request):
|
||||
"""Raw streaming upload — bypasses multipart parsing for large files."""
|
||||
filename = request.headers.get("X-Filename", "upload.bin")
|
||||
vmid_str = request.headers.get("X-VMID", "")
|
||||
vm_name = request.headers.get("X-VM-Name", "")
|
||||
|
||||
try:
|
||||
vmid = int(vmid_str)
|
||||
except ValueError:
|
||||
return JSONResponse({"phase": "error", "error": "Invalid VM ID."}, status_code=400)
|
||||
|
||||
err = _validate_vmid(vmid)
|
||||
if err:
|
||||
return JSONResponse({"phase": "error", "error": err}, status_code=400)
|
||||
|
||||
dest = STAGING / filename
|
||||
content_length = request.headers.get("content-length")
|
||||
if content_length:
|
||||
estimated_gb = int(content_length) / (1024**3)
|
||||
err = _check_disk_space(dest, estimated_gb)
|
||||
if err:
|
||||
return JSONResponse({"phase": "error", "error": err}, status_code=400)
|
||||
|
||||
logger.info("Raw upload: %s (%s bytes)", filename, content_length or "unknown")
|
||||
written = 0
|
||||
try:
|
||||
with dest.open("wb") as f:
|
||||
async for chunk in request.stream():
|
||||
f.write(chunk)
|
||||
written += len(chunk)
|
||||
if written % UPLOAD_PROGRESS_INTERVAL < len(chunk):
|
||||
logger.info("Upload progress: %s — %.1f GiB", filename, written / (1024**3))
|
||||
file_size_gb = round(written / (1024**3), 1)
|
||||
logger.info("Raw upload complete: %s (%.1f GiB)", filename, file_size_gb)
|
||||
except Exception as exc:
|
||||
if dest.exists():
|
||||
dest.unlink(missing_ok=True)
|
||||
logger.exception("Raw upload failed: %s", filename)
|
||||
return JSONResponse({"phase": "error", "error": f"Upload failed: {exc}"}, status_code=500)
|
||||
|
||||
return JSONResponse({
|
||||
"phase": "staged",
|
||||
"filename": filename,
|
||||
"vmid": vmid,
|
||||
"vm_name": vm_name,
|
||||
"file_size_gb": file_size_gb,
|
||||
})
|
||||
|
||||
|
||||
# ── Download ────────────────────────────────────────────────────
|
||||
url = (source_url or "").strip()
|
||||
if not url:
|
||||
|
|
|
|||
|
|
@ -139,8 +139,16 @@ async function startSession(e) {
|
|||
async function handleUpload(formData) {
|
||||
return new Promise((resolve) => {
|
||||
const xhr = new XMLHttpRequest();
|
||||
xhr.open('POST', '/session/upload');
|
||||
xhr.timeout = 7200000; // 2 hour timeout for very large uploads
|
||||
const file = formData.get('source_file');
|
||||
const vmid = formData.get('vmid');
|
||||
const vmName = formData.get('vm_name');
|
||||
|
||||
// Raw streaming upload — bypasses multipart parsing
|
||||
xhr.open('POST', '/session/upload-raw');
|
||||
xhr.timeout = 7200000;
|
||||
xhr.setRequestHeader('X-Filename', encodeURIComponent(file.name));
|
||||
xhr.setRequestHeader('X-VMID', vmid);
|
||||
xhr.setRequestHeader('X-VM-Name', vmName);
|
||||
|
||||
xhr.upload.addEventListener('progress', (e) => {
|
||||
if (e.lengthComputable) {
|
||||
|
|
@ -173,7 +181,7 @@ async function handleUpload(formData) {
|
|||
resolve();
|
||||
});
|
||||
xhr.addEventListener('error', () => { setPhase('Upload failed', 0, false); showError('Network error.'); resolve(); });
|
||||
xhr.send(formData);
|
||||
xhr.send(file); // raw binary — no multipart overhead
|
||||
});
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue