fix: raw streaming upload endpoint — bypasses multipart parser for large files
Starlette's multipart parser (pure Python) is too slow for >1 GB uploads — boundary scanning over gigabytes blocks the event loop indefinitely. New /session/upload-raw endpoint: - Receives raw binary body via request.stream() — no parsing overhead - Metadata (filename, vmid, vm_name) passed in HTTP headers - Async chunked write directly to staging — true zero-copy streaming - Same progress logging as before Frontend now sends File object directly via xhr.send(file) instead of FormData — eliminates multipart encoding on the client too.
This commit is contained in:
parent
837e674195
commit
f3477dc23b
2 changed files with 63 additions and 4 deletions
|
|
@ -28,7 +28,7 @@ from typing import Optional
|
||||||
import requests as http_requests
|
import requests as http_requests
|
||||||
|
|
||||||
from fastapi import FastAPI, Form, Request, UploadFile, File
|
from fastapi import FastAPI, Form, Request, UploadFile, File
|
||||||
from fastapi.responses import HTMLResponse, JSONResponse
|
from fastapi.responses import HTMLResponse, JSONResponse, StreamingResponse
|
||||||
from fastapi.staticfiles import StaticFiles
|
from fastapi.staticfiles import StaticFiles
|
||||||
|
|
||||||
from jinja2 import Environment, FileSystemLoader
|
from jinja2 import Environment, FileSystemLoader
|
||||||
|
|
@ -195,6 +195,57 @@ def session_upload(
|
||||||
"file_size_gb": file_size_gb,
|
"file_size_gb": file_size_gb,
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|
||||||
|
@app.post("/session/upload-raw")
|
||||||
|
async def upload_raw(request: Request):
|
||||||
|
"""Raw streaming upload — bypasses multipart parsing for large files."""
|
||||||
|
filename = request.headers.get("X-Filename", "upload.bin")
|
||||||
|
vmid_str = request.headers.get("X-VMID", "")
|
||||||
|
vm_name = request.headers.get("X-VM-Name", "")
|
||||||
|
|
||||||
|
try:
|
||||||
|
vmid = int(vmid_str)
|
||||||
|
except ValueError:
|
||||||
|
return JSONResponse({"phase": "error", "error": "Invalid VM ID."}, status_code=400)
|
||||||
|
|
||||||
|
err = _validate_vmid(vmid)
|
||||||
|
if err:
|
||||||
|
return JSONResponse({"phase": "error", "error": err}, status_code=400)
|
||||||
|
|
||||||
|
dest = STAGING / filename
|
||||||
|
content_length = request.headers.get("content-length")
|
||||||
|
if content_length:
|
||||||
|
estimated_gb = int(content_length) / (1024**3)
|
||||||
|
err = _check_disk_space(dest, estimated_gb)
|
||||||
|
if err:
|
||||||
|
return JSONResponse({"phase": "error", "error": err}, status_code=400)
|
||||||
|
|
||||||
|
logger.info("Raw upload: %s (%s bytes)", filename, content_length or "unknown")
|
||||||
|
written = 0
|
||||||
|
try:
|
||||||
|
with dest.open("wb") as f:
|
||||||
|
async for chunk in request.stream():
|
||||||
|
f.write(chunk)
|
||||||
|
written += len(chunk)
|
||||||
|
if written % UPLOAD_PROGRESS_INTERVAL < len(chunk):
|
||||||
|
logger.info("Upload progress: %s — %.1f GiB", filename, written / (1024**3))
|
||||||
|
file_size_gb = round(written / (1024**3), 1)
|
||||||
|
logger.info("Raw upload complete: %s (%.1f GiB)", filename, file_size_gb)
|
||||||
|
except Exception as exc:
|
||||||
|
if dest.exists():
|
||||||
|
dest.unlink(missing_ok=True)
|
||||||
|
logger.exception("Raw upload failed: %s", filename)
|
||||||
|
return JSONResponse({"phase": "error", "error": f"Upload failed: {exc}"}, status_code=500)
|
||||||
|
|
||||||
|
return JSONResponse({
|
||||||
|
"phase": "staged",
|
||||||
|
"filename": filename,
|
||||||
|
"vmid": vmid,
|
||||||
|
"vm_name": vm_name,
|
||||||
|
"file_size_gb": file_size_gb,
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
# ── Download ────────────────────────────────────────────────────
|
# ── Download ────────────────────────────────────────────────────
|
||||||
url = (source_url or "").strip()
|
url = (source_url or "").strip()
|
||||||
if not url:
|
if not url:
|
||||||
|
|
|
||||||
|
|
@ -139,8 +139,16 @@ async function startSession(e) {
|
||||||
async function handleUpload(formData) {
|
async function handleUpload(formData) {
|
||||||
return new Promise((resolve) => {
|
return new Promise((resolve) => {
|
||||||
const xhr = new XMLHttpRequest();
|
const xhr = new XMLHttpRequest();
|
||||||
xhr.open('POST', '/session/upload');
|
const file = formData.get('source_file');
|
||||||
xhr.timeout = 7200000; // 2 hour timeout for very large uploads
|
const vmid = formData.get('vmid');
|
||||||
|
const vmName = formData.get('vm_name');
|
||||||
|
|
||||||
|
// Raw streaming upload — bypasses multipart parsing
|
||||||
|
xhr.open('POST', '/session/upload-raw');
|
||||||
|
xhr.timeout = 7200000;
|
||||||
|
xhr.setRequestHeader('X-Filename', encodeURIComponent(file.name));
|
||||||
|
xhr.setRequestHeader('X-VMID', vmid);
|
||||||
|
xhr.setRequestHeader('X-VM-Name', vmName);
|
||||||
|
|
||||||
xhr.upload.addEventListener('progress', (e) => {
|
xhr.upload.addEventListener('progress', (e) => {
|
||||||
if (e.lengthComputable) {
|
if (e.lengthComputable) {
|
||||||
|
|
@ -173,7 +181,7 @@ async function handleUpload(formData) {
|
||||||
resolve();
|
resolve();
|
||||||
});
|
});
|
||||||
xhr.addEventListener('error', () => { setPhase('Upload failed', 0, false); showError('Network error.'); resolve(); });
|
xhr.addEventListener('error', () => { setPhase('Upload failed', 0, false); showError('Network error.'); resolve(); });
|
||||||
xhr.send(formData);
|
xhr.send(file); // raw binary — no multipart overhead
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue