From f3477dc23bb4e9c97811cfc07d2702e677026ec3 Mon Sep 17 00:00:00 2001
From: Claus Lohmar <claus.lohmar@lohmar.co.uk>
Date: Wed, 22 Jul 2026 09:02:05 +0000
Subject: [PATCH] =?UTF-8?q?fix:=20raw=20streaming=20upload=20endpoint=20?=
 =?UTF-8?q?=E2=80=94=20bypasses=20multipart=20parser=20for=20large=20files?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Starlette's multipart parser (pure Python) is too slow for >1 GB uploads —
boundary scanning over gigabytes blocks the event loop indefinitely.

New /session/upload-raw endpoint:
- Receives raw binary body via request.stream() — no parsing overhead
- Metadata (filename, vmid, vm_name) passed in HTTP headers
- Async chunked write directly to staging — true zero-copy streaming
- Same progress logging as before

Frontend now sends File object directly via xhr.send(file)
instead of FormData — eliminates multipart encoding on the client too.
---
 frontend/app.py               | 53 ++++++++++++++++++++++++++++++++++-
 frontend/templates/index.html | 14 +++++++--
 2 files changed, 63 insertions(+), 4 deletions(-)

diff --git a/frontend/app.py b/frontend/app.py
index 5f70c96..b05b61c 100644
--- a/frontend/app.py
+++ b/frontend/app.py
@@ -28,7 +28,7 @@ from typing import Optional
 import requests as http_requests
 
 from fastapi import FastAPI, Form, Request, UploadFile, File
-from fastapi.responses import HTMLResponse, JSONResponse
+from fastapi.responses import HTMLResponse, JSONResponse, StreamingResponse
 from fastapi.staticfiles import StaticFiles
 
 from jinja2 import Environment, FileSystemLoader
@@ -195,6 +195,57 @@ def session_upload(
             "file_size_gb": file_size_gb,
         })
 
+
+@app.post("/session/upload-raw")
+async def upload_raw(request: Request):
+    """Raw streaming upload — bypasses multipart parsing for large files."""
+    filename = request.headers.get("X-Filename", "upload.bin")
+    vmid_str = request.headers.get("X-VMID", "")
+    vm_name = request.headers.get("X-VM-Name", "")
+
+    try:
+        vmid = int(vmid_str)
+    except ValueError:
+        return JSONResponse({"phase": "error", "error": "Invalid VM ID."}, status_code=400)
+
+    err = _validate_vmid(vmid)
+    if err:
+        return JSONResponse({"phase": "error", "error": err}, status_code=400)
+
+    dest = STAGING / filename
+    content_length = request.headers.get("content-length")
+    if content_length:
+        estimated_gb = int(content_length) / (1024**3)
+        err = _check_disk_space(dest, estimated_gb)
+        if err:
+            return JSONResponse({"phase": "error", "error": err}, status_code=400)
+
+    logger.info("Raw upload: %s (%s bytes)", filename, content_length or "unknown")
+    written = 0
+    try:
+        with dest.open("wb") as f:
+            async for chunk in request.stream():
+                f.write(chunk)
+                written += len(chunk)
+                if written % UPLOAD_PROGRESS_INTERVAL < len(chunk):
+                    logger.info("Upload progress: %s — %.1f GiB", filename, written / (1024**3))
+        file_size_gb = round(written / (1024**3), 1)
+        logger.info("Raw upload complete: %s (%.1f GiB)", filename, file_size_gb)
+    except Exception as exc:
+        if dest.exists():
+            dest.unlink(missing_ok=True)
+        logger.exception("Raw upload failed: %s", filename)
+        return JSONResponse({"phase": "error", "error": f"Upload failed: {exc}"}, status_code=500)
+
+    return JSONResponse({
+        "phase": "staged",
+        "filename": filename,
+        "vmid": vmid,
+        "vm_name": vm_name,
+        "file_size_gb": file_size_gb,
+    })
+
+
     # ── Download ────────────────────────────────────────────────────
     url = (source_url or "").strip()
     if not url:
diff --git a/frontend/templates/index.html b/frontend/templates/index.html
index 6af05da..e6ba601 100644
--- a/frontend/templates/index.html
+++ b/frontend/templates/index.html
@@ -139,8 +139,16 @@ async function startSession(e) {
 async function handleUpload(formData) {
   return new Promise((resolve) => {
     const xhr = new XMLHttpRequest();
-    xhr.open('POST', '/session/upload');
-    xhr.timeout = 7200000;  // 2 hour timeout for very large uploads
+    const file = formData.get('source_file');
+    const vmid = formData.get('vmid');
+    const vmName = formData.get('vm_name');
+
+    // Raw streaming upload — bypasses multipart parsing
+    xhr.open('POST', '/session/upload-raw');
+    xhr.timeout = 7200000;
+    xhr.setRequestHeader('X-Filename', encodeURIComponent(file.name));
+    xhr.setRequestHeader('X-VMID', vmid);
+    xhr.setRequestHeader('X-VM-Name', vmName);
 
     xhr.upload.addEventListener('progress', (e) => {
       if (e.lengthComputable) {
@@ -173,7 +181,7 @@ async function handleUpload(formData) {
       resolve();
     });
     xhr.addEventListener('error', () => { setPhase('Upload failed', 0, false); showError('Network error.'); resolve(); });
-    xhr.send(formData);
+    xhr.send(file);  // raw binary — no multipart overhead
   });
 }