diff --git a/frontend/app.py b/frontend/app.py index 6f08985..5f70c96 100644 --- a/frontend/app.py +++ b/frontend/app.py @@ -455,6 +455,128 @@ async def session_cleanup(job_id: str, request: Request): # --------------------------------------------------------------------------- # Run # --------------------------------------------------------------------------- +# Track active SCP pulls for progress polling +_active_scp: dict[str, dict] = {} + + +# --------------------------------------------------------------------------- +# SCP Pull — large file transfer from remote servers +# --------------------------------------------------------------------------- + +@app.get("/scp", response_class=HTMLResponse) +async def scp_page(request: Request): + """SCP pull page — fetch files directly from remote servers.""" + return render("scp.html", request=request, backend_url=BACKEND_URL, + session_id=uuid.uuid4().hex) + + +@app.post("/scp/start") +def scp_start( + request: Request, + scp_host: str = Form(...), + scp_port: int = Form(22), + scp_user: str = Form(...), + scp_pass: str = Form(...), + scp_path: str = Form(...), + session_id: str = Form(""), +): + """Start an SCP pull in the background. Returns JSON with phase + filename.""" + remote = f"{scp_user}@{scp_host}:{scp_path}" + filename = Path(scp_path).name + if not filename: + return JSONResponse({"phase": "error", "error": "Invalid remote path."}, status_code=400) + + session_dir = STAGING / session_id if session_id else STAGING + session_dir.mkdir(parents=True, exist_ok=True) + dest = session_dir / filename + + # Check disk space + usage = shutil.disk_usage(STAGING) + free_gb = usage.free / (1024**3) + if free_gb < 10: + logger.warning("Low disk: %.1f GB free — SCP pull may fail", free_gb) + + env = os.environ.copy() + env["SSHPASS"] = scp_pass + + cmd = [ + "sshpass", "-e", + "scp", + "-o", "StrictHostKeyChecking=no", + "-o", "ConnectTimeout=10", + "-P", str(scp_port), + remote, str(dest), + ] + + logger.info("Starting SCP pull: %s → %s", remote, dest) + try: + proc = subprocess.Popen(cmd, env=env, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + except Exception as exc: + return JSONResponse({"phase": "error", "error": f"Failed to start SCP: {exc}"}, status_code=500) + + key = f"{session_id}/{filename}" + _active_scp[key] = { + "proc": proc, + "dest": dest, + "start_time": time.time(), + } + + return JSONResponse({ + "phase": "pulling", + "filename": filename, + "session_id": session_id, + }) + + +@app.get("/scp/progress/{session_id}/{filename}") +def scp_progress(session_id: str, filename: str): + """Poll SCP progress by checking output file size.""" + key = f"{session_id}/{filename}" + info = _active_scp.get(key) + if not info: + dest = STAGING / session_id / filename + if dest.exists(): + return JSONResponse({ + "phase": "complete", + "file_size_bytes": dest.stat().st_size, + "file_size_gb": round(dest.stat().st_size / (1024**3), 1), + }) + return JSONResponse({"phase": "unknown", "error": "No active SCP pull."}, status_code=404) + + proc = info["proc"] + dest = info["dest"] + current_bytes = dest.stat().st_size if dest.exists() else 0 + + poll = proc.poll() + if poll is not None: + _active_scp.pop(key, None) + if poll != 0: + if dest.exists(): + dest.unlink(missing_ok=True) + return JSONResponse({ + "phase": "error", + "error": f"SCP pull failed (exit code {poll}). Check credentials and remote path.", + "file_size_bytes": current_bytes, + }) + final_bytes = dest.stat().st_size + logger.info("SCP pull complete: %s (%.1f GiB)", filename, final_bytes / (1024**3)) + return JSONResponse({ + "phase": "complete", + "file_size_bytes": final_bytes, + "file_size_gb": round(final_bytes / (1024**3), 1), + }) + + elapsed = max(time.time() - info.get("start_time", 0), 1) + speed_mbps = round(current_bytes / elapsed / 1_000_000, 1) + + return JSONResponse({ + "phase": "pulling", + "file_size_bytes": current_bytes, + "file_size_gb": round(current_bytes / (1024**3), 1), + "speed_mbps": speed_mbps, + }) + + if __name__ == "__main__": import uvicorn uvicorn.run(app, host="0.0.0.0", port=5000) diff --git a/frontend/install.sh b/frontend/install.sh index 0f66c98..1322734 100755 --- a/frontend/install.sh +++ b/frontend/install.sh @@ -37,6 +37,8 @@ REQUIRED=( python3-pip "pip3" wget "wget" curl "curl" + openssh-client "scp" + sshpass "sshpass" ) for ((i=0; i<${#REQUIRED[@]}; i+=2)); do diff --git a/frontend/templates/index.html b/frontend/templates/index.html index d2a8abd..6af05da 100644 --- a/frontend/templates/index.html +++ b/frontend/templates/index.html @@ -43,6 +43,15 @@ +
+ For files larger than 10 GB or on remote servers — pull directly via SCP: +
+ + SCP Pull (Large Files +10 GB) + ++ Pull a disk image directly from a customer server via SCP. + No laptop middle-hop — files land directly in the staging area. + Supports files of any size. +
+ + + + +