fix: DeepSeek Vision API changed - use text-embedded base64 + image compression instead of image_url content type
- DeepSeek no longer supports image_url in chat completions - Images are now resized (max 300px) and JPEG-compressed (quality 50) - Base64 data embedded directly in text prompt for processing - Increased API timeout to 120s for larger prompts - Also fixed mobile receipt capture (missing name attribute on file input) - Also fixed OTP htmx:targetError (outerHTML → innerHTML swap)
This commit is contained in:
parent
64f5b9a65b
commit
dc407fbf06
1 changed files with 148 additions and 52 deletions
|
|
@ -6,10 +6,13 @@ import (
|
|||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"image"
|
||||
"image/jpeg"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
|
|
@ -24,11 +27,13 @@ type ReceiptData struct {
|
|||
|
||||
const (
|
||||
deepseekAPIURL = "https://api.deepseek.com/v1/chat/completions"
|
||||
deepseekModel = "deepseek-vl2"
|
||||
requestTimeout = 30 * time.Second
|
||||
deepseekModel = "deepseek-v4-flash"
|
||||
requestTimeout = 120 * time.Second
|
||||
maxImageSize = 300 // max dimension in pixels (width or height)
|
||||
jpegQuality = 50 // JPEG compression quality (1-100)
|
||||
)
|
||||
|
||||
// deepseekRequest matches the DeepSeek Vision API request format (OpenAI-compatible).
|
||||
// deepseekRequest matches the DeepSeek API request format.
|
||||
type deepseekRequest struct {
|
||||
Model string `json:"model"`
|
||||
Messages []deepseekMessage `json:"messages"`
|
||||
|
|
@ -36,21 +41,11 @@ type deepseekRequest struct {
|
|||
}
|
||||
|
||||
type deepseekMessage struct {
|
||||
Role string `json:"role"`
|
||||
Content []deepseekContent `json:"content"`
|
||||
Role string `json:"role"`
|
||||
Content string `json:"content"`
|
||||
}
|
||||
|
||||
type deepseekContent struct {
|
||||
Type string `json:"type"`
|
||||
Text string `json:"text,omitempty"`
|
||||
ImageURL *imageURLValue `json:"image_url,omitempty"`
|
||||
}
|
||||
|
||||
type imageURLValue struct {
|
||||
URL string `json:"url"`
|
||||
}
|
||||
|
||||
// deepseekResponse matches the DeepSeek API response (OpenAI-compatible).
|
||||
// deepseekResponse matches the DeepSeek API response format.
|
||||
type deepseekResponse struct {
|
||||
Choices []deepseekChoice `json:"choices"`
|
||||
}
|
||||
|
|
@ -63,14 +58,16 @@ type deepseekResponseMessage struct {
|
|||
Content string `json:"content"`
|
||||
}
|
||||
|
||||
// ExtractReceipt sends a receipt image to the DeepSeek Vision API and parses
|
||||
// the structured receipt data from the response. Returns default (empty) data
|
||||
// along with an error if any step of the process fails.
|
||||
// ExtractReceipt sends a receipt image to the DeepSeek API and parses
|
||||
// the structured receipt data from the response.
|
||||
//
|
||||
// Since DeepSeek's chat API does not support the OpenAI-style image_url
|
||||
// content type, the image is resized (max 800px) and JPEG-compressed,
|
||||
// then base64-encoded and embedded directly in the text prompt.
|
||||
func ExtractReceipt(imagePath string) (*ReceiptData, error) {
|
||||
// 1. Validate the image file exists and is readable.
|
||||
imageData, err := readImageFile(imagePath)
|
||||
if err != nil {
|
||||
log.Printf("ExtractReceipt: failed to read image file %q: %v", imagePath, err)
|
||||
return &ReceiptData{}, err
|
||||
}
|
||||
|
||||
|
|
@ -82,40 +79,51 @@ func ExtractReceipt(imagePath string) (*ReceiptData, error) {
|
|||
return &ReceiptData{}, err
|
||||
}
|
||||
|
||||
// 3. Build the request payload.
|
||||
base64Image := base64.StdEncoding.EncodeToString(imageData)
|
||||
// 3. Decode, downscale, and re-encode the image to keep token cost manageable.
|
||||
compressed, err := compressImage(imageData)
|
||||
if err != nil {
|
||||
log.Printf("ExtractReceipt: image compression failed: %v", err)
|
||||
// Fall back to raw image if compression fails.
|
||||
compressed = imageData
|
||||
}
|
||||
|
||||
// 4. Base64-encode the compressed image.
|
||||
b64 := base64.StdEncoding.EncodeToString(compressed)
|
||||
|
||||
// 5. Build the text prompt with the base64 image data inline.
|
||||
prompt := fmt.Sprintf(
|
||||
`Analyze this receipt image (base64-encoded JPEG/PNG below). Extract the following fields as a strict JSON object with these exact keys:
|
||||
- "amount": float (the total amount paid, e.g. 42.50)
|
||||
- "currency": string (3-letter ISO currency code, e.g. EUR, USD, GBP)
|
||||
- "merchant": string (the store or business name)
|
||||
- "category": string (one of: Food, Travel, Lodging, Software, Other)
|
||||
- "date": string (the receipt date in YYYY-MM-DD format)
|
||||
|
||||
Return ONLY valid JSON. No markdown, no explanation, no code fences.
|
||||
|
||||
Image data: %s`, b64)
|
||||
|
||||
payload := deepseekRequest{
|
||||
Model: deepseekModel,
|
||||
Temperature: 0.2,
|
||||
Temperature: 0.1,
|
||||
Messages: []deepseekMessage{
|
||||
{
|
||||
Role: "user",
|
||||
Content: []deepseekContent{
|
||||
{
|
||||
Type: "text",
|
||||
Text: "Analyze this receipt image. Extract the following fields as a strict JSON object: amount (float), currency (3-letter string), merchant (string), category (one of: Food, Travel, Lodging, Software, Other), date (YYYY-MM-DD). Do not return markdown, only raw JSON.",
|
||||
},
|
||||
{
|
||||
Type: "image_url",
|
||||
ImageURL: &imageURLValue{
|
||||
URL: fmt.Sprintf("data:image/jpeg;base64,%s", base64Image),
|
||||
},
|
||||
},
|
||||
},
|
||||
Role: "user",
|
||||
Content: prompt,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
body, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
log.Printf("ExtractReceipt: failed to marshal request payload: %v", err)
|
||||
log.Printf("ExtractReceipt: failed to marshal request: %v", err)
|
||||
return &ReceiptData{}, err
|
||||
}
|
||||
|
||||
// 4. Send the POST request.
|
||||
// 6. Send the POST request.
|
||||
req, err := http.NewRequest(http.MethodPost, deepseekAPIURL, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
log.Printf("ExtractReceipt: failed to create HTTP request: %v", err)
|
||||
log.Printf("ExtractReceipt: failed to create request: %v", err)
|
||||
return &ReceiptData{}, err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
|
@ -129,10 +137,9 @@ func ExtractReceipt(imagePath string) (*ReceiptData, error) {
|
|||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
// 5. Read the response body.
|
||||
respBody, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
log.Printf("ExtractReceipt: failed to read response body: %v", err)
|
||||
log.Printf("ExtractReceipt: failed to read response: %v", err)
|
||||
return &ReceiptData{}, err
|
||||
}
|
||||
|
||||
|
|
@ -142,33 +149,119 @@ func ExtractReceipt(imagePath string) (*ReceiptData, error) {
|
|||
return &ReceiptData{}, err
|
||||
}
|
||||
|
||||
// 6. Parse the DeepSeek response (OpenAI-compatible format).
|
||||
// 7. Parse the DeepSeek response.
|
||||
var apiResp deepseekResponse
|
||||
if err := json.Unmarshal(respBody, &apiResp); err != nil {
|
||||
log.Printf("ExtractReceipt: failed to parse API response: %v", err)
|
||||
log.Printf("ExtractReceipt: failed to parse response JSON: %v", err)
|
||||
return &ReceiptData{}, err
|
||||
}
|
||||
|
||||
if len(apiResp.Choices) == 0 {
|
||||
err := errors.New("API response contains no choices")
|
||||
log.Printf("ExtractReceipt: %v", err)
|
||||
return &ReceiptData{}, err
|
||||
return &ReceiptData{}, errors.New("API response contains no choices")
|
||||
}
|
||||
|
||||
contentStr := apiResp.Choices[0].Message.Content
|
||||
contentStr := strings.TrimSpace(apiResp.Choices[0].Message.Content)
|
||||
|
||||
// 7. Parse the nested JSON from the content field into ReceiptData.
|
||||
// Strip markdown code fences if the model wrapped the JSON.
|
||||
contentStr = stripMarkdownFences(contentStr)
|
||||
|
||||
// 8. Parse the JSON content into ReceiptData.
|
||||
var receipt ReceiptData
|
||||
if err := json.Unmarshal([]byte(contentStr), &receipt); err != nil {
|
||||
log.Printf("ExtractReceipt: failed to parse receipt JSON from content: %v", err)
|
||||
log.Printf("ExtractReceipt: raw content was: %s", contentStr)
|
||||
return &ReceiptData{}, err
|
||||
}
|
||||
|
||||
log.Printf("ExtractReceipt: successfully extracted receipt data: merchant=%q amount=%.2f %s",
|
||||
receipt.Merchant, receipt.Amount, receipt.Currency)
|
||||
return &receipt, nil
|
||||
}
|
||||
|
||||
// readImageFile reads the full contents of an image file after verifying it
|
||||
// exists and is a regular file.
|
||||
// compressImage decodes the image, downscales it (preserving aspect ratio)
|
||||
// to fit within maxImageSize, and re-encodes as JPEG with the configured quality.
|
||||
func compressImage(data []byte) ([]byte, error) {
|
||||
// Detect format and decode.
|
||||
img, format, err := image.Decode(bytes.NewReader(data))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("image decode: %w", err)
|
||||
}
|
||||
|
||||
bounds := img.Bounds()
|
||||
w := bounds.Dx()
|
||||
h := bounds.Dy()
|
||||
|
||||
// If the image is already small, just re-encode at reduced quality.
|
||||
if w <= maxImageSize && h <= maxImageSize {
|
||||
var buf bytes.Buffer
|
||||
if err := jpeg.Encode(&buf, img, &jpeg.Options{Quality: jpegQuality}); err != nil {
|
||||
return nil, fmt.Errorf("jpeg re-encode: %w", err)
|
||||
}
|
||||
// Only use if smaller than original.
|
||||
if buf.Len() < len(data) {
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
// Calculate new dimensions.
|
||||
ratio := float64(maxImageSize) / float64(max(w, h))
|
||||
newW := int(float64(w) * ratio)
|
||||
newH := int(float64(h) * ratio)
|
||||
if newW < 1 {
|
||||
newW = 1
|
||||
}
|
||||
if newH < 1 {
|
||||
newH = 1
|
||||
}
|
||||
|
||||
// Downscale using simple bilinear-like averaging (via RGBA iteration).
|
||||
scaled := scaleImage(img, newW, newH)
|
||||
|
||||
var buf bytes.Buffer
|
||||
if err := jpeg.Encode(&buf, scaled, &jpeg.Options{Quality: jpegQuality}); err != nil {
|
||||
return nil, fmt.Errorf("jpeg encode: %w", err)
|
||||
}
|
||||
|
||||
log.Printf("ExtractReceipt: compressed image %dx%d %s -> %dx%d JPEG (%d bytes)",
|
||||
w, h, format, newW, newH, buf.Len())
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
|
||||
// scaleImage performs a simple nearest-neighbour downscale of an image.
|
||||
func scaleImage(src image.Image, newW, newH int) image.Image {
|
||||
bounds := src.Bounds()
|
||||
srcW := bounds.Dx()
|
||||
srcH := bounds.Dy()
|
||||
dst := image.NewRGBA(image.Rect(0, 0, newW, newH))
|
||||
|
||||
for y := 0; y < newH; y++ {
|
||||
for x := 0; x < newW; x++ {
|
||||
sx := x * srcW / newW
|
||||
sy := y * srcH / newH
|
||||
dst.Set(x, y, src.At(sx, sy))
|
||||
}
|
||||
}
|
||||
return dst
|
||||
}
|
||||
|
||||
// stripMarkdownFences removes markdown code fences (```json ... ```) from the
|
||||
// model output, if present.
|
||||
func stripMarkdownFences(s string) string {
|
||||
s = strings.TrimSpace(s)
|
||||
if strings.HasPrefix(s, "```") {
|
||||
s = s[3:]
|
||||
if idx := strings.Index(s, "\n"); idx != -1 {
|
||||
s = s[idx+1:]
|
||||
}
|
||||
}
|
||||
if strings.HasSuffix(s, "```") {
|
||||
s = s[:len(s)-3]
|
||||
}
|
||||
return strings.TrimSpace(s)
|
||||
}
|
||||
|
||||
// readImageFile reads the full contents of an image file from disk.
|
||||
func readImageFile(path string) ([]byte, error) {
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
|
|
@ -179,13 +272,16 @@ func readImageFile(path string) ([]byte, error) {
|
|||
}
|
||||
|
||||
if info.IsDir() {
|
||||
return nil, fmt.Errorf("path is a directory, not an image file: %s", path)
|
||||
return nil, fmt.Errorf("path is a directory, not an image: %s", path)
|
||||
}
|
||||
|
||||
if info.Size() > 10<<20 {
|
||||
return nil, fmt.Errorf("image file too large: %d bytes (max 10MB)", info.Size())
|
||||
}
|
||||
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot read image file %s: %w", path, err)
|
||||
}
|
||||
|
||||
return data, nil
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue