fix: AI receipt extraction now uses OCR (Tesseract) + LLM (DeepSeek) pipeline
- Problem: deepseek-v4-flash is text-only, cannot process base64 images - hallucinated fake data - Solution: Two-step pipeline that actually extracts real data: 1. Tesseract OCR extracts raw text from the receipt image 2. DeepSeek v4 parses the OCR text into structured JSON - Benefits: works with any image format, fast, accurate, no hallucinated data - Properly handles HEIC/HEIF via heif-convert before OCR
This commit is contained in:
parent
ebe06082cf
commit
fecf4af8a2
1 changed files with 136 additions and 228 deletions
|
|
@ -2,12 +2,10 @@ package ai
|
|||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"image"
|
||||
"image/jpeg"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
|
|
@ -30,9 +28,7 @@ type ReceiptData struct {
|
|||
const (
|
||||
deepseekAPIURL = "https://api.deepseek.com/v1/chat/completions"
|
||||
deepseekModel = "deepseek-v4-flash"
|
||||
requestTimeout = 120 * time.Second
|
||||
maxImageSize = 300 // max dimension in pixels (width or height)
|
||||
jpegQuality = 50 // JPEG compression quality (1-100)
|
||||
requestTimeout = 60 * time.Second
|
||||
)
|
||||
|
||||
// deepseekRequest matches the DeepSeek API request format.
|
||||
|
|
@ -60,73 +56,161 @@ type deepseekResponseMessage struct {
|
|||
Content string `json:"content"`
|
||||
}
|
||||
|
||||
// ExtractReceipt sends a receipt image to the DeepSeek API and parses
|
||||
// the structured receipt data from the response.
|
||||
// ExtractReceipt extracts structured receipt data from an image using a two-step
|
||||
// pipeline: OCR (Tesseract) → LLM parsing (DeepSeek).
|
||||
//
|
||||
// Since DeepSeek's chat API does not support the OpenAI-style image_url
|
||||
// content type, the image is resized (max 800px) and JPEG-compressed,
|
||||
// then base64-encoded and embedded directly in the text prompt.
|
||||
// Step 1 — OCR: Run Tesseract on the receipt image to extract raw text.
|
||||
// Step 2 — LLM: Send the OCR text to DeepSeek to parse into structured JSON.
|
||||
//
|
||||
// This works with any image format (JPEG, PNG, HEIC, etc.) and produces
|
||||
// real extracted data rather than AI-hallucinated values.
|
||||
func ExtractReceipt(imagePath string) (*ReceiptData, error) {
|
||||
// 1. Validate the image file exists and is readable.
|
||||
imageData, err := readImageFile(imagePath)
|
||||
if err != nil {
|
||||
return &ReceiptData{}, err
|
||||
}
|
||||
|
||||
// 2. Get the API key from the environment.
|
||||
apiKey := os.Getenv("DEEPSEEK_API_KEY")
|
||||
if apiKey == "" {
|
||||
err := errors.New("DEEPSEEK_API_KEY environment variable is not set")
|
||||
log.Printf("ExtractReceipt: %v", err)
|
||||
return &ReceiptData{}, errors.New("DEEPSEEK_API_KEY environment variable is not set")
|
||||
}
|
||||
|
||||
// Step 1: OCR — extract text from the receipt image.
|
||||
ocrText, err := ocrImage(imagePath)
|
||||
if err != nil {
|
||||
log.Printf("ExtractReceipt: OCR failed: %v", err)
|
||||
return &ReceiptData{}, fmt.Errorf("OCR failed: %w", err)
|
||||
}
|
||||
|
||||
ocrText = strings.TrimSpace(ocrText)
|
||||
if ocrText == "" {
|
||||
return &ReceiptData{}, errors.New("OCR returned no text from the receipt image")
|
||||
}
|
||||
|
||||
log.Printf("ExtractReceipt: OCR extracted %d characters of text", len(ocrText))
|
||||
|
||||
// Step 2: LLM — send OCR text to DeepSeek for structured parsing.
|
||||
receipt, err := parseReceiptText(ocrText, apiKey)
|
||||
if err != nil {
|
||||
log.Printf("ExtractReceipt: LLM parsing failed: %v", err)
|
||||
return &ReceiptData{}, err
|
||||
}
|
||||
|
||||
// 3. Decode, downscale, and re-encode the image to keep token cost manageable.
|
||||
compressed, err := compressImage(imageData)
|
||||
if err != nil {
|
||||
log.Printf("ExtractReceipt: image compression failed: %v", err)
|
||||
// Fall back to raw image if compression fails.
|
||||
compressed = imageData
|
||||
log.Printf("ExtractReceipt: parsed receipt — merchant=%q amount=%.2f %s category=%q date=%q",
|
||||
receipt.Merchant, receipt.Amount, receipt.Currency, receipt.Category, receipt.Date)
|
||||
return receipt, nil
|
||||
}
|
||||
|
||||
// 4. Base64-encode the compressed image.
|
||||
b64 := base64.StdEncoding.EncodeToString(compressed)
|
||||
// ocrImage runs Tesseract OCR on the image file and returns the extracted text.
|
||||
func ocrImage(imagePath string) (string, error) {
|
||||
// First, try to convert the image to a format Tesseract handles well.
|
||||
// For HEIC files, this also handles the conversion.
|
||||
jpegPath, err := ensureJPEG(imagePath)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("image preparation: %w", err)
|
||||
}
|
||||
if jpegPath != imagePath {
|
||||
defer os.Remove(jpegPath)
|
||||
}
|
||||
|
||||
// 5. Build the text prompt with the base64 image data inline.
|
||||
prompt := fmt.Sprintf(
|
||||
`Analyze this receipt image (base64-encoded JPEG/PNG below). Extract the following fields as a strict JSON object with these exact keys:
|
||||
- "amount": float (the total amount paid, e.g. 42.50)
|
||||
- "currency": string (3-letter ISO currency code, e.g. EUR, USD, GBP)
|
||||
- "merchant": string (the store or business name)
|
||||
- "category": string (one of: Food, Travel, Lodging, Software, Other)
|
||||
- "date": string (the receipt date in YYYY-MM-DD format)
|
||||
cmd := exec.Command("tesseract", jpegPath, "stdout", "--psm", "6")
|
||||
var stdout, stderr bytes.Buffer
|
||||
cmd.Stdout = &stdout
|
||||
cmd.Stderr = &stderr
|
||||
|
||||
Return ONLY valid JSON. No markdown, no explanation, no code fences.
|
||||
if err := cmd.Run(); err != nil {
|
||||
return "", fmt.Errorf("tesseract failed: %w, stderr: %s", err, strings.TrimSpace(stderr.String()))
|
||||
}
|
||||
|
||||
Image data: %s`, b64)
|
||||
return stdout.String(), nil
|
||||
}
|
||||
|
||||
// ensureJPEG converts the image at the given path to JPEG if it isn't already.
|
||||
// Returns the path to a JPEG file (may be the original if already JPEG-compatible).
|
||||
// The caller should remove the returned path if it differs from the input path.
|
||||
func ensureJPEG(path string) (string, error) {
|
||||
// Read the file to check format.
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// Try to decode with Go's stdlib — if it works, we can re-encode as JPEG.
|
||||
// Tesseract can handle PNG/BMP/TIFF natively, so only convert if Go
|
||||
// cannot decode the format (e.g., HEIC).
|
||||
_, _, err = image.Decode(bytes.NewReader(data))
|
||||
if err == nil {
|
||||
// Image is in a format Go understands — Tesseract will handle it.
|
||||
// No conversion needed unless it's a very large image.
|
||||
return path, nil
|
||||
}
|
||||
|
||||
// Go couldn't decode it (likely HEIC). Try external conversion.
|
||||
log.Printf("ensureJPEG: Go cannot decode %s, trying external conversion", path)
|
||||
outPath := path + ".ocr-convert.jpg"
|
||||
if err := convertToJPEG(data, outPath); err != nil {
|
||||
return "", fmt.Errorf("external conversion failed: %w", err)
|
||||
}
|
||||
|
||||
return outPath, nil
|
||||
}
|
||||
|
||||
// convertToJPEG tries multiple tools to convert image data to JPEG.
|
||||
func convertToJPEG(data []byte, outPath string) error {
|
||||
tmpDir := os.TempDir()
|
||||
inPath := filepath.Join(tmpDir, "ef-convert-"+fmt.Sprintf("%d", time.Now().UnixNano()))
|
||||
if err := os.WriteFile(inPath, data, 0644); err != nil {
|
||||
return err
|
||||
}
|
||||
defer os.Remove(inPath)
|
||||
|
||||
// Try heif-convert first.
|
||||
if err := exec.Command("heif-convert", inPath, outPath).Run(); err == nil {
|
||||
if _, err := os.Stat(outPath); err == nil {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// Fall back to ImageMagick.
|
||||
if err := exec.Command("convert", inPath, outPath).Run(); err == nil {
|
||||
if _, err := os.Stat(outPath); err == nil {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
return errors.New("all image conversion tools failed (tried heif-convert, convert)")
|
||||
}
|
||||
|
||||
// parseReceiptText sends raw OCR text to DeepSeek and returns structured data.
|
||||
func parseReceiptText(ocrText, apiKey string) (*ReceiptData, error) {
|
||||
prompt := fmt.Sprintf(`Extract receipt information from the following OCR text.
|
||||
|
||||
Return ONLY a valid JSON object (no markdown, no explanation) with these exact keys:
|
||||
- "merchant": the store or business name (string)
|
||||
- "amount": the total amount paid (number)
|
||||
- "currency": 3-letter currency code, e.g. KES, USD, EUR (string)
|
||||
- "category": one of: Food, Travel, Lodging, Software, Other (string)
|
||||
- "date": the receipt date in YYYY-MM-DD format (string)
|
||||
|
||||
If a field is not found in the text, use null for the value.
|
||||
Do NOT make up or invent data that is not present in the OCR text.
|
||||
|
||||
OCR text from receipt:
|
||||
---
|
||||
%s
|
||||
---`, ocrText)
|
||||
|
||||
payload := deepseekRequest{
|
||||
Model: deepseekModel,
|
||||
Temperature: 0.1,
|
||||
Messages: []deepseekMessage{
|
||||
{
|
||||
Role: "user",
|
||||
Content: prompt,
|
||||
},
|
||||
{Role: "user", Content: prompt},
|
||||
},
|
||||
}
|
||||
|
||||
body, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
log.Printf("ExtractReceipt: failed to marshal request: %v", err)
|
||||
return &ReceiptData{}, err
|
||||
return nil, fmt.Errorf("marshal request: %w", err)
|
||||
}
|
||||
|
||||
// 6. Send the POST request.
|
||||
req, err := http.NewRequest(http.MethodPost, deepseekAPIURL, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
log.Printf("ExtractReceipt: failed to create request: %v", err)
|
||||
return &ReceiptData{}, err
|
||||
return nil, fmt.Errorf("create request: %w", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Authorization", "Bearer "+apiKey)
|
||||
|
|
@ -134,134 +218,40 @@ Image data: %s`, b64)
|
|||
client := &http.Client{Timeout: requestTimeout}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
log.Printf("ExtractReceipt: API request failed: %v", err)
|
||||
return &ReceiptData{}, err
|
||||
return nil, fmt.Errorf("API request failed: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
respBody, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
log.Printf("ExtractReceipt: failed to read response: %v", err)
|
||||
return &ReceiptData{}, err
|
||||
return nil, fmt.Errorf("read response: %w", err)
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
err := fmt.Errorf("API returned status %d: %s", resp.StatusCode, string(respBody))
|
||||
log.Printf("ExtractReceipt: %v", err)
|
||||
return &ReceiptData{}, err
|
||||
return nil, fmt.Errorf("API returned status %d: %s", resp.StatusCode, string(respBody))
|
||||
}
|
||||
|
||||
// 7. Parse the DeepSeek response.
|
||||
var apiResp deepseekResponse
|
||||
if err := json.Unmarshal(respBody, &apiResp); err != nil {
|
||||
log.Printf("ExtractReceipt: failed to parse response JSON: %v", err)
|
||||
return &ReceiptData{}, err
|
||||
return nil, fmt.Errorf("parse response: %w", err)
|
||||
}
|
||||
|
||||
if len(apiResp.Choices) == 0 {
|
||||
return &ReceiptData{}, errors.New("API response contains no choices")
|
||||
return nil, errors.New("API response contains no choices")
|
||||
}
|
||||
|
||||
contentStr := strings.TrimSpace(apiResp.Choices[0].Message.Content)
|
||||
|
||||
// Strip markdown code fences if the model wrapped the JSON.
|
||||
contentStr = stripMarkdownFences(contentStr)
|
||||
|
||||
// 8. Parse the JSON content into ReceiptData.
|
||||
var receipt ReceiptData
|
||||
if err := json.Unmarshal([]byte(contentStr), &receipt); err != nil {
|
||||
log.Printf("ExtractReceipt: failed to parse receipt JSON from content: %v", err)
|
||||
log.Printf("ExtractReceipt: raw content was: %s", contentStr)
|
||||
return &ReceiptData{}, err
|
||||
return nil, fmt.Errorf("parse receipt JSON: %w (content: %s)", err, contentStr)
|
||||
}
|
||||
|
||||
log.Printf("ExtractReceipt: successfully extracted receipt data: merchant=%q amount=%.2f %s",
|
||||
receipt.Merchant, receipt.Amount, receipt.Currency)
|
||||
return &receipt, nil
|
||||
}
|
||||
|
||||
// compressImage decodes the image, downscales it (preserving aspect ratio)
|
||||
// to fit within maxImageSize, and re-encodes as JPEG with the configured quality.
|
||||
// If Go's built-in decoders cannot handle the format (e.g. HEIC from iPhones),
|
||||
// it attempts to convert via external tools (heif-convert or ImageMagick).
|
||||
func compressImage(data []byte) ([]byte, error) {
|
||||
// Try Go's built-in image decoders first (covers JPEG, PNG, GIF, BMP, TIFF, WebP).
|
||||
img, format, err := image.Decode(bytes.NewReader(data))
|
||||
if err != nil {
|
||||
// Built-in decoder failed — try external tools for HEIC and other formats.
|
||||
log.Printf("compressImage: Go decoder failed (%v), trying external conversion", err)
|
||||
converted, err := convertWithExternalTool(data)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("image decode and external conversion both failed: %w", err)
|
||||
}
|
||||
// Try decoding the converted data.
|
||||
img, format, err = image.Decode(bytes.NewReader(converted))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("decoding converted image also failed: %w", err)
|
||||
}
|
||||
data = converted
|
||||
}
|
||||
|
||||
bounds := img.Bounds()
|
||||
w := bounds.Dx()
|
||||
h := bounds.Dy()
|
||||
|
||||
// If the image is already small, just re-encode at reduced quality.
|
||||
if w <= maxImageSize && h <= maxImageSize {
|
||||
var buf bytes.Buffer
|
||||
if err := jpeg.Encode(&buf, img, &jpeg.Options{Quality: jpegQuality}); err != nil {
|
||||
return nil, fmt.Errorf("jpeg re-encode: %w", err)
|
||||
}
|
||||
// Only use if smaller than original.
|
||||
if buf.Len() < len(data) {
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
// Calculate new dimensions.
|
||||
ratio := float64(maxImageSize) / float64(max(w, h))
|
||||
newW := int(float64(w) * ratio)
|
||||
newH := int(float64(h) * ratio)
|
||||
if newW < 1 {
|
||||
newW = 1
|
||||
}
|
||||
if newH < 1 {
|
||||
newH = 1
|
||||
}
|
||||
|
||||
// Downscale using simple bilinear-like averaging (via RGBA iteration).
|
||||
scaled := scaleImage(img, newW, newH)
|
||||
|
||||
var buf bytes.Buffer
|
||||
if err := jpeg.Encode(&buf, scaled, &jpeg.Options{Quality: jpegQuality}); err != nil {
|
||||
return nil, fmt.Errorf("jpeg encode: %w", err)
|
||||
}
|
||||
|
||||
log.Printf("ExtractReceipt: compressed image %dx%d %s -> %dx%d JPEG (%d bytes)",
|
||||
w, h, format, newW, newH, buf.Len())
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
|
||||
// scaleImage performs a simple nearest-neighbour downscale of an image.
|
||||
func scaleImage(src image.Image, newW, newH int) image.Image {
|
||||
bounds := src.Bounds()
|
||||
srcW := bounds.Dx()
|
||||
srcH := bounds.Dy()
|
||||
dst := image.NewRGBA(image.Rect(0, 0, newW, newH))
|
||||
|
||||
for y := 0; y < newH; y++ {
|
||||
for x := 0; x < newW; x++ {
|
||||
sx := x * srcW / newW
|
||||
sy := y * srcH / newH
|
||||
dst.Set(x, y, src.At(sx, sy))
|
||||
}
|
||||
}
|
||||
return dst
|
||||
}
|
||||
|
||||
// stripMarkdownFences removes markdown code fences (```json ... ```) from the
|
||||
// model output, if present.
|
||||
// stripMarkdownFences removes markdown code fences from model output.
|
||||
func stripMarkdownFences(s string) string {
|
||||
s = strings.TrimSpace(s)
|
||||
if strings.HasPrefix(s, "```") {
|
||||
|
|
@ -275,85 +265,3 @@ func stripMarkdownFences(s string) string {
|
|||
}
|
||||
return strings.TrimSpace(s)
|
||||
}
|
||||
|
||||
// convertWithExternalTool tries to convert an unsupported image format (e.g. HEIC)
|
||||
// to JPEG using heif-convert or ImageMagick's convert command.
|
||||
func convertWithExternalTool(data []byte) ([]byte, error) {
|
||||
// Write the unknown image to a temp file (needed for CLI tools).
|
||||
tmpDir := os.TempDir()
|
||||
inPath := filepath.Join(tmpDir, "ef-convert-in-"+fmt.Sprintf("%d", time.Now().UnixNano()))
|
||||
outPath := filepath.Join(tmpDir, "ef-convert-out-"+fmt.Sprintf("%d", time.Now().UnixNano())+".jpg")
|
||||
|
||||
if err := os.WriteFile(inPath, data, 0644); err != nil {
|
||||
return nil, fmt.Errorf("write temp input: %w", err)
|
||||
}
|
||||
defer os.Remove(inPath)
|
||||
defer os.Remove(outPath)
|
||||
|
||||
// Try heif-convert first (fastest, handles HEIC natively).
|
||||
if heifErr := tryHEIFConvert(inPath, outPath); heifErr == nil {
|
||||
outData, err := os.ReadFile(outPath)
|
||||
if err == nil && len(outData) > 0 {
|
||||
log.Printf("convertWithExternalTool: heif-convert succeeded (%d bytes)", len(outData))
|
||||
return outData, nil
|
||||
}
|
||||
}
|
||||
|
||||
// Fall back to ImageMagick convert.
|
||||
if magickErr := exec.Command("convert", inPath, "-resize", fmt.Sprintf("%dx%d>", maxImageSize, maxImageSize), "-quality", fmt.Sprintf("%d", jpegQuality), outPath).Run(); magickErr == nil {
|
||||
outData, err := os.ReadFile(outPath)
|
||||
if err == nil && len(outData) > 0 {
|
||||
log.Printf("convertWithExternalTool: ImageMagick convert succeeded (%d bytes)", len(outData))
|
||||
return outData, nil
|
||||
}
|
||||
}
|
||||
|
||||
return nil, errors.New("all external conversion tools failed")
|
||||
}
|
||||
|
||||
// tryHEIFConvert attempts to convert a HEIC/HEIF file to JPEG using heif-convert.
|
||||
func tryHEIFConvert(inPath, outPath string) error {
|
||||
// heif-convert outputs to {input}.jpg by default.
|
||||
defaultOut := inPath + ".jpg"
|
||||
defer os.Remove(defaultOut)
|
||||
|
||||
cmd := exec.Command("heif-convert", inPath, outPath)
|
||||
if output, err := cmd.CombinedOutput(); err != nil {
|
||||
return fmt.Errorf("heif-convert failed: %w, output: %s", err, string(output))
|
||||
}
|
||||
|
||||
// If it wrote to the default path instead of our outPath, move it.
|
||||
if _, err := os.Stat(outPath); os.IsNotExist(err) {
|
||||
if _, err := os.Stat(defaultOut); err == nil {
|
||||
os.Rename(defaultOut, outPath)
|
||||
}
|
||||
}
|
||||
|
||||
_, err := os.Stat(outPath)
|
||||
return err
|
||||
}
|
||||
|
||||
// readImageFile reads the full contents of an image file from disk.
|
||||
func readImageFile(path string) ([]byte, error) {
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil, fmt.Errorf("image file does not exist: %s", path)
|
||||
}
|
||||
return nil, fmt.Errorf("cannot stat image file %s: %w", path, err)
|
||||
}
|
||||
|
||||
if info.IsDir() {
|
||||
return nil, fmt.Errorf("path is a directory, not an image: %s", path)
|
||||
}
|
||||
|
||||
if info.Size() > 10<<20 {
|
||||
return nil, fmt.Errorf("image file too large: %d bytes (max 10MB)", info.Size())
|
||||
}
|
||||
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot read image file %s: %w", path, err)
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue