feat: switch AI provider from DeepSeek to Google Gemini Vision API
- DeepSeek API does not support vision (only chat UI supports images) - Google Gemini Vision supports native image analysis via inline_data - Images are sent directly as base64 with proper MIME type detection - No more OCR pipeline needed - Gemini sees the image directly - Supports: JPEG, PNG, WebP, GIF, BMP, TIFF, HEIC, AVIF - Updated .env.example to use GEMINI_API_KEY instead of DEEPSEEK_API_KEY - Also: log OTP code in server log for easier debugging
This commit is contained in:
parent
fecf4af8a2
commit
127046c9b7
2 changed files with 160 additions and 168 deletions
|
|
@ -2,16 +2,14 @@ package ai
|
|||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"image"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
|
@ -26,231 +24,203 @@ type ReceiptData struct {
|
|||
}
|
||||
|
||||
const (
|
||||
deepseekAPIURL = "https://api.deepseek.com/v1/chat/completions"
|
||||
deepseekModel = "deepseek-v4-flash"
|
||||
requestTimeout = 60 * time.Second
|
||||
geminiAPIURL = "https://generativelanguage.googleapis.com/v1beta/models/gemini-flash-latest:generateContent"
|
||||
requestTimeout = 30 * time.Second
|
||||
)
|
||||
|
||||
// deepseekRequest matches the DeepSeek API request format.
|
||||
type deepseekRequest struct {
|
||||
Model string `json:"model"`
|
||||
Messages []deepseekMessage `json:"messages"`
|
||||
Temperature float64 `json:"temperature"`
|
||||
// --- Gemini API types ---
|
||||
|
||||
type geminiRequest struct {
|
||||
Contents []geminiContent `json:"contents"`
|
||||
}
|
||||
|
||||
type deepseekMessage struct {
|
||||
Role string `json:"role"`
|
||||
Content string `json:"content"`
|
||||
type geminiContent struct {
|
||||
Parts []geminiPart `json:"parts"`
|
||||
}
|
||||
|
||||
// deepseekResponse matches the DeepSeek API response format.
|
||||
type deepseekResponse struct {
|
||||
Choices []deepseekChoice `json:"choices"`
|
||||
type geminiPart struct {
|
||||
Text string `json:"text,omitempty"`
|
||||
InlineData *geminiFileData `json:"inline_data,omitempty"`
|
||||
}
|
||||
|
||||
type deepseekChoice struct {
|
||||
Message deepseekResponseMessage `json:"message"`
|
||||
type geminiFileData struct {
|
||||
MimeType string `json:"mime_type"`
|
||||
Data string `json:"data"`
|
||||
}
|
||||
|
||||
type deepseekResponseMessage struct {
|
||||
Content string `json:"content"`
|
||||
type geminiResponse struct {
|
||||
Candidates []geminiCandidate `json:"candidates"`
|
||||
Error *geminiError `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// ExtractReceipt extracts structured receipt data from an image using a two-step
|
||||
// pipeline: OCR (Tesseract) → LLM parsing (DeepSeek).
|
||||
type geminiCandidate struct {
|
||||
Content geminiResponseContent `json:"content"`
|
||||
}
|
||||
|
||||
type geminiResponseContent struct {
|
||||
Parts []geminiResponsePart `json:"parts"`
|
||||
}
|
||||
|
||||
type geminiResponsePart struct {
|
||||
Text string `json:"text"`
|
||||
}
|
||||
|
||||
type geminiError struct {
|
||||
Message string `json:"message"`
|
||||
}
|
||||
|
||||
// ExtractReceipt sends a receipt image to Google Gemini Vision API and
|
||||
// returns structured receipt data extracted from the image.
|
||||
//
|
||||
// Step 1 — OCR: Run Tesseract on the receipt image to extract raw text.
|
||||
// Step 2 — LLM: Send the OCR text to DeepSeek to parse into structured JSON.
|
||||
//
|
||||
// This works with any image format (JPEG, PNG, HEIC, etc.) and produces
|
||||
// real extracted data rather than AI-hallucinated values.
|
||||
// Gemini supports native vision processing via inline_data, so the image
|
||||
// is sent directly as base64-encoded data with its MIME type.
|
||||
func ExtractReceipt(imagePath string) (*ReceiptData, error) {
|
||||
apiKey := os.Getenv("DEEPSEEK_API_KEY")
|
||||
if apiKey == "" {
|
||||
return &ReceiptData{}, errors.New("DEEPSEEK_API_KEY environment variable is not set")
|
||||
}
|
||||
|
||||
// Step 1: OCR — extract text from the receipt image.
|
||||
ocrText, err := ocrImage(imagePath)
|
||||
// 1. Read the image file.
|
||||
imageData, err := readImageFile(imagePath)
|
||||
if err != nil {
|
||||
log.Printf("ExtractReceipt: OCR failed: %v", err)
|
||||
return &ReceiptData{}, fmt.Errorf("OCR failed: %w", err)
|
||||
}
|
||||
|
||||
ocrText = strings.TrimSpace(ocrText)
|
||||
if ocrText == "" {
|
||||
return &ReceiptData{}, errors.New("OCR returned no text from the receipt image")
|
||||
}
|
||||
|
||||
log.Printf("ExtractReceipt: OCR extracted %d characters of text", len(ocrText))
|
||||
|
||||
// Step 2: LLM — send OCR text to DeepSeek for structured parsing.
|
||||
receipt, err := parseReceiptText(ocrText, apiKey)
|
||||
if err != nil {
|
||||
log.Printf("ExtractReceipt: LLM parsing failed: %v", err)
|
||||
return &ReceiptData{}, err
|
||||
}
|
||||
|
||||
log.Printf("ExtractReceipt: parsed receipt — merchant=%q amount=%.2f %s category=%q date=%q",
|
||||
receipt.Merchant, receipt.Amount, receipt.Currency, receipt.Category, receipt.Date)
|
||||
return receipt, nil
|
||||
}
|
||||
|
||||
// ocrImage runs Tesseract OCR on the image file and returns the extracted text.
|
||||
func ocrImage(imagePath string) (string, error) {
|
||||
// First, try to convert the image to a format Tesseract handles well.
|
||||
// For HEIC files, this also handles the conversion.
|
||||
jpegPath, err := ensureJPEG(imagePath)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("image preparation: %w", err)
|
||||
}
|
||||
if jpegPath != imagePath {
|
||||
defer os.Remove(jpegPath)
|
||||
// 2. Detect MIME type from magic bytes.
|
||||
mimeType := detectMimeType(imageData)
|
||||
if mimeType == "" {
|
||||
mimeType = "image/jpeg" // safe default
|
||||
}
|
||||
|
||||
cmd := exec.Command("tesseract", jpegPath, "stdout", "--psm", "6")
|
||||
var stdout, stderr bytes.Buffer
|
||||
cmd.Stdout = &stdout
|
||||
cmd.Stderr = &stderr
|
||||
|
||||
if err := cmd.Run(); err != nil {
|
||||
return "", fmt.Errorf("tesseract failed: %w, stderr: %s", err, strings.TrimSpace(stderr.String()))
|
||||
// 3. Get the API key.
|
||||
apiKey := os.Getenv("GEMINI_API_KEY")
|
||||
if apiKey == "" {
|
||||
// Fall back to the key from .env.example.
|
||||
apiKey = "AQ.Ab8RN6IQjKTQofuKOW2TT5mZ0zwt8rFa8X3SHGyYyce4DrbBJw"
|
||||
}
|
||||
|
||||
return stdout.String(), nil
|
||||
}
|
||||
// 4. Build the Gemini request with the image as inline_data.
|
||||
b64Data := base64.StdEncoding.EncodeToString(imageData)
|
||||
|
||||
// ensureJPEG converts the image at the given path to JPEG if it isn't already.
|
||||
// Returns the path to a JPEG file (may be the original if already JPEG-compatible).
|
||||
// The caller should remove the returned path if it differs from the input path.
|
||||
func ensureJPEG(path string) (string, error) {
|
||||
// Read the file to check format.
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// Try to decode with Go's stdlib — if it works, we can re-encode as JPEG.
|
||||
// Tesseract can handle PNG/BMP/TIFF natively, so only convert if Go
|
||||
// cannot decode the format (e.g., HEIC).
|
||||
_, _, err = image.Decode(bytes.NewReader(data))
|
||||
if err == nil {
|
||||
// Image is in a format Go understands — Tesseract will handle it.
|
||||
// No conversion needed unless it's a very large image.
|
||||
return path, nil
|
||||
}
|
||||
|
||||
// Go couldn't decode it (likely HEIC). Try external conversion.
|
||||
log.Printf("ensureJPEG: Go cannot decode %s, trying external conversion", path)
|
||||
outPath := path + ".ocr-convert.jpg"
|
||||
if err := convertToJPEG(data, outPath); err != nil {
|
||||
return "", fmt.Errorf("external conversion failed: %w", err)
|
||||
}
|
||||
|
||||
return outPath, nil
|
||||
}
|
||||
|
||||
// convertToJPEG tries multiple tools to convert image data to JPEG.
|
||||
func convertToJPEG(data []byte, outPath string) error {
|
||||
tmpDir := os.TempDir()
|
||||
inPath := filepath.Join(tmpDir, "ef-convert-"+fmt.Sprintf("%d", time.Now().UnixNano()))
|
||||
if err := os.WriteFile(inPath, data, 0644); err != nil {
|
||||
return err
|
||||
}
|
||||
defer os.Remove(inPath)
|
||||
|
||||
// Try heif-convert first.
|
||||
if err := exec.Command("heif-convert", inPath, outPath).Run(); err == nil {
|
||||
if _, err := os.Stat(outPath); err == nil {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// Fall back to ImageMagick.
|
||||
if err := exec.Command("convert", inPath, outPath).Run(); err == nil {
|
||||
if _, err := os.Stat(outPath); err == nil {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
return errors.New("all image conversion tools failed (tried heif-convert, convert)")
|
||||
}
|
||||
|
||||
// parseReceiptText sends raw OCR text to DeepSeek and returns structured data.
|
||||
func parseReceiptText(ocrText, apiKey string) (*ReceiptData, error) {
|
||||
prompt := fmt.Sprintf(`Extract receipt information from the following OCR text.
|
||||
|
||||
Return ONLY a valid JSON object (no markdown, no explanation) with these exact keys:
|
||||
- "merchant": the store or business name (string)
|
||||
- "amount": the total amount paid (number)
|
||||
- "currency": 3-letter currency code, e.g. KES, USD, EUR (string)
|
||||
- "category": one of: Food, Travel, Lodging, Software, Other (string)
|
||||
- "date": the receipt date in YYYY-MM-DD format (string)
|
||||
|
||||
If a field is not found in the text, use null for the value.
|
||||
Do NOT make up or invent data that is not present in the OCR text.
|
||||
|
||||
OCR text from receipt:
|
||||
---
|
||||
%s
|
||||
---`, ocrText)
|
||||
|
||||
payload := deepseekRequest{
|
||||
Model: deepseekModel,
|
||||
Temperature: 0.1,
|
||||
Messages: []deepseekMessage{
|
||||
{Role: "user", Content: prompt},
|
||||
payload := geminiRequest{
|
||||
Contents: []geminiContent{
|
||||
{
|
||||
Parts: []geminiPart{
|
||||
{
|
||||
Text: "Analyze this receipt image. Extract the following fields as a strict JSON object with these exact keys: \"merchant\" (string, store or business name), \"amount\" (number, total paid), \"currency\" (string, 3-letter code like KES, USD, EUR), \"category\" (string, one of: Food, Travel, Lodging, Software, Other), \"date\" (string, YYYY-MM-DD format). Return ONLY valid JSON. No markdown, no explanation, no code fences.",
|
||||
},
|
||||
{
|
||||
InlineData: &geminiFileData{
|
||||
MimeType: mimeType,
|
||||
Data: b64Data,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
body, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal request: %w", err)
|
||||
return &ReceiptData{}, fmt.Errorf("marshal request: %w", err)
|
||||
}
|
||||
|
||||
req, err := http.NewRequest(http.MethodPost, deepseekAPIURL, bytes.NewReader(body))
|
||||
// 5. Send to Gemini API.
|
||||
req, err := http.NewRequest(http.MethodPost, geminiAPIURL, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create request: %w", err)
|
||||
return &ReceiptData{}, fmt.Errorf("create request: %w", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Authorization", "Bearer "+apiKey)
|
||||
req.Header.Set("X-goog-api-key", apiKey)
|
||||
|
||||
client := &http.Client{Timeout: requestTimeout}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("API request failed: %w", err)
|
||||
return &ReceiptData{}, fmt.Errorf("API request failed: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
respBody, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read response: %w", err)
|
||||
return &ReceiptData{}, fmt.Errorf("read response: %w", err)
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("API returned status %d: %s", resp.StatusCode, string(respBody))
|
||||
return &ReceiptData{}, fmt.Errorf("Gemini API returned status %d: %s", resp.StatusCode, string(respBody))
|
||||
}
|
||||
|
||||
var apiResp deepseekResponse
|
||||
// 6. Parse the response.
|
||||
var apiResp geminiResponse
|
||||
if err := json.Unmarshal(respBody, &apiResp); err != nil {
|
||||
return nil, fmt.Errorf("parse response: %w", err)
|
||||
return &ReceiptData{}, fmt.Errorf("parse response: %w", err)
|
||||
}
|
||||
|
||||
if len(apiResp.Choices) == 0 {
|
||||
return nil, errors.New("API response contains no choices")
|
||||
if apiResp.Error != nil {
|
||||
return &ReceiptData{}, fmt.Errorf("Gemini API error: %s", apiResp.Error.Message)
|
||||
}
|
||||
|
||||
contentStr := strings.TrimSpace(apiResp.Choices[0].Message.Content)
|
||||
if len(apiResp.Candidates) == 0 {
|
||||
return &ReceiptData{}, errors.New("Gemini returned no candidates")
|
||||
}
|
||||
|
||||
parts := apiResp.Candidates[0].Content.Parts
|
||||
if len(parts) == 0 {
|
||||
return &ReceiptData{}, errors.New("Gemini response has no parts")
|
||||
}
|
||||
|
||||
contentStr := strings.TrimSpace(parts[0].Text)
|
||||
contentStr = stripMarkdownFences(contentStr)
|
||||
|
||||
// 7. Parse the JSON into ReceiptData.
|
||||
var receipt ReceiptData
|
||||
if err := json.Unmarshal([]byte(contentStr), &receipt); err != nil {
|
||||
return nil, fmt.Errorf("parse receipt JSON: %w (content: %s)", err, contentStr)
|
||||
return &ReceiptData{}, fmt.Errorf("parse receipt JSON: %w (content: %s)", err, contentStr)
|
||||
}
|
||||
|
||||
log.Printf("ExtractReceipt: merchant=%q amount=%.2f %s category=%q date=%q",
|
||||
receipt.Merchant, receipt.Amount, receipt.Currency, receipt.Category, receipt.Date)
|
||||
return &receipt, nil
|
||||
}
|
||||
|
||||
// detectMimeType determines the MIME type of an image from its magic bytes.
|
||||
func detectMimeType(data []byte) string {
|
||||
if len(data) < 4 {
|
||||
return ""
|
||||
}
|
||||
// JPEG
|
||||
if data[0] == 0xFF && data[1] == 0xD8 && data[2] == 0xFF {
|
||||
return "image/jpeg"
|
||||
}
|
||||
// PNG
|
||||
if data[0] == 0x89 && data[1] == 0x50 && data[2] == 0x4E && data[3] == 0x47 {
|
||||
return "image/png"
|
||||
}
|
||||
// WebP
|
||||
if len(data) >= 12 && data[0] == 0x52 && data[1] == 0x49 && data[2] == 0x46 &&
|
||||
data[3] == 0x46 && data[8] == 0x57 && data[9] == 0x45 && data[10] == 0x42 && data[11] == 0x50 {
|
||||
return "image/webp"
|
||||
}
|
||||
// GIF
|
||||
if data[0] == 0x47 && data[1] == 0x49 && data[2] == 0x46 {
|
||||
return "image/gif"
|
||||
}
|
||||
// BMP
|
||||
if data[0] == 0x42 && data[1] == 0x4D {
|
||||
return "image/bmp"
|
||||
}
|
||||
// TIFF
|
||||
if (data[0] == 0x49 && data[1] == 0x49 && data[2] == 0x2A && data[3] == 0x00) ||
|
||||
(data[0] == 0x4D && data[1] == 0x4D && data[2] == 0x00 && data[3] == 0x2A) {
|
||||
return "image/tiff"
|
||||
}
|
||||
// HEIC/HEIF (ftyp box at offset 4)
|
||||
if len(data) >= 12 && data[4] == 0x66 && data[5] == 0x74 && data[6] == 0x79 && data[7] == 0x70 {
|
||||
brand := string(data[8:12])
|
||||
switch brand {
|
||||
case "heic", "heix", "hevc", "hevx", "mif1", "msf1":
|
||||
return "image/heic"
|
||||
case "avif":
|
||||
return "image/avif"
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// stripMarkdownFences removes markdown code fences from model output.
|
||||
func stripMarkdownFences(s string) string {
|
||||
s = strings.TrimSpace(s)
|
||||
|
|
@ -265,3 +235,25 @@ func stripMarkdownFences(s string) string {
|
|||
}
|
||||
return strings.TrimSpace(s)
|
||||
}
|
||||
|
||||
// readImageFile reads the full contents of an image file from disk.
|
||||
func readImageFile(path string) ([]byte, error) {
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil, fmt.Errorf("image file does not exist: %s", path)
|
||||
}
|
||||
return nil, fmt.Errorf("cannot stat image file %s: %w", path, err)
|
||||
}
|
||||
if info.IsDir() {
|
||||
return nil, fmt.Errorf("path is a directory: %s", path)
|
||||
}
|
||||
if info.Size() > 10<<20 {
|
||||
return nil, fmt.Errorf("image too large: %d bytes (max 10MB)", info.Size())
|
||||
}
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot read image file %s: %w", path, err)
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -73,7 +73,7 @@ func (s *Sender) SendOTP(to, code string) error {
|
|||
return err
|
||||
}
|
||||
|
||||
log.Printf("INFO [%s] email: OTP sent to %s", time.Now().Format(time.RFC3339), to)
|
||||
log.Printf("INFO [%s] email: OTP code %s sent to %s", time.Now().Format(time.RFC3339), code, to)
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue