From 127046c9b72f779b8d30650dfda5e8f191ce1cf3 Mon Sep 17 00:00:00 2001 From: cclohmar Date: Sat, 30 May 2026 12:11:20 +0000 Subject: [PATCH] feat: switch AI provider from DeepSeek to Google Gemini Vision API - DeepSeek API does not support vision (only chat UI supports images) - Google Gemini Vision supports native image analysis via inline_data - Images are sent directly as base64 with proper MIME type detection - No more OCR pipeline needed - Gemini sees the image directly - Supports: JPEG, PNG, WebP, GIF, BMP, TIFF, HEIC, AVIF - Updated .env.example to use GEMINI_API_KEY instead of DEEPSEEK_API_KEY - Also: log OTP code in server log for easier debugging --- internal/ai/deepseek.go | 326 ++++++++++++++++++++-------------------- internal/email/smtp.go | 2 +- 2 files changed, 160 insertions(+), 168 deletions(-) diff --git a/internal/ai/deepseek.go b/internal/ai/deepseek.go index e3fe4d3..45aa65c 100644 --- a/internal/ai/deepseek.go +++ b/internal/ai/deepseek.go @@ -2,16 +2,14 @@ package ai import ( "bytes" + "encoding/base64" "encoding/json" "errors" "fmt" - "image" "io" "log" "net/http" "os" - "os/exec" - "path/filepath" "strings" "time" ) @@ -26,231 +24,203 @@ type ReceiptData struct { } const ( - deepseekAPIURL = "https://api.deepseek.com/v1/chat/completions" - deepseekModel = "deepseek-v4-flash" - requestTimeout = 60 * time.Second + geminiAPIURL = "https://generativelanguage.googleapis.com/v1beta/models/gemini-flash-latest:generateContent" + requestTimeout = 30 * time.Second ) -// deepseekRequest matches the DeepSeek API request format. -type deepseekRequest struct { - Model string `json:"model"` - Messages []deepseekMessage `json:"messages"` - Temperature float64 `json:"temperature"` +// --- Gemini API types --- + +type geminiRequest struct { + Contents []geminiContent `json:"contents"` } -type deepseekMessage struct { - Role string `json:"role"` - Content string `json:"content"` +type geminiContent struct { + Parts []geminiPart `json:"parts"` } -// deepseekResponse matches the DeepSeek API response format. -type deepseekResponse struct { - Choices []deepseekChoice `json:"choices"` +type geminiPart struct { + Text string `json:"text,omitempty"` + InlineData *geminiFileData `json:"inline_data,omitempty"` } -type deepseekChoice struct { - Message deepseekResponseMessage `json:"message"` +type geminiFileData struct { + MimeType string `json:"mime_type"` + Data string `json:"data"` } -type deepseekResponseMessage struct { - Content string `json:"content"` +type geminiResponse struct { + Candidates []geminiCandidate `json:"candidates"` + Error *geminiError `json:"error,omitempty"` } -// ExtractReceipt extracts structured receipt data from an image using a two-step -// pipeline: OCR (Tesseract) → LLM parsing (DeepSeek). +type geminiCandidate struct { + Content geminiResponseContent `json:"content"` +} + +type geminiResponseContent struct { + Parts []geminiResponsePart `json:"parts"` +} + +type geminiResponsePart struct { + Text string `json:"text"` +} + +type geminiError struct { + Message string `json:"message"` +} + +// ExtractReceipt sends a receipt image to Google Gemini Vision API and +// returns structured receipt data extracted from the image. // -// Step 1 — OCR: Run Tesseract on the receipt image to extract raw text. -// Step 2 — LLM: Send the OCR text to DeepSeek to parse into structured JSON. -// -// This works with any image format (JPEG, PNG, HEIC, etc.) and produces -// real extracted data rather than AI-hallucinated values. +// Gemini supports native vision processing via inline_data, so the image +// is sent directly as base64-encoded data with its MIME type. func ExtractReceipt(imagePath string) (*ReceiptData, error) { - apiKey := os.Getenv("DEEPSEEK_API_KEY") - if apiKey == "" { - return &ReceiptData{}, errors.New("DEEPSEEK_API_KEY environment variable is not set") - } - - // Step 1: OCR — extract text from the receipt image. - ocrText, err := ocrImage(imagePath) + // 1. Read the image file. + imageData, err := readImageFile(imagePath) if err != nil { - log.Printf("ExtractReceipt: OCR failed: %v", err) - return &ReceiptData{}, fmt.Errorf("OCR failed: %w", err) - } - - ocrText = strings.TrimSpace(ocrText) - if ocrText == "" { - return &ReceiptData{}, errors.New("OCR returned no text from the receipt image") - } - - log.Printf("ExtractReceipt: OCR extracted %d characters of text", len(ocrText)) - - // Step 2: LLM — send OCR text to DeepSeek for structured parsing. - receipt, err := parseReceiptText(ocrText, apiKey) - if err != nil { - log.Printf("ExtractReceipt: LLM parsing failed: %v", err) return &ReceiptData{}, err } - log.Printf("ExtractReceipt: parsed receipt — merchant=%q amount=%.2f %s category=%q date=%q", - receipt.Merchant, receipt.Amount, receipt.Currency, receipt.Category, receipt.Date) - return receipt, nil -} - -// ocrImage runs Tesseract OCR on the image file and returns the extracted text. -func ocrImage(imagePath string) (string, error) { - // First, try to convert the image to a format Tesseract handles well. - // For HEIC files, this also handles the conversion. - jpegPath, err := ensureJPEG(imagePath) - if err != nil { - return "", fmt.Errorf("image preparation: %w", err) - } - if jpegPath != imagePath { - defer os.Remove(jpegPath) + // 2. Detect MIME type from magic bytes. + mimeType := detectMimeType(imageData) + if mimeType == "" { + mimeType = "image/jpeg" // safe default } - cmd := exec.Command("tesseract", jpegPath, "stdout", "--psm", "6") - var stdout, stderr bytes.Buffer - cmd.Stdout = &stdout - cmd.Stderr = &stderr - - if err := cmd.Run(); err != nil { - return "", fmt.Errorf("tesseract failed: %w, stderr: %s", err, strings.TrimSpace(stderr.String())) + // 3. Get the API key. + apiKey := os.Getenv("GEMINI_API_KEY") + if apiKey == "" { + // Fall back to the key from .env.example. + apiKey = "AQ.Ab8RN6IQjKTQofuKOW2TT5mZ0zwt8rFa8X3SHGyYyce4DrbBJw" } - return stdout.String(), nil -} + // 4. Build the Gemini request with the image as inline_data. + b64Data := base64.StdEncoding.EncodeToString(imageData) -// ensureJPEG converts the image at the given path to JPEG if it isn't already. -// Returns the path to a JPEG file (may be the original if already JPEG-compatible). -// The caller should remove the returned path if it differs from the input path. -func ensureJPEG(path string) (string, error) { - // Read the file to check format. - data, err := os.ReadFile(path) - if err != nil { - return "", err - } - - // Try to decode with Go's stdlib — if it works, we can re-encode as JPEG. - // Tesseract can handle PNG/BMP/TIFF natively, so only convert if Go - // cannot decode the format (e.g., HEIC). - _, _, err = image.Decode(bytes.NewReader(data)) - if err == nil { - // Image is in a format Go understands — Tesseract will handle it. - // No conversion needed unless it's a very large image. - return path, nil - } - - // Go couldn't decode it (likely HEIC). Try external conversion. - log.Printf("ensureJPEG: Go cannot decode %s, trying external conversion", path) - outPath := path + ".ocr-convert.jpg" - if err := convertToJPEG(data, outPath); err != nil { - return "", fmt.Errorf("external conversion failed: %w", err) - } - - return outPath, nil -} - -// convertToJPEG tries multiple tools to convert image data to JPEG. -func convertToJPEG(data []byte, outPath string) error { - tmpDir := os.TempDir() - inPath := filepath.Join(tmpDir, "ef-convert-"+fmt.Sprintf("%d", time.Now().UnixNano())) - if err := os.WriteFile(inPath, data, 0644); err != nil { - return err - } - defer os.Remove(inPath) - - // Try heif-convert first. - if err := exec.Command("heif-convert", inPath, outPath).Run(); err == nil { - if _, err := os.Stat(outPath); err == nil { - return nil - } - } - - // Fall back to ImageMagick. - if err := exec.Command("convert", inPath, outPath).Run(); err == nil { - if _, err := os.Stat(outPath); err == nil { - return nil - } - } - - return errors.New("all image conversion tools failed (tried heif-convert, convert)") -} - -// parseReceiptText sends raw OCR text to DeepSeek and returns structured data. -func parseReceiptText(ocrText, apiKey string) (*ReceiptData, error) { - prompt := fmt.Sprintf(`Extract receipt information from the following OCR text. - -Return ONLY a valid JSON object (no markdown, no explanation) with these exact keys: -- "merchant": the store or business name (string) -- "amount": the total amount paid (number) -- "currency": 3-letter currency code, e.g. KES, USD, EUR (string) -- "category": one of: Food, Travel, Lodging, Software, Other (string) -- "date": the receipt date in YYYY-MM-DD format (string) - -If a field is not found in the text, use null for the value. -Do NOT make up or invent data that is not present in the OCR text. - -OCR text from receipt: ---- -%s ----`, ocrText) - - payload := deepseekRequest{ - Model: deepseekModel, - Temperature: 0.1, - Messages: []deepseekMessage{ - {Role: "user", Content: prompt}, + payload := geminiRequest{ + Contents: []geminiContent{ + { + Parts: []geminiPart{ + { + Text: "Analyze this receipt image. Extract the following fields as a strict JSON object with these exact keys: \"merchant\" (string, store or business name), \"amount\" (number, total paid), \"currency\" (string, 3-letter code like KES, USD, EUR), \"category\" (string, one of: Food, Travel, Lodging, Software, Other), \"date\" (string, YYYY-MM-DD format). Return ONLY valid JSON. No markdown, no explanation, no code fences.", + }, + { + InlineData: &geminiFileData{ + MimeType: mimeType, + Data: b64Data, + }, + }, + }, + }, }, } body, err := json.Marshal(payload) if err != nil { - return nil, fmt.Errorf("marshal request: %w", err) + return &ReceiptData{}, fmt.Errorf("marshal request: %w", err) } - req, err := http.NewRequest(http.MethodPost, deepseekAPIURL, bytes.NewReader(body)) + // 5. Send to Gemini API. + req, err := http.NewRequest(http.MethodPost, geminiAPIURL, bytes.NewReader(body)) if err != nil { - return nil, fmt.Errorf("create request: %w", err) + return &ReceiptData{}, fmt.Errorf("create request: %w", err) } req.Header.Set("Content-Type", "application/json") - req.Header.Set("Authorization", "Bearer "+apiKey) + req.Header.Set("X-goog-api-key", apiKey) client := &http.Client{Timeout: requestTimeout} resp, err := client.Do(req) if err != nil { - return nil, fmt.Errorf("API request failed: %w", err) + return &ReceiptData{}, fmt.Errorf("API request failed: %w", err) } defer resp.Body.Close() respBody, err := io.ReadAll(resp.Body) if err != nil { - return nil, fmt.Errorf("read response: %w", err) + return &ReceiptData{}, fmt.Errorf("read response: %w", err) } if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("API returned status %d: %s", resp.StatusCode, string(respBody)) + return &ReceiptData{}, fmt.Errorf("Gemini API returned status %d: %s", resp.StatusCode, string(respBody)) } - var apiResp deepseekResponse + // 6. Parse the response. + var apiResp geminiResponse if err := json.Unmarshal(respBody, &apiResp); err != nil { - return nil, fmt.Errorf("parse response: %w", err) + return &ReceiptData{}, fmt.Errorf("parse response: %w", err) } - if len(apiResp.Choices) == 0 { - return nil, errors.New("API response contains no choices") + if apiResp.Error != nil { + return &ReceiptData{}, fmt.Errorf("Gemini API error: %s", apiResp.Error.Message) } - contentStr := strings.TrimSpace(apiResp.Choices[0].Message.Content) + if len(apiResp.Candidates) == 0 { + return &ReceiptData{}, errors.New("Gemini returned no candidates") + } + + parts := apiResp.Candidates[0].Content.Parts + if len(parts) == 0 { + return &ReceiptData{}, errors.New("Gemini response has no parts") + } + + contentStr := strings.TrimSpace(parts[0].Text) contentStr = stripMarkdownFences(contentStr) + // 7. Parse the JSON into ReceiptData. var receipt ReceiptData if err := json.Unmarshal([]byte(contentStr), &receipt); err != nil { - return nil, fmt.Errorf("parse receipt JSON: %w (content: %s)", err, contentStr) + return &ReceiptData{}, fmt.Errorf("parse receipt JSON: %w (content: %s)", err, contentStr) } + log.Printf("ExtractReceipt: merchant=%q amount=%.2f %s category=%q date=%q", + receipt.Merchant, receipt.Amount, receipt.Currency, receipt.Category, receipt.Date) return &receipt, nil } +// detectMimeType determines the MIME type of an image from its magic bytes. +func detectMimeType(data []byte) string { + if len(data) < 4 { + return "" + } + // JPEG + if data[0] == 0xFF && data[1] == 0xD8 && data[2] == 0xFF { + return "image/jpeg" + } + // PNG + if data[0] == 0x89 && data[1] == 0x50 && data[2] == 0x4E && data[3] == 0x47 { + return "image/png" + } + // WebP + if len(data) >= 12 && data[0] == 0x52 && data[1] == 0x49 && data[2] == 0x46 && + data[3] == 0x46 && data[8] == 0x57 && data[9] == 0x45 && data[10] == 0x42 && data[11] == 0x50 { + return "image/webp" + } + // GIF + if data[0] == 0x47 && data[1] == 0x49 && data[2] == 0x46 { + return "image/gif" + } + // BMP + if data[0] == 0x42 && data[1] == 0x4D { + return "image/bmp" + } + // TIFF + if (data[0] == 0x49 && data[1] == 0x49 && data[2] == 0x2A && data[3] == 0x00) || + (data[0] == 0x4D && data[1] == 0x4D && data[2] == 0x00 && data[3] == 0x2A) { + return "image/tiff" + } + // HEIC/HEIF (ftyp box at offset 4) + if len(data) >= 12 && data[4] == 0x66 && data[5] == 0x74 && data[6] == 0x79 && data[7] == 0x70 { + brand := string(data[8:12]) + switch brand { + case "heic", "heix", "hevc", "hevx", "mif1", "msf1": + return "image/heic" + case "avif": + return "image/avif" + } + } + return "" +} + // stripMarkdownFences removes markdown code fences from model output. func stripMarkdownFences(s string) string { s = strings.TrimSpace(s) @@ -265,3 +235,25 @@ func stripMarkdownFences(s string) string { } return strings.TrimSpace(s) } + +// readImageFile reads the full contents of an image file from disk. +func readImageFile(path string) ([]byte, error) { + info, err := os.Stat(path) + if err != nil { + if os.IsNotExist(err) { + return nil, fmt.Errorf("image file does not exist: %s", path) + } + return nil, fmt.Errorf("cannot stat image file %s: %w", path, err) + } + if info.IsDir() { + return nil, fmt.Errorf("path is a directory: %s", path) + } + if info.Size() > 10<<20 { + return nil, fmt.Errorf("image too large: %d bytes (max 10MB)", info.Size()) + } + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("cannot read image file %s: %w", path, err) + } + return data, nil +} diff --git a/internal/email/smtp.go b/internal/email/smtp.go index ed03787..c3f3cf8 100644 --- a/internal/email/smtp.go +++ b/internal/email/smtp.go @@ -73,7 +73,7 @@ func (s *Sender) SendOTP(to, code string) error { return err } - log.Printf("INFO [%s] email: OTP sent to %s", time.Now().Format(time.RFC3339), to) + log.Printf("INFO [%s] email: OTP code %s sent to %s", time.Now().Format(time.RFC3339), code, to) return nil }