From ebe06082cfec979ad97a4aad476eec174d277baa Mon Sep 17 00:00:00 2001 From: cclohmar Date: Sat, 30 May 2026 11:54:31 +0000 Subject: [PATCH] fix: support HEIC/HEIF photos from iPhone + broaden accepted image formats - detectImageExtension now handles: JPEG, PNG, WebP, GIF, BMP, TIFF, HEIC, AVIF - Added heif-convert + ImageMagick fallback for decoding unsupported formats - AI extraction properly converts HEIC to JPEG before analysis - Model: deepseek-v4-flash (confirmed working) --- internal/ai/deepseek.go | 76 ++++++++++++++++++++++++++++++++++- internal/handlers/expenses.go | 52 +++++++++++++++++++++--- 2 files changed, 120 insertions(+), 8 deletions(-) diff --git a/internal/ai/deepseek.go b/internal/ai/deepseek.go index 0a95190..aee1ac4 100644 --- a/internal/ai/deepseek.go +++ b/internal/ai/deepseek.go @@ -12,6 +12,8 @@ import ( "log" "net/http" "os" + "os/exec" + "path/filepath" "strings" "time" ) @@ -180,11 +182,24 @@ Image data: %s`, b64) // compressImage decodes the image, downscales it (preserving aspect ratio) // to fit within maxImageSize, and re-encodes as JPEG with the configured quality. +// If Go's built-in decoders cannot handle the format (e.g. HEIC from iPhones), +// it attempts to convert via external tools (heif-convert or ImageMagick). func compressImage(data []byte) ([]byte, error) { - // Detect format and decode. + // Try Go's built-in image decoders first (covers JPEG, PNG, GIF, BMP, TIFF, WebP). img, format, err := image.Decode(bytes.NewReader(data)) if err != nil { - return nil, fmt.Errorf("image decode: %w", err) + // Built-in decoder failed — try external tools for HEIC and other formats. + log.Printf("compressImage: Go decoder failed (%v), trying external conversion", err) + converted, err := convertWithExternalTool(data) + if err != nil { + return nil, fmt.Errorf("image decode and external conversion both failed: %w", err) + } + // Try decoding the converted data. + img, format, err = image.Decode(bytes.NewReader(converted)) + if err != nil { + return nil, fmt.Errorf("decoding converted image also failed: %w", err) + } + data = converted } bounds := img.Bounds() @@ -261,6 +276,63 @@ func stripMarkdownFences(s string) string { return strings.TrimSpace(s) } +// convertWithExternalTool tries to convert an unsupported image format (e.g. HEIC) +// to JPEG using heif-convert or ImageMagick's convert command. +func convertWithExternalTool(data []byte) ([]byte, error) { + // Write the unknown image to a temp file (needed for CLI tools). + tmpDir := os.TempDir() + inPath := filepath.Join(tmpDir, "ef-convert-in-"+fmt.Sprintf("%d", time.Now().UnixNano())) + outPath := filepath.Join(tmpDir, "ef-convert-out-"+fmt.Sprintf("%d", time.Now().UnixNano())+".jpg") + + if err := os.WriteFile(inPath, data, 0644); err != nil { + return nil, fmt.Errorf("write temp input: %w", err) + } + defer os.Remove(inPath) + defer os.Remove(outPath) + + // Try heif-convert first (fastest, handles HEIC natively). + if heifErr := tryHEIFConvert(inPath, outPath); heifErr == nil { + outData, err := os.ReadFile(outPath) + if err == nil && len(outData) > 0 { + log.Printf("convertWithExternalTool: heif-convert succeeded (%d bytes)", len(outData)) + return outData, nil + } + } + + // Fall back to ImageMagick convert. + if magickErr := exec.Command("convert", inPath, "-resize", fmt.Sprintf("%dx%d>", maxImageSize, maxImageSize), "-quality", fmt.Sprintf("%d", jpegQuality), outPath).Run(); magickErr == nil { + outData, err := os.ReadFile(outPath) + if err == nil && len(outData) > 0 { + log.Printf("convertWithExternalTool: ImageMagick convert succeeded (%d bytes)", len(outData)) + return outData, nil + } + } + + return nil, errors.New("all external conversion tools failed") +} + +// tryHEIFConvert attempts to convert a HEIC/HEIF file to JPEG using heif-convert. +func tryHEIFConvert(inPath, outPath string) error { + // heif-convert outputs to {input}.jpg by default. + defaultOut := inPath + ".jpg" + defer os.Remove(defaultOut) + + cmd := exec.Command("heif-convert", inPath, outPath) + if output, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("heif-convert failed: %w, output: %s", err, string(output)) + } + + // If it wrote to the default path instead of our outPath, move it. + if _, err := os.Stat(outPath); os.IsNotExist(err) { + if _, err := os.Stat(defaultOut); err == nil { + os.Rename(defaultOut, outPath) + } + } + + _, err := os.Stat(outPath) + return err +} + // readImageFile reads the full contents of an image file from disk. func readImageFile(path string) ([]byte, error) { info, err := os.Stat(path) diff --git a/internal/handlers/expenses.go b/internal/handlers/expenses.go index 034f23f..6382308 100644 --- a/internal/handlers/expenses.go +++ b/internal/handlers/expenses.go @@ -320,19 +320,59 @@ func setCurrentEventID(w http.ResponseWriter, eventID string) { // --------------------------------------------------------------------------- // detectImageExtension examines the magic bytes of the provided data to -// determine whether it is a JPEG or PNG image. Returns "jpg", "png", or -// an empty string if the format is not recognised. +// determine its image format. Supports JPEG, PNG, WebP, GIF, BMP, TIFF, +// and HEIC/HEIF (common on iPhones). Returns the file extension (without +// dot) or an empty string if the format is not recognised. func detectImageExtension(data []byte) string { if len(data) < 4 { return "" } - // JPEG magic: 0xFF 0xD8 0xFF - if data[0] == 0xFF && data[1] == 0xD8 && data[2] == 0xFF { + + // JPEG: FF D8 FF + if len(data) >= 3 && data[0] == 0xFF && data[1] == 0xD8 && data[2] == 0xFF { return "jpg" } - // PNG magic: 0x89 'P' 'N' 'G' 0x0D 0x0A 0x1A 0x0A - if data[0] == 0x89 && data[1] == 0x50 && data[2] == 0x4E && data[3] == 0x47 { + + // PNG: 89 50 4E 47 0D 0A 1A 0A + if len(data) >= 8 && data[0] == 0x89 && data[1] == 0x50 && data[2] == 0x4E && + data[3] == 0x47 && data[4] == 0x0D && data[5] == 0x0A && data[6] == 0x1A && data[7] == 0x0A { return "png" } + + // WebP: 52 49 46 46 .... 57 45 42 50 + if len(data) >= 12 && data[0] == 0x52 && data[1] == 0x49 && data[2] == 0x46 && + data[3] == 0x46 && data[8] == 0x57 && data[9] == 0x45 && data[10] == 0x42 && data[11] == 0x50 { + return "webp" + } + + // GIF: 47 49 46 38 (39 61 or 37 61) + if len(data) >= 6 && data[0] == 0x47 && data[1] == 0x49 && data[2] == 0x46 && + data[3] == 0x38 && (data[4] == 0x39 || data[4] == 0x37) && data[5] == 0x61 { + return "gif" + } + + // BMP: 42 4D + if data[0] == 0x42 && data[1] == 0x4D { + return "bmp" + } + + // TIFF: 49 49 2A 00 or 4D 4D 00 2A + if (data[0] == 0x49 && data[1] == 0x49 && data[2] == 0x2A && data[3] == 0x00) || + (data[0] == 0x4D && data[1] == 0x4D && data[2] == 0x00 && data[3] == 0x2A) { + return "tiff" + } + + // HEIC/HEIF/AVIF: .... 66 74 79 70 ... (ftyp box) + // The ftyp box starts at offset 4 with brand at offset 8. + if len(data) >= 12 && data[4] == 0x66 && data[5] == 0x74 && data[6] == 0x79 && data[7] == 0x70 { + brand := string(data[8:12]) + switch brand { + case "heic", "heix", "hevc", "hevx", "mif1", "msf1": + return "heic" + case "avif": + return "avif" + } + } + return "" }