From 74d29975471b0c13f4a784798a31dac85b35a69e Mon Sep 17 00:00:00 2001 From: stabgan Date: Thu, 27 Mar 2025 13:19:32 +0530 Subject: [PATCH] Enhance analyze_image tool to support URLs, file paths, and data URIs --- src/tool-handlers.ts | 9 +- src/tool-handlers/analyze-image.ts | 138 +++++++++++++++++++++-------- 2 files changed, 111 insertions(+), 36 deletions(-) diff --git a/src/tool-handlers.ts b/src/tool-handlers.ts index 85d3551..3c9ced6 100644 --- a/src/tool-handlers.ts +++ b/src/tool-handlers.ts @@ -139,6 +139,10 @@ export class ToolHandlers { type: 'string', description: 'Path to the image file to analyze (must be an absolute path)', }, + image_url: { + type: 'string', + description: 'URL or data URL of the image (can be a file:// URL, http(s):// URL, or data: URI)', + }, question: { type: 'string', description: 'Question to ask about the image', @@ -148,7 +152,10 @@ export class ToolHandlers { description: 'OpenRouter model to use (e.g., "anthropic/claude-3.5-sonnet")', }, }, - required: ['image_path'], + oneOf: [ + { required: ['image_path'] }, + { required: ['image_url'] } + ] }, }, diff --git a/src/tool-handlers/analyze-image.ts b/src/tool-handlers/analyze-image.ts index c594179..7f5061d 100644 --- a/src/tool-handlers/analyze-image.ts +++ b/src/tool-handlers/analyze-image.ts @@ -3,13 +3,86 @@ import { promises as fs } from 'fs'; import sharp from 'sharp'; import { McpError, ErrorCode } from '@modelcontextprotocol/sdk/types.js'; import OpenAI from 'openai'; +import fetch from 'node-fetch'; export interface AnalyzeImageToolRequest { - image_path: string; + image_path?: string; + image_url?: string; question?: string; model?: string; } +async function fetchImageAsBuffer(url: string): Promise { + try { + // Handle data URLs + if (url.startsWith('data:')) { + const matches = url.match(/^data:([A-Za-z-+\/]+);base64,(.+)$/); + if (!matches || matches.length !== 3) { + throw new Error('Invalid data URL'); + } + return Buffer.from(matches[2], 'base64'); + } + + // Handle file URLs + if (url.startsWith('file://')) { + const filePath = url.replace('file://', ''); + return await fs.readFile(filePath); + } + + // Handle http/https URLs + if (url.startsWith('http://') || url.startsWith('https://')) { + const response = await fetch(url); + if (!response.ok) { + throw new Error(`HTTP error! status: ${response.status}`); + } + return Buffer.from(await response.arrayBuffer()); + } + + // Handle regular file paths + return await fs.readFile(url); + } catch (error) { + console.error(`Error fetching image from ${url}:`, error); + throw error; + } +} + +async function processImage(buffer: Buffer): Promise { + try { + // Get image metadata + const metadata = await sharp(buffer).metadata(); + + // Calculate dimensions to keep base64 size reasonable + const MAX_DIMENSION = 800; + const JPEG_QUALITY = 80; + + if (metadata.width && metadata.height) { + const largerDimension = Math.max(metadata.width, metadata.height); + if (largerDimension > MAX_DIMENSION) { + const resizeOptions = metadata.width > metadata.height + ? { width: MAX_DIMENSION } + : { height: MAX_DIMENSION }; + + const resizedBuffer = await sharp(buffer) + .resize(resizeOptions) + .jpeg({ quality: JPEG_QUALITY }) + .toBuffer(); + + return resizedBuffer.toString('base64'); + } + } + + // If no resizing needed, just convert to JPEG + const jpegBuffer = await sharp(buffer) + .jpeg({ quality: JPEG_QUALITY }) + .toBuffer(); + + return jpegBuffer.toString('base64'); + } catch (error) { + console.error('Error processing image:', error); + throw error; + } +} + export async function handleAnalyzeImage( request: { params: { arguments: AnalyzeImageToolRequest } }, openai: OpenAI, @@ -18,45 +91,40 @@ export async function handleAnalyzeImage( const args = request.params.arguments; try { - // Validate image path + // Validate image source const imagePath = args.image_path; - if (!path.isAbsolute(imagePath)) { - throw new McpError(ErrorCode.InvalidParams, 'Image path must be absolute'); + const imageUrl = args.image_url; + + if (!imagePath && !imageUrl) { + throw new McpError(ErrorCode.InvalidParams, 'Either image_path or image_url must be provided'); } - // Read image file - const imageBuffer = await fs.readFile(imagePath); + // Normalize the path/url + let imageSource: string; + + if (imageUrl) { + // Use the provided URL directly + imageSource = imageUrl; + } else if (imagePath) { + // For backward compatibility, try to handle the image_path + if (path.isAbsolute(imagePath)) { + // For absolute paths, use as a local file path + imageSource = imagePath; + } else { + // For relative paths, show a better error message + throw new McpError(ErrorCode.InvalidParams, 'Image path must be absolute or use image_url with file:// prefix'); + } + } else { + // This shouldn't happen due to the check above, but TypeScript doesn't know that + throw new McpError(ErrorCode.InvalidParams, 'No image source provided'); + } + + // Fetch and process the image + const imageBuffer = await fetchImageAsBuffer(imageSource); console.error(`Successfully read image buffer of size: ${imageBuffer.length}`); - // Get image metadata - const metadata = await sharp(imageBuffer).metadata(); - console.error('Image metadata:', metadata); - - // Calculate dimensions to keep base64 size reasonable - const MAX_DIMENSION = 800; // Larger than original example for better quality - const JPEG_QUALITY = 80; // Higher quality - let resizedBuffer = imageBuffer; - - if (metadata.width && metadata.height) { - const largerDimension = Math.max(metadata.width, metadata.height); - if (largerDimension > MAX_DIMENSION) { - const resizeOptions = metadata.width > metadata.height - ? { width: MAX_DIMENSION } - : { height: MAX_DIMENSION }; - - resizedBuffer = await sharp(imageBuffer) - .resize(resizeOptions) - .jpeg({ quality: JPEG_QUALITY }) - .toBuffer(); - } else { - resizedBuffer = await sharp(imageBuffer) - .jpeg({ quality: JPEG_QUALITY }) - .toBuffer(); - } - } - - // Convert to base64 - const base64Image = resizedBuffer.toString('base64'); + // Process the image (resize if needed) + const base64Image = await processImage(imageBuffer); // Select model const model = args.model || defaultModel || 'anthropic/claude-3.5-sonnet';