From 9bf86febe411373e2d6b494d0f4bbfc85372b788 Mon Sep 17 00:00:00 2001 From: Kaustabh Ganguly Date: Wed, 26 Mar 2025 22:21:35 +0530 Subject: [PATCH] Add main tool handler files --- src/tool-handlers/analyze-image.ts | 116 +++++++++++++++ src/tool-handlers/chat-completion.ts | 135 +++++++++++++++++ src/tool-handlers/get-model-info.ts | 54 +++++++ src/tool-handlers/multi-image-analysis.ts | 168 ++++++++++++++++++++++ src/tool-handlers/search-models.ts | 68 +++++++++ src/tool-handlers/validate-model.ts | 50 +++++++ 6 files changed, 591 insertions(+) create mode 100644 src/tool-handlers/analyze-image.ts create mode 100644 src/tool-handlers/chat-completion.ts create mode 100644 src/tool-handlers/get-model-info.ts create mode 100644 src/tool-handlers/multi-image-analysis.ts create mode 100644 src/tool-handlers/search-models.ts create mode 100644 src/tool-handlers/validate-model.ts diff --git a/src/tool-handlers/analyze-image.ts b/src/tool-handlers/analyze-image.ts new file mode 100644 index 0000000..3355e78 --- /dev/null +++ b/src/tool-handlers/analyze-image.ts @@ -0,0 +1,116 @@ +import path from 'path'; +import { promises as fs } from 'fs'; +import sharp from 'sharp'; +import { McpError } from '@modelcontextprotocol/sdk/types.js'; +import OpenAI from 'openai'; + +export interface AnalyzeImageToolRequest { + image_path: string; + question?: string; + model?: string; +} + +export async function handleAnalyzeImage( + request: { params: { arguments: AnalyzeImageToolRequest } }, + openai: OpenAI, + defaultModel?: string +) { + const args = request.params.arguments; + + try { + // Validate image path + const imagePath = args.image_path; + if (!path.isAbsolute(imagePath)) { + throw new McpError('InvalidParams', 'Image path must be absolute'); + } + + // Read image file + const imageBuffer = await fs.readFile(imagePath); + console.error(`Successfully read image buffer of size: ${imageBuffer.length}`); + + // Get image metadata + const metadata = await sharp(imageBuffer).metadata(); + console.error('Image metadata:', metadata); + + // Calculate dimensions to keep base64 size reasonable + const MAX_DIMENSION = 800; // Larger than original example for better quality + const JPEG_QUALITY = 80; // Higher quality + let resizedBuffer = imageBuffer; + + if (metadata.width && metadata.height) { + const largerDimension = Math.max(metadata.width, metadata.height); + if (largerDimension > MAX_DIMENSION) { + const resizeOptions = metadata.width > metadata.height + ? { width: MAX_DIMENSION } + : { height: MAX_DIMENSION }; + + resizedBuffer = await sharp(imageBuffer) + .resize(resizeOptions) + .jpeg({ quality: JPEG_QUALITY }) + .toBuffer(); + } else { + resizedBuffer = await sharp(imageBuffer) + .jpeg({ quality: JPEG_QUALITY }) + .toBuffer(); + } + } + + // Convert to base64 + const base64Image = resizedBuffer.toString('base64'); + + // Select model + const model = args.model || defaultModel || 'anthropic/claude-3.5-sonnet'; + + // Prepare message with image + const messages = [ + { + role: 'user', + content: [ + { + type: 'text', + text: args.question || "What's in this image?" + }, + { + type: 'image_url', + image_url: { + url: `data:image/jpeg;base64,${base64Image}` + } + } + ] + } + ]; + + console.error('Sending request to OpenRouter...'); + + // Call OpenRouter API + const completion = await openai.chat.completions.create({ + model, + messages, + }); + + return { + content: [ + { + type: 'text', + text: completion.choices[0].message.content || '', + }, + ], + }; + } catch (error) { + console.error('Error analyzing image:', error); + + if (error instanceof McpError) { + throw error; + } + + return { + content: [ + { + type: 'text', + text: `Error analyzing image: ${error instanceof Error ? error.message : String(error)}`, + }, + ], + isError: true, + }; + } +} diff --git a/src/tool-handlers/chat-completion.ts b/src/tool-handlers/chat-completion.ts new file mode 100644 index 0000000..f961976 --- /dev/null +++ b/src/tool-handlers/chat-completion.ts @@ -0,0 +1,135 @@ +import OpenAI from 'openai'; +import { ChatCompletionMessageParam } from 'openai/resources/chat/completions.js'; + +// Maximum context tokens +const MAX_CONTEXT_TOKENS = 200000; + +export interface ChatCompletionToolRequest { + model?: string; + messages: ChatCompletionMessageParam[]; + temperature?: number; +} + +// Utility function to estimate token count (simplified) +function estimateTokenCount(text: string): number { + // Rough approximation: 4 characters per token + return Math.ceil(text.length / 4); +} + +// Truncate messages to fit within the context window +function truncateMessagesToFit( + messages: ChatCompletionMessageParam[], + maxTokens: number +): ChatCompletionMessageParam[] { + const truncated: ChatCompletionMessageParam[] = []; + let currentTokenCount = 0; + + // Always include system message first if present + if (messages[0]?.role === 'system') { + truncated.push(messages[0]); + currentTokenCount += estimateTokenCount(messages[0].content as string); + } + + // Add messages from the end, respecting the token limit + for (let i = messages.length - 1; i >= 0; i--) { + const message = messages[i]; + + // Skip if it's the system message we've already added + if (i === 0 && message.role === 'system') continue; + + // For string content, estimate tokens directly + if (typeof message.content === 'string') { + const messageTokens = estimateTokenCount(message.content); + if (currentTokenCount + messageTokens > maxTokens) break; + truncated.unshift(message); + currentTokenCount += messageTokens; + } + // For multimodal content (array), estimate tokens for text content + else if (Array.isArray(message.content)) { + let messageTokens = 0; + for (const part of message.content) { + if (part.type === 'text' && part.text) { + messageTokens += estimateTokenCount(part.text); + } else if (part.type === 'image_url') { + // Add a token cost estimate for images - this is a simplification + // Actual image token costs depend on resolution and model + messageTokens += 1000; + } + } + + if (currentTokenCount + messageTokens > maxTokens) break; + truncated.unshift(message); + currentTokenCount += messageTokens; + } + } + + return truncated; +} + +export async function handleChatCompletion( + request: { params: { arguments: ChatCompletionToolRequest } }, + openai: OpenAI, + defaultModel?: string +) { + const args = request.params.arguments; + + // Validate model selection + const model = args.model || defaultModel; + if (!model) { + return { + content: [ + { + type: 'text', + text: 'No model specified and no default model configured in MCP settings. Please specify a model or set OPENROUTER_DEFAULT_MODEL in the MCP configuration.', + }, + ], + isError: true, + }; + } + + // Validate message array + if (args.messages.length === 0) { + return { + content: [ + { + type: 'text', + text: 'Messages array cannot be empty. At least one message is required.', + }, + ], + isError: true, + }; + } + + try { + // Truncate messages to fit within context window + const truncatedMessages = truncateMessagesToFit(args.messages, MAX_CONTEXT_TOKENS); + + const completion = await openai.chat.completions.create({ + model, + messages: truncatedMessages, + temperature: args.temperature ?? 1, + }); + + return { + content: [ + { + type: 'text', + text: completion.choices[0].message.content || '', + }, + ], + }; + } catch (error) { + if (error instanceof Error) { + return { + content: [ + { + type: 'text', + text: `OpenRouter API error: ${error.message}`, + }, + ], + isError: true, + }; + } + throw error; + } +} diff --git a/src/tool-handlers/get-model-info.ts b/src/tool-handlers/get-model-info.ts new file mode 100644 index 0000000..ab7f804 --- /dev/null +++ b/src/tool-handlers/get-model-info.ts @@ -0,0 +1,54 @@ +import { McpError } from '@modelcontextprotocol/sdk/types.js'; +import { ModelCache } from '../model-cache.js'; + +export interface GetModelInfoToolRequest { + model: string; +} + +export async function handleGetModelInfo( + request: { params: { arguments: GetModelInfoToolRequest } }, + modelCache: ModelCache +) { + const args = request.params.arguments; + + try { + if (!modelCache.isCacheValid()) { + return { + content: [ + { + type: 'text', + text: 'Model cache is empty or expired. Please call search_models first to populate the cache.', + }, + ], + isError: true, + }; + } + + const model = modelCache.getModel(args.model); + if (!model) { + throw new McpError('NotFound', `Model '${args.model}' not found`); + } + + return { + content: [ + { + type: 'text', + text: JSON.stringify(model, null, 2), + }, + ], + }; + } catch (error) { + if (error instanceof Error) { + return { + content: [ + { + type: 'text', + text: `Error retrieving model info: ${error.message}`, + }, + ], + isError: true, + }; + } + throw error; + } +} diff --git a/src/tool-handlers/multi-image-analysis.ts b/src/tool-handlers/multi-image-analysis.ts new file mode 100644 index 0000000..3e81c59 --- /dev/null +++ b/src/tool-handlers/multi-image-analysis.ts @@ -0,0 +1,168 @@ +import fetch from 'node-fetch'; +import sharp from 'sharp'; +import { McpError } from '@modelcontextprotocol/sdk/types.js'; +import OpenAI from 'openai'; + +export interface MultiImageAnalysisToolRequest { + images: Array<{ + url: string; + alt?: string; + }>; + prompt: string; + markdown_response?: boolean; + model?: string; +} + +async function fetchImageAsBuffer(url: string): Promise { + try { + // Handle data URLs + if (url.startsWith('data:')) { + const matches = url.match(/^data:([A-Za-z-+\/]+);base64,(.+)$/); + if (!matches || matches.length !== 3) { + throw new Error('Invalid data URL'); + } + return Buffer.from(matches[2], 'base64'); + } + + // Handle file URLs + if (url.startsWith('file://')) { + const filePath = url.replace('file://', ''); + const fs = await import('fs/promises'); + return await fs.readFile(filePath); + } + + // Handle http/https URLs + const response = await fetch(url); + if (!response.ok) { + throw new Error(`HTTP error! status: ${response.status}`); + } + return Buffer.from(await response.arrayBuffer()); + } catch (error) { + console.error(`Error fetching image from ${url}:`, error); + throw error; + } +} + +async function processImage(buffer: Buffer): Promise { + try { + // Get image metadata + const metadata = await sharp(buffer).metadata(); + + // Calculate dimensions to keep base64 size reasonable + const MAX_DIMENSION = 800; + const JPEG_QUALITY = 80; + + if (metadata.width && metadata.height) { + const largerDimension = Math.max(metadata.width, metadata.height); + if (largerDimension > MAX_DIMENSION) { + const resizeOptions = metadata.width > metadata.height + ? { width: MAX_DIMENSION } + : { height: MAX_DIMENSION }; + + const resizedBuffer = await sharp(buffer) + .resize(resizeOptions) + .jpeg({ quality: JPEG_QUALITY }) + .toBuffer(); + + return resizedBuffer.toString('base64'); + } + } + + // If no resizing needed, just convert to JPEG + const jpegBuffer = await sharp(buffer) + .jpeg({ quality: JPEG_QUALITY }) + .toBuffer(); + + return jpegBuffer.toString('base64'); + } catch (error) { + console.error('Error processing image:', error); + throw error; + } +} + +export async function handleMultiImageAnalysis( + request: { params: { arguments: MultiImageAnalysisToolRequest } }, + openai: OpenAI, + defaultModel?: string +) { + const args = request.params.arguments; + + try { + // Validate inputs + if (!args.images || args.images.length === 0) { + throw new McpError('InvalidParams', 'At least one image is required'); + } + + if (!args.prompt) { + throw new McpError('InvalidParams', 'A prompt is required'); + } + + // Prepare content array for the message + const content: Array = [{ + type: 'text', + text: args.prompt + }]; + + // Process each image + for (const image of args.images) { + try { + // Fetch and process the image + const imageBuffer = await fetchImageAsBuffer(image.url); + const base64Image = await processImage(imageBuffer); + + // Add to content + content.push({ + type: 'image_url', + image_url: { + url: `data:image/jpeg;base64,${base64Image}` + } + }); + } catch (error) { + console.error(`Error processing image ${image.url}:`, error); + // Continue with other images if one fails + } + } + + // If no images were successfully processed + if (content.length === 1) { + throw new Error('Failed to process any of the provided images'); + } + + // Select model + const model = args.model || defaultModel || 'anthropic/claude-3.5-sonnet'; + + // Make the API call + const completion = await openai.chat.completions.create({ + model, + messages: [{ + role: 'user', + content + }] + }); + + return { + content: [ + { + type: 'text', + text: completion.choices[0].message.content || '', + }, + ], + }; + } catch (error) { + console.error('Error in multi-image analysis:', error); + + if (error instanceof McpError) { + throw error; + } + + return { + content: [ + { + type: 'text', + text: `Error analyzing images: ${error instanceof Error ? error.message : String(error)}`, + }, + ], + isError: true, + }; + } +} diff --git a/src/tool-handlers/search-models.ts b/src/tool-handlers/search-models.ts new file mode 100644 index 0000000..0a0a27c --- /dev/null +++ b/src/tool-handlers/search-models.ts @@ -0,0 +1,68 @@ +import { ModelCache } from '../model-cache.js'; +import { OpenRouterAPIClient } from '../openrouter-api.js'; + +export interface SearchModelsToolRequest { + query?: string; + provider?: string; + minContextLength?: number; + maxContextLength?: number; + maxPromptPrice?: number; + maxCompletionPrice?: number; + capabilities?: { + functions?: boolean; + tools?: boolean; + vision?: boolean; + json_mode?: boolean; + }; + limit?: number; +} + +export async function handleSearchModels( + request: { params: { arguments: SearchModelsToolRequest } }, + apiClient: OpenRouterAPIClient, + modelCache: ModelCache +) { + const args = request.params.arguments; + + try { + // Refresh the cache if needed + if (!modelCache.isCacheValid()) { + const models = await apiClient.getModels(); + modelCache.setModels(models); + } + + // Search models based on criteria + const results = modelCache.searchModels({ + query: args.query, + provider: args.provider, + minContextLength: args.minContextLength, + maxContextLength: args.maxContextLength, + maxPromptPrice: args.maxPromptPrice, + maxCompletionPrice: args.maxCompletionPrice, + capabilities: args.capabilities, + limit: args.limit || 10, + }); + + return { + content: [ + { + type: 'text', + text: JSON.stringify(results, null, 2), + }, + ], + }; + } catch (error) { + if (error instanceof Error) { + return { + content: [ + { + type: 'text', + text: `Error searching models: ${error.message}`, + }, + ], + isError: true, + }; + } + throw error; + } +} diff --git a/src/tool-handlers/validate-model.ts b/src/tool-handlers/validate-model.ts new file mode 100644 index 0000000..d6c3abd --- /dev/null +++ b/src/tool-handlers/validate-model.ts @@ -0,0 +1,50 @@ +import { ModelCache } from '../model-cache.js'; + +export interface ValidateModelToolRequest { + model: string; +} + +export async function handleValidateModel( + request: { params: { arguments: ValidateModelToolRequest } }, + modelCache: ModelCache +) { + const args = request.params.arguments; + + try { + if (!modelCache.isCacheValid()) { + return { + content: [ + { + type: 'text', + text: 'Model cache is empty or expired. Please call search_models first to populate the cache.', + }, + ], + isError: true, + }; + } + + const isValid = modelCache.hasModel(args.model); + + return { + content: [ + { + type: 'text', + text: JSON.stringify({ valid: isValid }), + }, + ], + }; + } catch (error) { + if (error instanceof Error) { + return { + content: [ + { + type: 'text', + text: `Error validating model: ${error.message}`, + }, + ], + isError: true, + }; + } + throw error; + } +}