diff --git a/README.md b/README.md index 4f457f2..c4a4089 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,7 @@ This is heavily vibecoded and may not be production-ready. It is intended for pe ## Features - **Free Mode (Default)**: Automatically selects and uses free models from OpenRouter with intelligent fallback. Enabled by default unless `FREE_MODE=false` is set. - **Model Filtering**: Create a `models-filter/filter` file with model name patterns (one per line). Supports partial matching - `gemini` matches `gemini-2.0-flash-exp:free`. Works in both free and non-free modes. +- **Tool Use Filtering**: Filter for only free models that support function calling/tool use by setting `TOOL_USE_ONLY=true`. Models are filtered based on their `supported_parameters` containing "tools" or "tool_choice". - **Ollama-like API**: The server listens on `11434` and exposes endpoints similar to Ollama (e.g., `/api/chat`, `/api/tags`). - **Model Listing**: Fetch a list of available models from OpenRouter. - **Model Details**: Retrieve metadata about a specific model. @@ -34,6 +35,11 @@ The proxy operates in **free mode** by default, automatically selecting from ava export OPENAI_API_KEY="your-openrouter-api-key" ./ollama-proxy + # To only use free models that support tool use/function calling + export TOOL_USE_ONLY=true + export OPENAI_API_KEY="your-openrouter-api-key" + ./ollama-proxy + #### How Free Mode Works - **Automatic Model Discovery**: Fetches and caches available free models from OpenRouter @@ -128,6 +134,7 @@ curl -X POST http://localhost:11434/v1/chat/completions \ ```bash OPENAI_API_KEY=your-openrouter-api-key FREE_MODE=true + TOOL_USE_ONLY=false ``` @@ -136,7 +143,11 @@ curl -X POST http://localhost:11434/v1/chat/completions \ mkdir -p models-filter echo "gemini" > models-filter/filter # Only show Gemini models ``` -4. **Run with Docker Compose**: + +4. **Optional: Enable tool use filtering**: + Set `TOOL_USE_ONLY=true` in your `.env` file to only use models that support function calling/tool use. This filters models based on their `supported_parameters` containing "tools" or "tool_choice". + +5. **Run with Docker Compose**: ```bash docker compose up -d ``` @@ -148,6 +159,9 @@ The service will be available at `http://localhost:11434`. ```bash docker build -t ollama-proxy . docker run -p 11434:11434 -e OPENAI_API_KEY="your-openrouter-api-key" ollama-proxy + +# To enable tool use filtering +docker run -p 11434:11434 -e OPENAI_API_KEY="your-openrouter-api-key" -e TOOL_USE_ONLY=true ollama-proxy ``` diff --git a/docker-compose.yml b/docker-compose.yml index 8e17ca0..ed1b381 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,15 +1,14 @@ -version: '3.8' - services: ollama-proxy: build: . ports: - - "11434:11434" + - "21434:11434" env_file: - .env environment: - OPENAI_API_KEY=${OPENAI_API_KEY} - FREE_MODE=${FREE_MODE:-true} + - TOOL_USE_ONLY=${TOOL_USE_ONLY:-false} volumes: - ./models-filter:/models-filter:ro - proxy-data:/data diff --git a/free_models.go b/free_models.go index b7c14b4..d1fdb5d 100644 --- a/free_models.go +++ b/free_models.go @@ -12,9 +12,10 @@ import ( type orModels struct { Data []struct { - ID string `json:"id"` - ContextLength int `json:"context_length"` - TopProvider struct { + ID string `json:"id"` + ContextLength int `json:"context_length"` + SupportedParameters []string `json:"supported_parameters"` + TopProvider struct { ContextLength int `json:"context_length"` } `json:"top_provider"` Pricing struct { @@ -24,6 +25,16 @@ type orModels struct { } `json:"data"` } +// supportsToolUse checks if a model supports tool use by looking for "tools" in supported_parameters +func supportsToolUse(supportedParams []string) bool { + for _, param := range supportedParams { + if param == "tools" || param == "tool_choice" { + return true + } + } + return false +} + func fetchFreeModels(apiKey string) ([]string, error) { req, err := http.NewRequest("GET", "https://openrouter.ai/api/v1/models", nil) if err != nil { @@ -42,6 +53,10 @@ func fetchFreeModels(apiKey string) ([]string, error) { if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { return nil, err } + + // Check if tool use filtering is enabled + toolUseOnly := strings.ToLower(os.Getenv("TOOL_USE_ONLY")) == "true" + type item struct { id string ctx int @@ -49,6 +64,11 @@ func fetchFreeModels(apiKey string) ([]string, error) { var items []item for _, m := range result.Data { if m.Pricing.Prompt == "0" && m.Pricing.Completion == "0" { + // If tool use filtering is enabled, skip models that don't support tools + if toolUseOnly && !supportsToolUse(m.SupportedParameters) { + continue + } + ctx := m.TopProvider.ContextLength if ctx == 0 { ctx = m.ContextLength diff --git a/main.go b/main.go index 85bfaca..312623d 100644 --- a/main.go +++ b/main.go @@ -100,6 +100,9 @@ func main() { r.GET("/api/tags", func(c *gin.Context) { var newModels []map[string]interface{} + + // Check if tool use filtering is enabled + toolUseOnly := strings.ToLower(os.Getenv("TOOL_USE_ONLY")) == "true" if freeMode { // In free mode, show only available free models @@ -142,29 +145,96 @@ func main() { } } else { // Non-free mode: use original logic - models, err := provider.GetModels() - if err != nil { - slog.Error("Error getting models", "Error", err) - c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) - return - } - filter := modelFilter - newModels = make([]map[string]interface{}, 0, len(models)) - for _, m := range models { - // Если фильтр пустой, значит пропускаем проверку и берём все модели - if len(filter) > 0 { - if _, ok := filter[m.Model]; !ok { - continue - } + if toolUseOnly { + // If tool use filtering is enabled, we need to fetch full model details from OpenRouter + req, err := http.NewRequest("GET", "https://openrouter.ai/api/v1/models", nil) + if err != nil { + slog.Error("Error creating request for models", "Error", err) + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + req.Header.Set("Authorization", "Bearer "+os.Getenv("OPENAI_API_KEY")) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + slog.Error("Error fetching models from OpenRouter", "Error", err) + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + slog.Error("Unexpected status from OpenRouter", "status", resp.Status) + c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to fetch models"}) + return + } + + var result orModels + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + slog.Error("Error decoding models response", "Error", err) + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + + // Filter models based on tool use support and model filter + currentTime := time.Now().Format(time.RFC3339) + newModels = make([]map[string]interface{}, 0, len(result.Data)) + for _, m := range result.Data { + if !supportsToolUse(m.SupportedParameters) { + continue // Skip models that don't support tool use + } + + // Extract display name from full model name + parts := strings.Split(m.ID, "/") + displayName := parts[len(parts)-1] + + // Apply model filter if it exists + if !isModelInFilter(displayName, modelFilter) { + continue // Skip models not in filter + } + + newModels = append(newModels, map[string]interface{}{ + "name": displayName, + "model": displayName, + "modified_at": currentTime, + "size": 270898672, + "digest": "9077fe9d2ae1a4a41a868836b56b8163731a8fe16621397028c2c76f838c6907", + "details": map[string]interface{}{ + "parent_model": "", + "format": "gguf", + "family": "tool-enabled", + "families": []string{"tool-enabled"}, + "parameter_size": "varies", + "quantization_level": "Q4_K_M", + }, + }) + } + } else { + // Standard non-free mode: get all models from provider + models, err := provider.GetModels() + if err != nil { + slog.Error("Error getting models", "Error", err) + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + filter := modelFilter + newModels = make([]map[string]interface{}, 0, len(models)) + for _, m := range models { + // Если фильтр пустой, значит пропускаем проверку и берём все модели + if len(filter) > 0 { + if _, ok := filter[m.Model]; !ok { + continue + } + } + newModels = append(newModels, map[string]interface{}{ + "name": m.Name, + "model": m.Model, + "modified_at": m.ModifiedAt, + "size": 270898672, + "digest": "9077fe9d2ae1a4a41a868836b56b8163731a8fe16621397028c2c76f838c6907", + "details": m.Details, + }) } - newModels = append(newModels, map[string]interface{}{ - "name": m.Name, - "model": m.Model, - "modified_at": m.ModifiedAt, - "size": 270898672, - "digest": "9077fe9d2ae1a4a41a868836b56b8163731a8fe16621397028c2c76f838c6907", - "details": m.Details, - }) } } @@ -564,6 +634,9 @@ func main() { // Add OpenAI-compatible models endpoint r.GET("/v1/models", func(c *gin.Context) { var models []gin.H + + // Check if tool use filtering is enabled + toolUseOnly := strings.ToLower(os.Getenv("TOOL_USE_ONLY")) == "true" if freeMode { // In free mode, show only available free models @@ -603,25 +676,81 @@ func main() { } } else { // Non-free mode: get all models from provider - providerModels, err := provider.GetModels() - if err != nil { - slog.Error("Error getting models", "Error", err) - c.JSON(http.StatusInternalServerError, gin.H{"error": gin.H{"message": err.Error()}}) - return - } - - for _, m := range providerModels { - if len(modelFilter) > 0 { - if _, ok := modelFilter[m.Model]; !ok { - continue - } + if toolUseOnly { + // If tool use filtering is enabled, we need to fetch full model details from OpenRouter + req, err := http.NewRequest("GET", "https://openrouter.ai/api/v1/models", nil) + if err != nil { + slog.Error("Error creating request for models", "Error", err) + c.JSON(http.StatusInternalServerError, gin.H{"error": gin.H{"message": err.Error()}}) + return + } + req.Header.Set("Authorization", "Bearer "+os.Getenv("OPENAI_API_KEY")) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + slog.Error("Error fetching models from OpenRouter", "Error", err) + c.JSON(http.StatusInternalServerError, gin.H{"error": gin.H{"message": err.Error()}}) + return + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + slog.Error("Unexpected status from OpenRouter", "status", resp.Status) + c.JSON(http.StatusInternalServerError, gin.H{"error": gin.H{"message": "Failed to fetch models"}}) + return + } + + var result orModels + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + slog.Error("Error decoding models response", "Error", err) + c.JSON(http.StatusInternalServerError, gin.H{"error": gin.H{"message": err.Error()}}) + return + } + + // Filter models based on tool use support and model filter + for _, m := range result.Data { + if !supportsToolUse(m.SupportedParameters) { + continue // Skip models that don't support tool use + } + + // Extract display name from full model name + parts := strings.Split(m.ID, "/") + displayName := parts[len(parts)-1] + + // Apply model filter if it exists + if !isModelInFilter(displayName, modelFilter) { + continue // Skip models not in filter + } + + models = append(models, gin.H{ + "id": displayName, + "object": "model", + "created": time.Now().Unix(), + "owned_by": "openrouter", + }) + } + } else { + // Standard non-free mode: get all models from provider + providerModels, err := provider.GetModels() + if err != nil { + slog.Error("Error getting models", "Error", err) + c.JSON(http.StatusInternalServerError, gin.H{"error": gin.H{"message": err.Error()}}) + return + } + + for _, m := range providerModels { + if len(modelFilter) > 0 { + if _, ok := modelFilter[m.Model]; !ok { + continue + } + } + models = append(models, gin.H{ + "id": m.Model, + "object": "model", + "created": time.Now().Unix(), + "owned_by": "openrouter", + }) } - models = append(models, gin.H{ - "id": m.Model, - "object": "model", - "created": time.Now().Unix(), - "owned_by": "openrouter", - }) } }