From 6189c2f8d069908d459ef084aa6129f91cb4e6ee Mon Sep 17 00:00:00 2001 From: Aljaz Date: Sun, 22 Jun 2025 15:09:15 +0200 Subject: [PATCH] Add free mode with automatic free model selection --- failure_store.go | 45 +++++++++++++++ free_models.go | 87 ++++++++++++++++++++++++++++ go.mod | 1 + go.sum | 32 +---------- main.go | 144 ++++++++++++++++++++++++++++++++++++++--------- 5 files changed, 252 insertions(+), 57 deletions(-) create mode 100644 failure_store.go create mode 100644 free_models.go diff --git a/failure_store.go b/failure_store.go new file mode 100644 index 0000000..8e51743 --- /dev/null +++ b/failure_store.go @@ -0,0 +1,45 @@ +package main + +import ( + "database/sql" + _ "github.com/mattn/go-sqlite3" + "time" +) + +type FailureStore struct { + db *sql.DB +} + +func NewFailureStore(path string) (*FailureStore, error) { + db, err := sql.Open("sqlite3", path) + if err != nil { + return nil, err + } + if _, err = db.Exec(`CREATE TABLE IF NOT EXISTS failures (model TEXT PRIMARY KEY, failed_at INTEGER)`); err != nil { + db.Close() + return nil, err + } + return &FailureStore{db: db}, nil +} + +func (s *FailureStore) Close() error { return s.db.Close() } + +func (s *FailureStore) MarkFailure(model string) error { + _, err := s.db.Exec(`INSERT INTO failures(model, failed_at) VALUES(?, ?) ON CONFLICT(model) DO UPDATE SET failed_at=excluded.failed_at`, model, time.Now().Unix()) + return err +} + +func (s *FailureStore) ShouldSkip(model string) (bool, error) { + var ts int64 + err := s.db.QueryRow(`SELECT failed_at FROM failures WHERE model=?`, model).Scan(&ts) + if err == sql.ErrNoRows { + return false, nil + } + if err != nil { + return false, err + } + if time.Since(time.Unix(ts, 0)) < 15*time.Minute { + return true, nil + } + return false, nil +} diff --git a/free_models.go b/free_models.go new file mode 100644 index 0000000..3804fd4 --- /dev/null +++ b/free_models.go @@ -0,0 +1,87 @@ +package main + +import ( + "encoding/json" + "fmt" + "net/http" + "os" + "sort" + "strings" +) + +type orModels struct { + Data []struct { + ID string `json:"id"` + ContextLength int `json:"context_length"` + TopProvider struct { + ContextLength int `json:"context_length"` + } `json:"top_provider"` + Pricing struct { + Prompt string `json:"prompt"` + Completion string `json:"completion"` + } `json:"pricing"` + } `json:"data"` +} + +func fetchFreeModels(apiKey string) ([]string, error) { + req, err := http.NewRequest("GET", "https://openrouter.ai/api/v1/models", nil) + if err != nil { + return nil, err + } + req.Header.Set("Authorization", "Bearer "+apiKey) + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("unexpected status: %s", resp.Status) + } + var result orModels + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, err + } + type item struct { + id string + ctx int + } + var items []item + for _, m := range result.Data { + if m.Pricing.Prompt == "0" && m.Pricing.Completion == "0" { + ctx := m.TopProvider.ContextLength + if ctx == 0 { + ctx = m.ContextLength + } + items = append(items, item{id: m.ID, ctx: ctx}) + } + } + sort.Slice(items, func(i, j int) bool { return items[i].ctx > items[j].ctx }) + models := make([]string, len(items)) + for i, it := range items { + models[i] = it.id + } + return models, nil +} + +func ensureFreeModelFile(apiKey, path string) ([]string, error) { + if _, err := os.Stat(path); err == nil { + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var models []string + for _, line := range strings.Split(string(data), "\n") { + line = strings.TrimSpace(line) + if line != "" { + models = append(models, line) + } + } + return models, nil + } + models, err := fetchFreeModels(apiKey) + if err != nil { + return nil, err + } + _ = os.WriteFile(path, []byte(strings.Join(models, "\n")), 0644) + return models, nil +} diff --git a/go.mod b/go.mod index 106338b..27df0da 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.23 require ( github.com/gin-gonic/gin v1.10.0 + github.com/mattn/go-sqlite3 v1.14.18 github.com/sashabaranov/go-openai v1.36.0 ) diff --git a/go.sum b/go.sum index 7e50890..f77b988 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,5 @@ -github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc0= -github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4= github.com/bytedance/sonic v1.12.6 h1:/isNmCUF2x3Sh8RAp/4mh4ZGkcFAX/hLrzrK3AvpRzk= github.com/bytedance/sonic v1.12.6/go.mod h1:B8Gt/XvtZ3Fqj+iSKMypzymZxw/FVwgIGKzMzT9r/rk= -github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM= github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= github.com/bytedance/sonic/loader v0.2.1 h1:1GgorWTqf12TA8mma4DDSbaQigE2wOgQo7iCjjJv3+E= github.com/bytedance/sonic/loader v0.2.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= @@ -13,8 +10,6 @@ github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= -github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= github.com/gabriel-vasile/mimetype v1.4.7 h1:SKFKl7kD0RiPdbht0s7hFtjl489WcQ1VyPW8ZzUMYCA= github.com/gabriel-vasile/mimetype v1.4.7/go.mod h1:GDlAgAyIRT27BhFl53XNAFtfjzOkLaF35JdEG0P7LtU= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= @@ -27,12 +22,8 @@ github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/o github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= -github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBExVwjEviJTixqxL8= -github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= github.com/go-playground/validator/v10 v10.23.0 h1:/PwmTwZhS0dPkav3cdK9kV1FsAmrL8sThn8IHr/sO+o= github.com/go-playground/validator/v10 v10.23.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= -github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= -github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/goccy/go-json v0.10.4 h1:JSwxQzIqKfmFX1swYPpUThQZp/Ka4wzJdK0LWVytLPM= github.com/goccy/go-json v0.10.4/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= @@ -41,8 +32,6 @@ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= -github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/klauspost/cpuid/v2 v2.2.9 h1:66ze0taIn2H33fBvCkXuv9BmCwDfafmiIVpKV9kKGuY= github.com/klauspost/cpuid/v2 v2.2.9/go.mod h1:rqkxqrZ1EhYM9G+hXH7YdowN5R5RGN6NK4QwQ3WMXF8= github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= @@ -50,13 +39,13 @@ github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-sqlite3 v1.14.18 h1:JL0eqdCOq6DJVNPSvArO/bIV9/P7fbGrV00LZHc+5aI= +github.com/mattn/go-sqlite3 v1.14.18/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= -github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -66,46 +55,30 @@ github.com/sashabaranov/go-openai v1.36.0/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adO github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= -golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= -golang.org/x/arch v0.8.0 h1:3wRIsP3pM4yUptoR96otTUOXI367OS0+c9eeRi9doIc= -golang.org/x/arch v0.8.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= golang.org/x/arch v0.12.0 h1:UsYJhbzPYGsT0HbEdmYcqtCv8UNGvnaL561NnIUvaKg= golang.org/x/arch v0.12.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= -golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= -golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= -golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= -golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= -golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= -golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= -google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= google.golang.org/protobuf v1.36.1 h1:yBPeRvTftaleIgM3PZ/WBIZ7XM/eEYAaEyCwvyjq/gk= google.golang.org/protobuf v1.36.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= @@ -114,4 +87,3 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= -rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= diff --git a/main.go b/main.go index 4545384..f3b66c2 100644 --- a/main.go +++ b/main.go @@ -17,6 +17,9 @@ import ( ) var modelFilter map[string]struct{} +var freeModels []string +var failureStore *FailureStore +var freeMode bool func loadModelFilter(path string) (map[string]struct{}, error) { file, err := os.Open(path) @@ -55,6 +58,24 @@ func main() { } } + freeMode = strings.ToLower(os.Getenv("FREE_MODE")) == "true" + + if freeMode { + var err error + freeModels, err = ensureFreeModelFile(apiKey, "free-models") + if err != nil { + slog.Error("failed to load free models", "error", err) + return + } + failureStore, err = NewFailureStore("failures.db") + if err != nil { + slog.Error("failed to init failure store", "error", err) + return + } + defer failureStore.Close() + slog.Info("Free mode enabled", "models", len(freeModels)) + } + provider := NewOpenrouterProvider(apiKey) filter, err := loadModelFilter("models-filter") @@ -158,21 +179,30 @@ func main() { // для сбора полного ответа и отправки его одним JSON. // Пока реализуем только стриминг. if !streamRequested { - // Handle non-streaming response - fullModelName, err := provider.GetFullModelName(request.Model) - if err != nil { - slog.Error("Error getting full model name", "Error", err) - // Ollama returns 404 for invalid model names - c.JSON(http.StatusNotFound, gin.H{"error": err.Error()}) - return - } - - // Call Chat to get the complete response - response, err := provider.Chat(request.Messages, fullModelName) - if err != nil { - slog.Error("Failed to get chat response", "Error", err) - c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) - return + var response openai.ChatCompletionResponse + var fullModelName string + var err error + if freeMode { + response, fullModelName, err = getFreeChat(provider, request.Messages) + if err != nil { + slog.Error("free mode failed", "error", err) + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + } else { + fullModelName, err = provider.GetFullModelName(request.Model) + if err != nil { + slog.Error("Error getting full model name", "Error", err) + // Ollama returns 404 for invalid model names + c.JSON(http.StatusNotFound, gin.H{"error": err.Error()}) + return + } + response, err = provider.Chat(request.Messages, fullModelName) + if err != nil { + slog.Error("Failed to get chat response", "Error", err) + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } } // Format the response according to Ollama's format @@ -195,8 +225,8 @@ func main() { // Create Ollama-compatible response ollamaResponse := map[string]interface{}{ - "model": fullModelName, - "created_at": time.Now().Format(time.RFC3339), + "model": fullModelName, + "created_at": time.Now().Format(time.RFC3339), "message": map[string]string{ "role": "assistant", "content": content, @@ -210,22 +240,39 @@ func main() { "eval_duration": response.Usage.CompletionTokens * 10, // Approximate duration based on token count } + slog.Info("Used model", "model", fullModelName) + c.JSON(http.StatusOK, ollamaResponse) return } slog.Info("Requested model", "model", request.Model) - fullModelName, err := provider.GetFullModelName(request.Model) - if err != nil { - slog.Error("Error getting full model name", "Error", err, "model", request.Model) - // Ollama возвращает 404 на неправильное имя модели - c.JSON(http.StatusNotFound, gin.H{"error": err.Error()}) - return + var stream *openai.ChatCompletionStream + var fullModelName string + var err error + if freeMode { + stream, fullModelName, err = getFreeStream(provider, request.Messages) + if err != nil { + slog.Error("free mode failed", "error", err) + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + } else { + fullModelName, err = provider.GetFullModelName(request.Model) + if err != nil { + slog.Error("Error getting full model name", "Error", err, "model", request.Model) + c.JSON(http.StatusNotFound, gin.H{"error": err.Error()}) + return + } + stream, err = provider.ChatStream(request.Messages, fullModelName) + if err != nil { + slog.Error("Failed to create stream", "Error", err) + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } } slog.Info("Using model", "fullModelName", fullModelName) - // Call ChatStream to get the stream - stream, err := provider.ChatStream(request.Messages, fullModelName) if err != nil { slog.Error("Failed to create stream", "Error", err) c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) @@ -309,8 +356,8 @@ func main() { // ВАЖНО: Замените nil на 0 для числовых полей статистики finalResponse := map[string]interface{}{ - "model": fullModelName, - "created_at": time.Now().Format(time.RFC3339), + "model": fullModelName, + "created_at": time.Now().Format(time.RFC3339), "message": map[string]string{ "role": "assistant", "content": "", // Пустой контент для финального сообщения @@ -343,3 +390,46 @@ func main() { r.Run(":11434") } + +func getFreeChat(provider *OpenrouterProvider, msgs []openai.ChatCompletionMessage) (openai.ChatCompletionResponse, string, error) { + var resp openai.ChatCompletionResponse + for _, m := range freeModels { + skip, err := failureStore.ShouldSkip(m) + if err != nil { + slog.Error("db error", "error", err) + continue + } + if skip { + continue + } + resp, err = provider.Chat(msgs, m) + if err != nil { + slog.Warn("model failed", "model", m, "error", err) + _ = failureStore.MarkFailure(m) + continue + } + return resp, m, nil + } + return resp, "", fmt.Errorf("no free models available") +} + +func getFreeStream(provider *OpenrouterProvider, msgs []openai.ChatCompletionMessage) (*openai.ChatCompletionStream, string, error) { + for _, m := range freeModels { + skip, err := failureStore.ShouldSkip(m) + if err != nil { + slog.Error("db error", "error", err) + continue + } + if skip { + continue + } + stream, err := provider.ChatStream(msgs, m) + if err != nil { + slog.Warn("model failed", "model", m, "error", err) + _ = failureStore.MarkFailure(m) + continue + } + return stream, m, nil + } + return nil, "", fmt.Errorf("no free models available") +}