From cf6032ea7b3c284e2030092783501f05b4107582 Mon Sep 17 00:00:00 2001 From: fiatjaf Date: Mon, 17 Jun 2024 15:53:27 -0300 Subject: [PATCH] porn detector. --- .gitignore | 1 + go.mod | 12 +++++++-- go.sum | 27 ++++++++++++++++--- image_utils.go | 25 +++++++----------- justfile | 2 +- nsfw_checker_impl.go | 63 ++++++++++++++++++++++++++++++++++++++++++++ nsfw_checker_noop.go | 7 +++++ render_event.go | 29 +++++++++++++------- utils.go | 3 ++- 9 files changed, 136 insertions(+), 33 deletions(-) create mode 100644 nsfw_checker_impl.go create mode 100644 nsfw_checker_noop.go diff --git a/.gitignore b/.gitignore index 4a942e4..8648910 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ static/tailwind-bundle.min.css yarn.lock package-lock.json *_templ.go +.models diff --git a/go.mod b/go.mod index 0fb4a7d..d5fb392 100644 --- a/go.mod +++ b/go.mod @@ -3,8 +3,10 @@ module github.com/fiatjaf/njump go 1.21.4 require ( + github.com/Kagami/go-avif v0.1.0 github.com/PuerkitoBio/goquery v1.8.1 github.com/a-h/templ v0.2.680 + github.com/ccuetoh/nsfw v0.0.0-20220803031759-d5b32f8737f5 github.com/dgraph-io/badger/v4 v4.2.0 github.com/fiatjaf/eventstore v0.4.2 github.com/fiatjaf/khatru v0.4.2 @@ -27,7 +29,7 @@ require ( github.com/stretchr/testify v1.8.4 github.com/texttheater/golang-levenshtein v1.0.1 github.com/tylermmorton/tmpl v0.0.0-20231025031313-5552ee818c6d - golang.org/x/image v0.14.0 + golang.org/x/image v0.17.0 mvdan.cc/xurls/v2 v2.5.0 ) @@ -46,6 +48,8 @@ require ( github.com/dustin/go-humanize v1.0.1 // indirect github.com/fasthttp/websocket v1.5.7 // indirect github.com/fiatjaf/generic-ristretto v0.0.1 // indirect + github.com/galeone/tensorflow/tensorflow/go v0.0.0-20220620094824-6bb01e3a58fa // indirect + github.com/galeone/tfgo v0.0.0-20220622151904-fc7b7ccad83b // indirect github.com/gobwas/httphead v0.1.0 // indirect github.com/gobwas/pool v0.2.1 // indirect github.com/gobwas/ws v1.3.1 // indirect @@ -58,16 +62,20 @@ require ( github.com/gorilla/css v1.0.0 // indirect github.com/graph-gophers/dataloader/v7 v7.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.17.3 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect + github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/puzpuzpuz/xsync/v3 v3.0.2 // indirect github.com/savsgio/gotils v0.0.0-20230208104028-c358bd845dee // indirect github.com/sebest/xff v0.0.0-20210106013422-671bd2870b3a // indirect github.com/shopspring/decimal v1.3.1 // indirect + github.com/sirupsen/logrus v1.9.0 // indirect github.com/tidwall/gjson v1.17.0 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect @@ -78,7 +86,7 @@ require ( golang.org/x/exp v0.0.0-20231226003508-02704c960a9b // indirect golang.org/x/net v0.24.0 // indirect golang.org/x/sys v0.19.0 // indirect - golang.org/x/text v0.14.0 // indirect + golang.org/x/text v0.16.0 // indirect google.golang.org/protobuf v1.31.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index d3d7141..c883b44 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/Kagami/go-avif v0.1.0 h1:8GHAGLxCdFfhpd4Zg8j1EqO7rtcQNenxIDerC/uu68w= +github.com/Kagami/go-avif v0.1.0/go.mod h1:OPmPqzNdQq3+sXm0HqaUJQ9W/4k+Elbc3RSfJUemDKA= github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM= github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ= github.com/a-h/templ v0.2.680 h1:TflYFucxp5rmOxAXB9Xy3+QHTk8s8xG9+nCT/cLzjeE= @@ -35,6 +37,8 @@ github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku github.com/btcsuite/snappy-go v1.0.0/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= +github.com/ccuetoh/nsfw v0.0.0-20220803031759-d5b32f8737f5 h1:sT6iHLm2czqbzLM7JUSYu1LsxeT9OcV7HmsDgqKZFXk= +github.com/ccuetoh/nsfw v0.0.0-20220803031759-d5b32f8737f5/go.mod h1:iiI5TnZU77lDEoWjRs56gSohBFnd4gKVjNT9dBKDvNg= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= @@ -82,6 +86,10 @@ github.com/fogleman/gg v1.3.0 h1:/7zJX8F6AaYQc57WQCyN9cAIz+4bCJGO9B+dyW29am8= github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/galeone/tensorflow/tensorflow/go v0.0.0-20220620094824-6bb01e3a58fa h1:UsqNDZ0Olkhk+9wFchvc1JCQn7Bq6e694uQ+hGBlJcM= +github.com/galeone/tensorflow/tensorflow/go v0.0.0-20220620094824-6bb01e3a58fa/go.mod h1:nHvVZJgJuQ0V2Xe4BqhTeCKQSMWDRI/gDkN8UxAANtU= +github.com/galeone/tfgo v0.0.0-20220622151904-fc7b7ccad83b h1:TGgxstITwmNG7+OI95FNQI5wsYkkOSiOE8xOi0QjccU= +github.com/galeone/tfgo v0.0.0-20220622151904-fc7b7ccad83b/go.mod h1:KqgpdfIYdJEusyCqP9uUcnwvSAp7m37ML5lh6YPlUh4= github.com/go-text/typesetting-utils v0.0.0-20231211103740-d9332ae51f04 h1:zBx+p/W2aQYtNuyZNcTfinWvXBQwYtDfme051PR/lAY= github.com/go-text/typesetting-utils v0.0.0-20231211103740-d9332ae51f04/go.mod h1:DDxDdQEnB70R8owOx3LVpEFvpMK9eeH1o2r0yZhFI9o= github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU= @@ -130,6 +138,7 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= @@ -141,6 +150,8 @@ github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJS github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= @@ -163,6 +174,10 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/microcosm-cc/bluemonday v1.0.24 h1:NGQoPtwGVcbGkKfvyYk1yRqknzBuoMiUrO6R7uFTPlw= github.com/microcosm-cc/bluemonday v1.0.24/go.mod h1:ArQySAMps0790cHSkdPEJ7bGkF2VePWH773hsJNSHf8= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/nbd-wtf/emoji v0.0.3 h1:YtkT7MVPXvqU1SQjvC/CShlWexnREzqNCxmhUnL00CA= github.com/nbd-wtf/emoji v0.0.3/go.mod h1:tS6D9iI34qwBmWc5g8X7tVDkWXulqbTJRsvsM6QsS88= github.com/nbd-wtf/go-nostr v0.32.0 h1:ShRerjhXvqZbiVUc11iPqxLuOImxqbJQ0zTz4t6Tjps= @@ -204,11 +219,14 @@ github.com/sebest/xff v0.0.0-20210106013422-671bd2870b3a h1:iLcLb5Fwwz7g/DLK89F+ github.com/sebest/xff v0.0.0-20210106013422-671bd2870b3a/go.mod h1:wozgYq9WEBQBaIJe4YZ0qTSFAMxmcwBhQH0fO0R34Z0= github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= +github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= +github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/srwiley/rasterx v0.0.0-20220730225603-2ab79fcdd4ef h1:Ch6Q+AZUxDBCVqdkI8FSpFyZDtCVBc2VmejdNrm5rRQ= github.com/srwiley/rasterx v0.0.0-20220730225603-2ab79fcdd4ef/go.mod h1:nXTWP6+gD5+LUJ8krVhhoeHjvHTutPxMYl5SvkcnJNE= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -247,8 +265,8 @@ golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+ golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20231226003508-02704c960a9b h1:kLiC65FbiHWFAOu+lxwNPujcsl8VYyTYYEZnsOO1WK4= golang.org/x/exp v0.0.0-20231226003508-02704c960a9b/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= -golang.org/x/image v0.14.0 h1:tNgSxAFe3jC4uYqvZdTr84SZoM1KfwdC9SKIFrLjFn4= -golang.org/x/image v0.14.0/go.mod h1:HUYqC05R2ZcZ3ejNQsIHQDQiwWM4JBqmm6MKANTp4LE= +golang.org/x/image v0.17.0 h1:nTRVVdajgB8zCMZVsViyzhnMKPwYeroEERRC64JuLco= +golang.org/x/image v0.17.0/go.mod h1:4yyo5vMFQjVjUcVk4jEQcU9MGy/rulF5WvUILseCM2E= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -298,6 +316,7 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -314,8 +333,8 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= diff --git a/image_utils.go b/image_utils.go index 34828b6..0d1c145 100644 --- a/image_utils.go +++ b/image_utils.go @@ -3,27 +3,26 @@ package main import ( "bytes" "context" + "fmt" "image" "image/color" "image/draw" _ "image/gif" + _ "image/jpeg" "image/png" + _ "image/png" "math" "net/http" "net/url" "os" "os/exec" + "slices" "strings" "sync" "time" "unicode" - _ "golang.org/x/image/webp" - - "github.com/nfnt/resize" - - "slices" - + _ "github.com/Kagami/go-avif" "github.com/fogleman/gg" "github.com/go-text/typesetting/di" "github.com/go-text/typesetting/font" @@ -32,9 +31,11 @@ import ( "github.com/go-text/typesetting/opentype/api" "github.com/go-text/typesetting/shaping" "github.com/nbd-wtf/emoji" + "github.com/nfnt/resize" "github.com/pemistahl/lingua-go" "github.com/srwiley/rasterx" "golang.org/x/image/math/fixed" + _ "golang.org/x/image/webp" ) const ( @@ -299,13 +300,13 @@ gotScriptIndex: func fetchImageFromURL(url string) (image.Image, error) { response, err := http.Get(url) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to fetch image from %s: %w", url, err) } defer response.Body.Close() img, _, err := image.Decode(response.Body) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to decode image from %s: %w", url, err) } return img, nil @@ -732,13 +733,7 @@ func drawShapedBlockAt( } func drawImageAt(img draw.Image, imageUrl string, startY int) int { - resp, err := http.Get(imageUrl) - if err != nil { - return -1 - } - defer resp.Body.Close() - - srcImg, _, err := image.Decode(resp.Body) + srcImg, err := fetchImageFromURL(imageUrl) if err != nil { return -1 } diff --git a/justfile b/justfile index 2592fcc..f05b3f1 100644 --- a/justfile +++ b/justfile @@ -1,7 +1,7 @@ export PATH := "./node_modules/.bin:" + env_var('PATH') dev: - fd --no-ignore-vcs 'go|templ|base.css' | entr -r bash -c 'TAILWIND_DEBUG=true SKIP_LANGUAGE_MODEL=true && templ generate && go build -o /tmp/njump && /tmp/njump' + fd --no-ignore-vcs 'go|templ|base.css' | entr -r bash -c 'TAILWIND_DEBUG=true SKIP_LANGUAGE_MODEL=true && templ generate && go build -tags=nsfw -o /tmp/njump && /tmp/njump' build: templ tailwind go build -o ./njump diff --git a/nsfw_checker_impl.go b/nsfw_checker_impl.go new file mode 100644 index 0000000..cb749f7 --- /dev/null +++ b/nsfw_checker_impl.go @@ -0,0 +1,63 @@ +//go:build nsfw + +package main + +import ( + "fmt" + "image/png" + "os" + "sync" + + "github.com/ccuetoh/nsfw" +) + +var nsfwPredictor = func() *nsfw.Predictor { + p, err := nsfw.NewLatestPredictor() + if err != nil { + log.Fatal().Err(err).Msg("failed to start keras nsfw detector") + return p + } + log.Info().Msg("keras nsfw detector enabled") + return p +}() + +var tempFileLocks = [3]sync.Mutex{{}, {}, {}} + +func isImageNSFW(url string) bool { + img, err := fetchImageFromURL(url) + if err != nil { + return false // if we can't read it that means it's ok + } + + // grab mutex + var tempPath string + for i, mu := range tempFileLocks { + if ok := mu.TryLock(); ok { + tempPath = fmt.Sprintf("/tmp/nsfw-detection-%d.png", i) + defer mu.Unlock() + break + } + } + + if tempPath == "" { + // apparently we can't allocate a temporary file for this, so let's warn and return false + log.Warn().Msg("failed to allocate a temp file for nsfw detection") + return false + } + + tempFile, err := os.Create(tempPath) + if err != nil { + log.Warn().Err(err).Msg("failed to open a temp file for nsfw detection") + return false + } + if err := png.Encode(tempFile, img); err != nil { + log.Warn().Err(err).Msg("failed to encode png for nsfw detection") + tempFile.Close() + return false + } + tempFile.Close() // close here so the thing can read it below + + res := nsfwPredictor.Predict(nsfwPredictor.NewImage(tempPath, 3)) + + return res.Porn > 0.75 || res.Hentai > 0.75 +} diff --git a/nsfw_checker_noop.go b/nsfw_checker_noop.go new file mode 100644 index 0000000..91c39f9 --- /dev/null +++ b/nsfw_checker_noop.go @@ -0,0 +1,7 @@ +//go:build !nsfw + +package main + +func isImageNSFW(url string) bool { + return false +} diff --git a/render_event.go b/render_event.go index 8491d83..f8ba257 100644 --- a/render_event.go +++ b/render_event.go @@ -9,7 +9,6 @@ import ( "html/template" "net/http" "net/url" - "regexp" "strings" "time" @@ -111,6 +110,17 @@ func renderEvent(w http.ResponseWriter, r *http.Request) { // from here onwards we know we're rendering an event // + // if it's porn we return a 404 + for _, url := range urlRegex.FindAllString(data.event.Content, len(data.event.Content)+1) { + if imageExtensionMatcher.MatchString(url) { + if isImageNSFW(url) { + log.Warn().Str("url", url).Str("event", data.nevent).Msg("detect nsfw image") + http.Error(w, "event is unsuitable: "+err.Error(), 404) + return + } + } + } + // gather page style from user-agent style := getPreviewStyle(r) @@ -217,19 +227,18 @@ func renderEvent(w http.ResponseWriter, r *http.Request) { } // titleizedContent - titleizedContent := strings.TrimSpace( - strings.Replace( + titleizedContent := urlRegex.ReplaceAllString( + strings.TrimSpace( strings.Replace( - replaceUserReferencesWithNames(r.Context(), []string{data.event.Content}, "", "")[0], - "\r\n", " ", -1), - "\n", " ", -1, + strings.Replace( + replaceUserReferencesWithNames(r.Context(), []string{data.event.Content}, "", "")[0], + "\r\n", " ", -1), + "\n", " ", -1, + ), ), + "", ) - // Remove image/video urls - urlRegex := regexp.MustCompile(`(https?)://[^\s/$.?#]+\.(?i:jpg|jpeg|png|gif|bmp|mp4|mov|avi|mkv|webm|ogg)`) - titleizedContent = urlRegex.ReplaceAllString(titleizedContent, "") - if titleizedContent == "" { titleizedContent = subscript } diff --git a/utils.go b/utils.go index 0b255ec..d98a66b 100644 --- a/utils.go +++ b/utils.go @@ -37,8 +37,9 @@ var ( xurls.SchemesUnofficial = []string{"http"} return xurls.Strict() }() - imageExtensionMatcher = regexp.MustCompile(`.*\.(png|jpg|jpeg|gif|webp)((\?|\#).*)?$`) + imageExtensionMatcher = regexp.MustCompile(`.*\.(png|jpg|jpeg|gif|webp|avif)((\?|\#).*)?$`) videoExtensionMatcher = regexp.MustCompile(`.*\.(mp4|ogg|webm|mov)((\?|\#).*)?$`) + urlRegex = xurls.Strict() ) var kindNames = map[int]string{