From d13000b2b9768449e9d0c09633e8b1d193b64018 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 13 Oct 2026 21:39:43 +0400 Subject: [PATCH 01/51] RichMedia: skip the quote fallback link when picking the card URL Mastodon, Misskey and InlineQuotePolicy put "RE: " in a .quote-inline element at the start of a quote post. Its link was taken for the link card, which then showed the quoted post a second time. --- changelog.d/card-skip-quote-link.fix | 1 + lib/pleroma/html.ex | 8 +++--- test/pleroma/html_test.exs | 37 ++++++++++++++++++++++++++++ 3 files changed, 43 insertions(+), 3 deletions(-) create mode 100644 changelog.d/card-skip-quote-link.fix diff --git a/changelog.d/card-skip-quote-link.fix b/changelog.d/card-skip-quote-link.fix new file mode 100644 index 000000000..27fe096fa --- /dev/null +++ b/changelog.d/card-skip-quote-link.fix @@ -0,0 +1 @@ +Link preview cards skip the fallback link of quote posts (`RE: `), which pointed at the quoted post and showed it twice. diff --git a/lib/pleroma/html.ex b/lib/pleroma/html.ex index 4de7cbb76..aeac8bd04 100644 --- a/lib/pleroma/html.ex +++ b/lib/pleroma/html.ex @@ -66,14 +66,16 @@ defmodule Pleroma.HTML do end @spec extract_first_external_url_from_object(Pleroma.Object.t()) :: String.t() | nil - def extract_first_external_url_from_object(%{data: %{"content" => content}}) + def extract_first_external_url_from_object(%{data: %{"content" => content} = data}) when is_binary(content) do + # A quote post's fallback link ("RE: ", in a .quote-inline element or + # bare) points at the quoted post, which is shown already. content |> Floki.parse_fragment!() + |> Floki.filter_out(".quote-inline") |> Floki.find("a:not(.mention,.hashtag,.attachment,[rel~=\"tag\"])") - |> Enum.take(1) |> Floki.attribute("href") - |> Enum.at(0) + |> Enum.find(&(&1 != data["quoteUrl"])) end def extract_first_external_url_from_object(_), do: nil diff --git a/test/pleroma/html_test.exs b/test/pleroma/html_test.exs index d17b07540..5e7ad4749 100644 --- a/test/pleroma/html_test.exs +++ b/test/pleroma/html_test.exs @@ -299,5 +299,42 @@ defmodule Pleroma.HTMLTest do assert nil == HTML.extract_first_external_url_from_object(object) end + + test "skips the link in a Mastodon quote fallback paragraph" do + object = %Object{ + data: %{ + "quoteUrl" => "https://raru.re/users/ocean/statuses/116235427788617987", + "content" => + "

RE: raru.re/@ocean/116235427788617987

" <> + "

Berapa lapis?

youtube.com/watch?v=S0dJMhWDRDE

" + } + } + + assert HTML.extract_first_external_url_from_object(object) == + "https://www.youtube.com/watch?v=S0dJMhWDRDE" + end + + test "skips the link in a quote fallback span (Misskey, InlineQuotePolicy)" do + object = %Object{ + data: %{ + "content" => + "hello

RE: https://misskey.example/notes/abc" + } + } + + assert nil == HTML.extract_first_external_url_from_object(object) + end + + test "skips a link to the quoted post without fallback markup" do + object = %Object{ + data: %{ + "quoteUrl" => "https://remote.example/objects/1", + "content" => + "RE: https://remote.example/objects/1 cofe" + } + } + + assert HTML.extract_first_external_url_from_object(object) == "https://cofe.example/" + end end end From 1c8c6048ded10e8db600989969e222f8a882599d Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sat, 24 Oct 2026 14:12:47 +0400 Subject: [PATCH 02/51] fix(http): treat an empty proxy_url as no proxy --- changelog.d/empty-proxy-url.fix | 1 + lib/pleroma/http/adapter_helper.ex | 5 +++-- test/pleroma/http/adapter_helper/hackney_test.exs | 7 +++++++ test/pleroma/http/adapter_helper_test.exs | 6 ++++++ 4 files changed, 17 insertions(+), 2 deletions(-) create mode 100644 changelog.d/empty-proxy-url.fix diff --git a/changelog.d/empty-proxy-url.fix b/changelog.d/empty-proxy-url.fix new file mode 100644 index 000000000..4dd547aba --- /dev/null +++ b/changelog.d/empty-proxy-url.fix @@ -0,0 +1 @@ +HTTP: treat an empty proxy_url as no proxy; with Hackney 4 it made every outgoing request fail diff --git a/lib/pleroma/http/adapter_helper.ex b/lib/pleroma/http/adapter_helper.ex index 4f0e23f20..fa1d1ab54 100644 --- a/lib/pleroma/http/adapter_helper.ex +++ b/lib/pleroma/http/adapter_helper.ex @@ -20,8 +20,9 @@ defmodule Pleroma.HTTP.AdapterHelper do @callback options(keyword(), URI.t()) :: keyword() + # AdminFE stores an empty proxy field as "", which means no proxy. @spec format_proxy(String.t() | tuple() | nil) :: proxy() | nil - def format_proxy(nil), do: nil + def format_proxy(proxy) when proxy in [nil, ""], do: nil def format_proxy(proxy_url) do case parse_proxy(proxy_url) do @@ -32,7 +33,7 @@ defmodule Pleroma.HTTP.AdapterHelper do end @spec maybe_add_proxy(keyword(), proxy() | nil) :: keyword() - def maybe_add_proxy(opts, nil), do: opts + def maybe_add_proxy(opts, proxy) when proxy in [nil, ""], do: opts def maybe_add_proxy(opts, proxy), do: Keyword.put_new(opts, :proxy, proxy) @doc """ diff --git a/test/pleroma/http/adapter_helper/hackney_test.exs b/test/pleroma/http/adapter_helper/hackney_test.exs index 4a31ffb07..ecfd2323d 100644 --- a/test/pleroma/http/adapter_helper/hackney_test.exs +++ b/test/pleroma/http/adapter_helper/hackney_test.exs @@ -58,6 +58,13 @@ defmodule Pleroma.HTTP.AdapterHelper.HackneyTest do {:socks5, ~c"::1", 9050} end + # AdminFE stores an empty field as "", and Hackney 4 fails every request with it. + test "ignores an empty proxy_url", %{uri: uri} do + clear_config([:http, :proxy_url], "") + + refute Keyword.has_key?(Hackney.options([], uri), :proxy) + end + test "respect connection opts and no proxy", %{uri: uri} do opts = Hackney.options([a: 2, b: 1], uri) diff --git a/test/pleroma/http/adapter_helper_test.exs b/test/pleroma/http/adapter_helper_test.exs index cc0e24167..e75710d0d 100644 --- a/test/pleroma/http/adapter_helper_test.exs +++ b/test/pleroma/http/adapter_helper_test.exs @@ -12,6 +12,12 @@ defmodule Pleroma.HTTP.AdapterHelperTest do assert AdapterHelper.format_proxy(nil) == nil end + test "with an empty string" do + assert ExUnit.CaptureLog.capture_log(fn -> + assert AdapterHelper.format_proxy("") == nil + end) == "" + end + test "with string" do assert AdapterHelper.format_proxy("127.0.0.1:8123") == {{127, 0, 0, 1}, 8123} end From 51800008ebdcb026d38a9b9ceb1bcbc7788f4942 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 27 Oct 2026 17:27:05 +0400 Subject: [PATCH 03/51] build: add exif_stripper dependency --- mix.exs | 1 + mix.lock | 1 + 2 files changed, 2 insertions(+) diff --git a/mix.exs b/mix.exs index e2af5fda6..fc2d70b48 100644 --- a/mix.exs +++ b/mix.exs @@ -205,6 +205,7 @@ defmodule Pleroma.Mixfile do {:majic, "~> 1.2"}, {:open_api_spex, "~> 3.22"}, {:ecto_psql_extras, "~> 0.8"}, + {:exif_stripper, "~> 0.1.1"}, {:vix, "~> 0.36"}, {:elixir_make, "~> 0.7.8", override: true}, {:blurhash, "~> 0.1.0", hex: :rinpatch_blurhash}, diff --git a/mix.lock b/mix.lock index f8ce53f4c..aaa944985 100644 --- a/mix.lock +++ b/mix.lock @@ -47,6 +47,7 @@ "ex_doc": {:hex, :ex_doc, "0.38.2", "504d25eef296b4dec3b8e33e810bc8b5344d565998cd83914ffe1b8503737c02", [:mix], [{:earmark_parser, "~> 1.4.44", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "732f2d972e42c116a70802f9898c51b54916e542cc50968ac6980512ec90f42b"}, "ex_machina": {:hex, :ex_machina, "2.8.0", "a0e847b5712065055ec3255840e2c78ef9366634d62390839d4880483be38abe", [:mix], [{:ecto, "~> 2.2 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:ecto_sql, "~> 3.0", [hex: :ecto_sql, repo: "hexpm", optional: true]}], "hexpm", "79fe1a9c64c0c1c1fab6c4fa5d871682cb90de5885320c187d117004627a7729"}, "ex_syslogger": {:hex, :ex_syslogger, "1.5.2", "72b6aa2d47a236e999171f2e1ec18698740f40af0bd02c8c650bf5f1fd1bac79", [:mix], [{:poison, ">= 1.5.0", [hex: :poison, repo: "hexpm", optional: true]}, {:syslog, "~> 1.1.0", [hex: :syslog, repo: "hexpm", optional: false]}], "hexpm", "ab9fab4136dbc62651ec6f16fa4842f10cf02ab4433fa3d0976c01be99398399"}, + "exif_stripper": {:hex, :exif_stripper, "0.1.1", "4e9d682fecee3532974126894bc41629d7346702b51ee6d969c98ae78313328d", [:mix], [], "hexpm", "bd62115d7eb0d0458b2c92aeb4e54df8946b136113556cda7e2b572b638c1535"}, "exile": {:hex, :exile, "0.10.0", "b69e2d27a9af670b0f0a0898addca0eda78f6f5ba95ccfbc9bc6ccdd04925436", [:make, :mix], [{:elixir_make, "~> 0.6", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm", "c62ee8fee565b5ac4a898d0dcd58d2b04fb5eec1655af1ddcc9eb582c6732c33"}, "expo": {:hex, :expo, "0.5.1", "249e826a897cac48f591deba863b26c16682b43711dd15ee86b92f25eafd96d9", [:mix], [], "hexpm", "68a4233b0658a3d12ee00d27d37d856b1ba48607e7ce20fd376958d0ba6ce92b"}, "fast_html": {:hex, :fast_html, "2.5.0", "23578c1c1fa03db6a3786d78218b2d944df34b0fc3729e72de912f390913c80f", [:make, :mix], [{:elixir_make, "~> 0.4", [hex: :elixir_make, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 1.1", [hex: :nimble_pool, repo: "hexpm", optional: false]}], "hexpm", "69eb46ed98a5d9cca1ccd4a5ac94ce5dd626fc29513fbaa0a16cd8b2da67ae3e"}, From addbf235461c4a0edbb4e82a8e68230f5de2559f Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 27 Oct 2026 17:27:56 +0400 Subject: [PATCH 04/51] feat(upload): add Exif.StripLocation and Exif.ReadDescription filters Pure-Elixir filters based on exif_stripper. StripLocation also handles WebP and HEIC/AVIF, which the exiftool filter skipped. --- .../upload/filter/exif/read_description.ex | 32 ++++ .../upload/filter/exif/strip_location.ex | 25 +++ test/fixtures/strip_location_gps.heic | Bin 0 -> 2835 bytes test/fixtures/strip_location_gps.webp | Bin 0 -> 1382 bytes .../filter/exif/read_description_test.exs | 85 +++++++++++ .../filter/exif/strip_location_test.exs | 49 ++++++ .../filter/exiftool/read_description_test.exs | 144 ------------------ test/support/exif_helpers.ex | 44 ++++++ 8 files changed, 235 insertions(+), 144 deletions(-) create mode 100644 lib/pleroma/upload/filter/exif/read_description.ex create mode 100644 lib/pleroma/upload/filter/exif/strip_location.ex create mode 100644 test/fixtures/strip_location_gps.heic create mode 100644 test/fixtures/strip_location_gps.webp create mode 100644 test/pleroma/upload/filter/exif/read_description_test.exs create mode 100644 test/pleroma/upload/filter/exif/strip_location_test.exs delete mode 100644 test/pleroma/upload/filter/exiftool/read_description_test.exs create mode 100644 test/support/exif_helpers.ex diff --git a/lib/pleroma/upload/filter/exif/read_description.ex b/lib/pleroma/upload/filter/exif/read_description.ex new file mode 100644 index 000000000..654422b1b --- /dev/null +++ b/lib/pleroma/upload/filter/exif/read_description.ex @@ -0,0 +1,32 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2026 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Upload.Filter.Exif.ReadDescription do + @moduledoc """ + Reads a description from image metadata when the upload has none yet. + + Checks EXIF ImageDescription, then IPTC Caption-Abstract, then XMP. Blank + values and values longer than `[:instance, :description_limit]` are ignored. + """ + + @behaviour Pleroma.Upload.Filter + + @impl true + def filter(%Pleroma.Upload{description: description}) when is_binary(description), + do: {:ok, :noop} + + def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _} = upload) do + max_length = Pleroma.Config.get([:instance, :description_limit]) + + case ExifStripper.Description.read_file(file, max_length: max_length) do + description when is_binary(description) -> + {:ok, :filtered, %{upload | description: description}} + + nil -> + {:ok, :noop} + end + end + + def filter(_), do: {:ok, :noop} +end diff --git a/lib/pleroma/upload/filter/exif/strip_location.ex b/lib/pleroma/upload/filter/exif/strip_location.ex new file mode 100644 index 000000000..2e2b79f4c --- /dev/null +++ b/lib/pleroma/upload/filter/exif/strip_location.ex @@ -0,0 +1,25 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2026 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Upload.Filter.Exif.StripLocation do + @moduledoc """ + Strips GPS metadata from uploaded images and overwrites the file in place. + + Supports JPEG, PNG, WebP and ISO-BMFF images (HEIC, AVIF). This is lossless: + the image data is not re-encoded. Other image types are left untouched. + """ + + @behaviour Pleroma.Upload.Filter + + @impl true + def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do + case ExifStripper.GpsStripper.strip_gps_data(file, file) do + :ok -> {:ok, :filtered} + :noop -> {:ok, :noop} + {:error, _} = error -> error + end + end + + def filter(_), do: {:ok, :noop} +end diff --git a/test/fixtures/strip_location_gps.heic b/test/fixtures/strip_location_gps.heic new file mode 100644 index 0000000000000000000000000000000000000000..f27961d831a13737c89ef57c4f1394c0b2cac7a8 GIT binary patch literal 2835 zcmYjP2|Sc*7k_6l_I=5AC%TMn5;v8vge=+jLVjeK!MwK_21C>2TB5Q?vP(mSOY1Pn z+9;ZA;oi_XfsUaR z&^N#iv^3e_5p5KInlBuOM43b+hweZCpn?S25B$Tb5W%1T$WbJQ`ok6$1f)XYp=71+ zScD4FC=lL>=z}{I0Dy!TX~^-10bE6jAn3yI2Z?}F%<15EV|nm@Cx{S;VFW0M*gk6x zq=r(VvJQ|+2!^aAA($K*L?F|l3?zVwfhZvYnG6CQfH8!nz%Z&mWMMIeR0mRiFf@33 zH!&(m2O$DL@$vCN<}5UmDFWdG!3U560Co{r-&k1z08tbZI(?ThLI|<71t*~33>5ss z;`xiAn6e3CyYl}6OrGqMKKZIJ3<^2`Ll~L6ZI3rPXf&gDO`s}qP7vd zAJQKBK(oNJ!tB4$cbhkddGrq++6?n!2L!x3EO1x2-~Jto{0dn)NWb43Ail!`ZIFfg z|GMMzzQrN#a&mw1_qVGIzJ|>dRis=j_cPFnEjnb%%;K}oev8(?9uT__Jd2WG-{c$3 zzc5_o*BA5hZNRen(nhFF)RWYZW892@$JfpdwR;I`3OpQkA8X143eerpJo$!GOIpjQ zb4T#4E|#l2MFRQi9L?0t=<3rYugnYE$J~8xRQr4VZIPd#eKJmk!akak<8gb^(UNO2 zz*c^9I1rQN7TnsOn4@>?-c;Qq)DUYKyP@_(kXcW2O#XY*@A`ocmM7HiS9OKG8j_!W zz477d7G+uwa(%IFmQyD)87)NFNg!8jxMYY z`4SCYyvXIBWm*5uvbfW!REoMVaGNlt`H~^Rk6GDo^CF0@gej4 z{atigj3G|-{9uwZ)@`9|mpJZtqr%zj(ej(ThVCZuAKVfX-gEXY+_O3#y{sIJbtWIe zT|Rv}MBMZXx0@d4y5^L`x*ox5om1r}Ey{acYcJZvY2ysdOK4|Bgm@Q_BY7O0toUq4 zlXZ-16){5YmD*Do!lm=}E4GN%43l;tl}TpZUwto8D_&$r>*;S9|CW^~N~$I6-0LA9 ze3|Ar-{U_&>R>A)?u2j!bPc|}_gt8ZJ&X3X9q+oCk&|0J`J9{Xf)g^A#6OWPo84PE z9~f_Zt?#LvOituc)b8tNEtNLn;WGe|zt^MYWZHx+0is09{NLKPNVu7Z&bg-cN zirm0hy$ORX%OUDKsH;y8#|YHeC^LOB-Q8ULqw-@u&2NgUZ!R@%;rpzsf_xSpCblSc zx~I9-GyRlsw#g$$c;5=~ZXuI-Bn&zJzUw-KF33PN)X1lgaWs#GL^-YG4X!e86NBAFB3W}57H8C?k}Ptv16NVl zqk0cbMToz*{*FsMVJQw+XviRfd(-PzGGZ6~YEuD(SZ#;I_NY{?M zw{%^aPDnD_(`d?$lQ{gi^F+eKa#gvukCu<7Z=-5ZIBTP79Ud3Oro54NOWuBC4u{r; zQyx|2YwbyzT1_|KqnP0uCAEfReEV9%n!A;$A!*BQh^I^pWyg2MT|`(K@KwYqTymWD z!}N-AN$e|Y(p5^O!;S(=U0!pklS9O7o1`WYua~;rpM09qtwlX0P%)Mfi;NpS2cGaY zV**4n7&ff=pjRUF&kJ}vuRy*Yzw$EV$CP1U2o-gIe|Qy7S;^>Mq~?|4$b8++a)Bn# z+3fquq?CPTN!f*~GQdd(o~dj!W!%MQOfr{zxpZd0la70ySHL6KVQ*?>?|FW6F~xb( zXG+rSAI1<-YH~bQTUGy0!%=O)O{c&`QJTAUsHwc}y-&T)ksTKN#Mm{yU)YGe3M^M_ zq}uQ1lp*PkTVbyze;0$AATzy(EH~5}%cx0f{M>mNN#N0%uq*UxpbAr`3y(}(+|;{0 z%I)(W9LY#p$A2>-n_+(q&}q&5EUq(L@%cdZU0DH|{Caq$!AEN9Xh7N`#`}=oGY7S> z8s3V^uxE46T}ABFkKL2l;QaY}&j|&dz<=h$tz#wh5S>ObV|Mz=lk+u%L$Rq>gwfsA z9rV_dvb*`uYAi*a&>u;$-I#qv3eoa8wPkWAsaf47fhWjd^2nM;mB%DFo86_Y9M*qq zrkl>+BJH3P07{=#{6bxi zM#Z~Zj5U^o1+#%Ln3i{VR3b3s`CgP%f4q*V@oy>vVYCX>d&UDp!dL8dEtY519@{M4 zdN3MRM)!MIl4V{W;Bc~qE+n15BtDGn5 zx!BfPeZ0G^98Xnap%uH-a>9%_e`x$>BxFGKNS}>Xcj|8zb?F-T z8lCe5|84&L=@RP+ul}!+=KdeI$T_5hylnfZl!o;F=+-{>|0rX>7|Q5%)Z|_2QF+Hj zqwkwP^O7nswVSAmdyUYg$tgeDmsnCV-ouMOyvE&J-&_6_yWs879uaRfm?V?(Zq&d; z*f_cu9l*=Y)%Hygygyj`x!R;K=H0sga(Mx3OI|WED{uDH3+k%nGyjei8<3}hFxux8279WX;fjzI!Q5L&Ve`Xdpy%?9LFD>31JSM(#0)>qM=Q+X%?%6z1-$lY^u$1Cn{$wmsBL-C3305 zajRK0mP<&SlRAf0E~Uw(+~pMQ{8qir^SnOa&-1*$&-eTM_a(dF@iYYhj^i9j6q5A` z6##%Tq=NvM0ss*MjywQJ4UjKqf(2&4KF%V;xuh6_u@z)yB2utg=efNx{1On2eTnOH zd#m8Pn~n8*z;v9t*NNsbu3C2;)30Se7};?6 zRURv95s(M~e4XAsb$07&sgE}82(fow@LAL(u*p}lSRC$qrGeqnt89S2+L8w$wF4 zrDb&L*g}%!dOIuDxn`DfH?ogeaZOFAW0g3cC#)X2iBH4HX- zOl0HBcCO$o{8dOo8SwfzogO5ya&lSNsuw>7W44=7w*@)VcDDFMoxRa`ya zlQn*~xJ21)a;sBw|@SnqpJc$NkAgKiB)!Pm~t{1*2#q-Lsz1xR# zlFf&bZe41j%;s9@u0DgccHLmrG;Red-jen<{dAUp1S4h1p)^PRRLIwiM9=4@tKMyS zUzRWYdwOqF-neB}Q><@%3O_sDf)dEX9+rE-tKPy?`SOH~I) zv_~1&+w>@TzYp^?LV7hN*s2K?K2nC&LKk2i*ZnIo6M0f}pl=24rmo%NLkChd$!;BW zv(YOUGn=#?ZNm70`FJUFN92I>nK*V)vv=tqlY8(jWjhDetv(6YEbwxVG+owrI9Kcl`;4Y6 z1Ut>=%Z*>rS^)P=wIxx?fB6ubNtLlihEcV9vgmEnL*@XpuS#h3KA0)4JZ3++zi{Gi zxMb!8u-h9DC$MZRSlPC83;R5=gFMl4DmF*6$XV`SJ|ZXAdGb)jnf`3Im|E8XbEuE02_r^aanI^7xt*N&i*9+&{rY1f#Hb9LS>Kb +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Upload.Filter.Exif.ReadDescriptionTest do + use Pleroma.DataCase, async: true + + alias Pleroma.Upload.Filter.Exif.ReadDescription + + defp upload(fixture, attrs \\ []) do + path = Path.absname(Path.join("test/fixtures", fixture)) + + struct!( + %Pleroma.Upload{ + name: fixture, + content_type: "image/" <> String.trim_leading(Path.extname(fixture), "."), + path: path, + tempfile: path, + description: nil + }, + attrs + ) + end + + test "keeps description when not empty" do + upload = + upload("image_with_imagedescription_and_caption-abstract.jpg", + description: "Some description" + ) + + assert ReadDescription.filter(upload) == {:ok, :noop} + end + + test "otherwise returns ImageDescription when present" do + upload = upload("image_with_imagedescription_and_caption-abstract.jpg") + + assert ReadDescription.filter(upload) == + {:ok, :filtered, %{upload | description: "a descriptive white pixel"}} + end + + test "ignores stray data after the image" do + upload = upload("image_with_imagedescription_and_caption-abstract_and_stray_data_after.png") + + assert {:ok, :filtered, %{description: "a descriptive white pixel"}} = + ReadDescription.filter(upload) + + assert ReadDescription.filter(upload("image_with_stray_data_after.png")) == {:ok, :noop} + end + + test "otherwise returns iptc:Caption-Abstract when present" do + upload = upload("image_with_caption-abstract.jpg") + + assert ReadDescription.filter(upload) == + {:ok, :filtered, %{upload | description: "an abstract white pixel"}} + end + + test "reads ImageDescription from WebP and HEIC" do + assert {:ok, :filtered, %{description: "a webp with a location"}} = + ReadDescription.filter(upload("strip_location_gps.webp")) + + assert {:ok, :filtered, %{description: "a heic with a location"}} = + ReadDescription.filter(upload("strip_location_gps.heic")) + end + + test "does nothing when there is no description" do + assert ReadDescription.filter(upload("image_with_no_description.jpg")) == {:ok, :noop} + end + + test "ignores descriptions that exceed the maximum length" do + clear_config([:instance, :description_limit], 5) + + upload = upload("image_with_imagedescription_and_caption-abstract.jpg") + assert ReadDescription.filter(upload) == {:ok, :noop} + end + + test "ignores content with only whitespace" do + upload = upload("image_with_imagedescription_and_caption-abstract_whitespaces.jpg") + assert ReadDescription.filter(upload) == {:ok, :noop} + end + + test "does nothing when the file can't be read" do + upload = %{upload("image.jpg") | tempfile: Path.absname("test/fixtures/non-existant_tmp.jpg")} + assert ReadDescription.filter(upload) == {:ok, :noop} + end +end diff --git a/test/pleroma/upload/filter/exif/strip_location_test.exs b/test/pleroma/upload/filter/exif/strip_location_test.exs new file mode 100644 index 000000000..01ca37711 --- /dev/null +++ b/test/pleroma/upload/filter/exif/strip_location_test.exs @@ -0,0 +1,49 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2026 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Upload.Filter.Exif.StripLocationTest do + use Pleroma.DataCase, async: true + + import Pleroma.Tests.ExifHelpers + + alias Pleroma.Upload.Filter.Exif.StripLocation + + for {fixture, content_type} <- [ + {"DSCN0010.jpg", "image/jpeg"}, + {"DSCN0010.png", "image/png"}, + {"strip_location_gps.webp", "image/webp"}, + {"strip_location_gps.heic", "image/heic"} + ] do + test "strips GPS data from #{content_type}" do + upload = upload_copy(unquote(fixture), unquote(content_type)) + assert gps_tags(upload.tempfile) =~ "GPS Latitude" + + assert StripLocation.filter(upload) == {:ok, :filtered} + assert gps_tags(upload.tempfile) == "" + end + end + + test "keeps other EXIF data in HEIC" do + upload = upload_copy("strip_location_gps.heic", "image/heic") + + assert StripLocation.filter(upload) == {:ok, :filtered} + assert tag(upload.tempfile, "ImageDescription") == "a heic with a location" + assert tag(upload.tempfile, "ImageSize") == "64x48" + end + + test "does nothing when there is no GPS data" do + upload = upload_copy("image.jpg", "image/jpeg") + original = File.read!(upload.tempfile) + + assert StripLocation.filter(upload) == {:ok, :noop} + assert File.read!(upload.tempfile) == original + end + + test "ignores non-image uploads" do + upload = upload_copy("DSCN0010.jpg", "application/octet-stream") + + assert StripLocation.filter(upload) == {:ok, :noop} + assert gps_tags(upload.tempfile) =~ "GPS Latitude" + end +end diff --git a/test/pleroma/upload/filter/exiftool/read_description_test.exs b/test/pleroma/upload/filter/exiftool/read_description_test.exs deleted file mode 100644 index 9a1bd61d7..000000000 --- a/test/pleroma/upload/filter/exiftool/read_description_test.exs +++ /dev/null @@ -1,144 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2021 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Upload.Filter.Exiftool.ReadDescriptionTest do - use Pleroma.DataCase - alias Pleroma.Upload.Filter - - @uploads %Pleroma.Upload{ - name: "image_with_imagedescription_and_caption-abstract.jpg", - content_type: "image/jpeg", - path: Path.absname("test/fixtures/image_with_imagedescription_and_caption-abstract.jpg"), - tempfile: Path.absname("test/fixtures/image_with_imagedescription_and_caption-abstract.jpg"), - description: nil - } - - test "keeps description when not empty" do - uploads = %Pleroma.Upload{ - name: "image_with_imagedescription_and_caption-abstract.jpg", - content_type: "image/jpeg", - path: Path.absname("test/fixtures/image_with_imagedescription_and_caption-abstract.jpg"), - tempfile: - Path.absname("test/fixtures/image_with_imagedescription_and_caption-abstract.jpg"), - description: "Some description" - } - - assert Filter.Exiftool.ReadDescription.filter(uploads) == - {:ok, :noop} - end - - test "otherwise returns ImageDescription when present" do - uploads_after = %Pleroma.Upload{ - name: "image_with_imagedescription_and_caption-abstract.jpg", - content_type: "image/jpeg", - path: Path.absname("test/fixtures/image_with_imagedescription_and_caption-abstract.jpg"), - tempfile: - Path.absname("test/fixtures/image_with_imagedescription_and_caption-abstract.jpg"), - description: "a descriptive white pixel" - } - - assert Filter.Exiftool.ReadDescription.filter(@uploads) == - {:ok, :filtered, uploads_after} - end - - test "Ignores warnings" do - uploads = %Pleroma.Upload{ - name: "image_with_imagedescription_and_caption-abstract_and_stray_data_after.png", - content_type: "image/png", - path: - Path.absname( - "test/fixtures/image_with_imagedescription_and_caption-abstract_and_stray_data_after.png" - ), - tempfile: - Path.absname( - "test/fixtures/image_with_imagedescription_and_caption-abstract_and_stray_data_after.png" - ) - } - - assert {:ok, :filtered, %{description: "a descriptive white pixel"}} = - Filter.Exiftool.ReadDescription.filter(uploads) - - uploads = %Pleroma.Upload{ - name: "image_with_stray_data_after.png", - content_type: "image/png", - path: Path.absname("test/fixtures/image_with_stray_data_after.png"), - tempfile: Path.absname("test/fixtures/image_with_stray_data_after.png") - } - - assert {:ok, :filtered, %{description: nil}} = Filter.Exiftool.ReadDescription.filter(uploads) - end - - test "otherwise returns iptc:Caption-Abstract when present" do - upload = %Pleroma.Upload{ - name: "image_with_caption-abstract.jpg", - content_type: "image/jpeg", - path: Path.absname("test/fixtures/image_with_caption-abstract.jpg"), - tempfile: Path.absname("test/fixtures/image_with_caption-abstract.jpg"), - description: nil - } - - upload_after = %Pleroma.Upload{ - name: "image_with_caption-abstract.jpg", - content_type: "image/jpeg", - path: Path.absname("test/fixtures/image_with_caption-abstract.jpg"), - tempfile: Path.absname("test/fixtures/image_with_caption-abstract.jpg"), - description: "an abstract white pixel" - } - - assert Filter.Exiftool.ReadDescription.filter(upload) == - {:ok, :filtered, upload_after} - end - - test "otherwise returns nil" do - uploads = %Pleroma.Upload{ - name: "image_with_no_description.jpg", - content_type: "image/jpeg", - path: Path.absname("test/fixtures/image_with_no_description.jpg"), - tempfile: Path.absname("test/fixtures/image_with_no_description.jpg"), - description: nil - } - - assert Filter.Exiftool.ReadDescription.filter(uploads) == - {:ok, :filtered, uploads} - end - - test "Return nil when image description from EXIF data exceeds the maximum length" do - clear_config([:instance, :description_limit], 5) - - assert Filter.Exiftool.ReadDescription.filter(@uploads) == - {:ok, :filtered, @uploads} - end - - test "Ignores content with only whitespace" do - uploads = %Pleroma.Upload{ - name: "non-existant.jpg", - content_type: "image/jpeg", - path: - Path.absname( - "test/fixtures/image_with_imagedescription_and_caption-abstract_whitespaces.jpg" - ), - tempfile: - Path.absname( - "test/fixtures/image_with_imagedescription_and_caption-abstract_whitespaces.jpg" - ), - description: nil - } - - assert Filter.Exiftool.ReadDescription.filter(uploads) == - {:ok, :filtered, uploads} - end - - test "Return nil when image description from EXIF data can't be read" do - uploads = %Pleroma.Upload{ - name: "non-existant.jpg", - content_type: "image/jpeg", - path: Path.absname("test/fixtures/non-existant.jpg"), - tempfile: Path.absname("test/fixtures/non-existant_tmp.jpg"), - description: nil - } - - assert Filter.Exiftool.ReadDescription.filter(uploads) == - {:ok, :filtered, uploads} - end -end diff --git a/test/support/exif_helpers.ex b/test/support/exif_helpers.ex new file mode 100644 index 000000000..3a8a3b84e --- /dev/null +++ b/test/support/exif_helpers.ex @@ -0,0 +1,44 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2026 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Tests.ExifHelpers do + @moduledoc """ + Helpers for upload filter tests that use exiftool as an independent oracle. + """ + + @doc "Copies a fixture to a unique temporary file and returns an upload for it." + def upload_copy(fixture, content_type, attrs \\ []) do + source = Path.absname(Path.join("test/fixtures", fixture)) + + tempfile = + Path.join( + System.tmp_dir!(), + "pleroma_exif_#{System.unique_integer([:positive])}#{Path.extname(fixture)}" + ) + + File.cp!(source, tempfile) + ExUnit.Callbacks.on_exit(fn -> File.rm(tempfile) end) + + struct!( + %Pleroma.Upload{ + name: fixture, + content_type: content_type, + path: source, + tempfile: tempfile + }, + attrs + ) + end + + @doc "Returns the GPS tags exiftool finds in a file, as text." + def gps_tags(path), do: exiftool!(["-gps:all", path]) + + @doc "Returns the value of one tag as exiftool reads it." + def tag(path, tag), do: exiftool!(["-s3", "-#{tag}", path]) + + defp exiftool!(args) do + {output, 0} = System.cmd("exiftool", ["-m" | args]) + String.trim(output) + end +end From dd337b92f1b561c86b79f7491b08e068a3ac3a6f Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 27 Oct 2026 17:29:15 +0400 Subject: [PATCH 05/51] refactor(upload): make Exiftool filters deprecated aliases of Exif filters Old configs keep working and now also strip GPS from WebP and HEIC. The deprecation warning names the replacement for each old filter, and exiftool is no longer a startup requirement. --- changelog.d/exif-heic-gps.security | 1 + changelog.d/exif-stripper.change | 1 + lib/pleroma/application_requirements.ex | 2 - lib/pleroma/config/deprecation_warnings.ex | 54 +++++++++-------- lib/pleroma/upload/filter/exiftool.ex | 14 +++++ .../filter/exiftool/read_description.ex | 48 ++------------- .../upload/filter/exiftool/strip_location.ex | 27 ++------- .../config/deprecation_warnings_test.exs | 59 +++++++++---------- .../filter/exiftool/strip_location_test.exs | 51 ---------------- test/pleroma/upload/filter/exiftool_test.exs | 31 ++++++++++ 10 files changed, 113 insertions(+), 175 deletions(-) create mode 100644 changelog.d/exif-heic-gps.security create mode 100644 changelog.d/exif-stripper.change create mode 100644 lib/pleroma/upload/filter/exiftool.ex delete mode 100644 test/pleroma/upload/filter/exiftool/strip_location_test.exs create mode 100644 test/pleroma/upload/filter/exiftool_test.exs diff --git a/changelog.d/exif-heic-gps.security b/changelog.d/exif-heic-gps.security new file mode 100644 index 000000000..0ca439d0c --- /dev/null +++ b/changelog.d/exif-heic-gps.security @@ -0,0 +1 @@ +Strip GPS data from WebP and HEIC/AVIF uploads; the Exiftool.StripLocation filter used to skip them diff --git a/changelog.d/exif-stripper.change b/changelog.d/exif-stripper.change new file mode 100644 index 000000000..ba8d47a19 --- /dev/null +++ b/changelog.d/exif-stripper.change @@ -0,0 +1 @@ +Exiftool upload filters are replaced by the built-in Exif.StripLocation and Exif.ReadDescription filters; exiftool is no longer required, old filter names still work but are deprecated diff --git a/lib/pleroma/application_requirements.ex b/lib/pleroma/application_requirements.ex index 87ecb7e2d..d9f4eb1d0 100644 --- a/lib/pleroma/application_requirements.ex +++ b/lib/pleroma/application_requirements.ex @@ -168,8 +168,6 @@ defmodule Pleroma.ApplicationRequirements do defp check_system_commands!(:ok) do filter_commands_statuses = [ - check_filter(Pleroma.Upload.Filter.Exiftool.StripLocation, "exiftool"), - check_filter(Pleroma.Upload.Filter.Exiftool.ReadDescription, "exiftool"), check_filter(Pleroma.Upload.Filter.Mogrify, "mogrify"), check_filter(Pleroma.Upload.Filter.Mogrifun, "mogrify"), check_filter(Pleroma.Upload.Filter.AnalyzeMetadata, "ffprobe") diff --git a/lib/pleroma/config/deprecation_warnings.ex b/lib/pleroma/config/deprecation_warnings.ex index 934a4a02e..b383b646c 100644 --- a/lib/pleroma/config/deprecation_warnings.ex +++ b/lib/pleroma/config/deprecation_warnings.ex @@ -22,40 +22,44 @@ defmodule Pleroma.Config.DeprecationWarnings do @logger_formatter_config_knobs [:colors, :format, :metadata, :truncate, :utc_log] + @exiftool_filter_replacements %{ + Pleroma.Upload.Filter.Exiftool => Pleroma.Upload.Filter.Exif.StripLocation, + Pleroma.Upload.Filter.Exiftool.StripLocation => Pleroma.Upload.Filter.Exif.StripLocation, + Pleroma.Upload.Filter.Exiftool.ReadDescription => Pleroma.Upload.Filter.Exif.ReadDescription + } + def check_exiftool_filter do filters = Config.get([Pleroma.Upload]) |> Keyword.get(:filters, []) - if Pleroma.Upload.Filter.Exiftool in filters do - Logger.warning(""" - !!!DEPRECATION WARNING!!! - Your config is using Exiftool as a filter instead of Exiftool.StripLocation. This should work for now, but you are advised to change to the new configuration to prevent possible issues later: + deprecated_filters = + filters + |> Enum.filter(&Map.has_key?(@exiftool_filter_replacements, &1)) + |> Enum.uniq() - ``` - config :pleroma, Pleroma.Upload, - filters: [Pleroma.Upload.Filter.Exiftool] - ``` + case deprecated_filters do + [] -> + :ok - Is now + deprecated_filters -> + replacements = + Enum.map_join(deprecated_filters, fn filter -> + "\n* `#{inspect(filter)}` is now `#{inspect(@exiftool_filter_replacements[filter])}`" + end) + Logger.warning(""" + !!!DEPRECATION WARNING!!! + Your config is using Exiftool upload filters. Pleroma no longer needs exiftool, the filters were replaced by built-in ones. The old names should work for now, but you are advised to change to the new configuration to prevent possible issues later: + #{replacements} + """) - ``` - config :pleroma, Pleroma.Upload, - filters: [Pleroma.Upload.Filter.Exiftool.StripLocation] - ``` - """) + new_config = + filters + |> Enum.map(&Map.get(@exiftool_filter_replacements, &1, &1)) + |> Enum.uniq() - new_config = - filters - |> Enum.map(fn - Pleroma.Upload.Filter.Exiftool -> Pleroma.Upload.Filter.Exiftool.StripLocation - filter -> filter - end) + Config.put([Pleroma.Upload, :filters], new_config) - Config.put([Pleroma.Upload, :filters], new_config) - - :error - else - :ok + :error end end diff --git a/lib/pleroma/upload/filter/exiftool.ex b/lib/pleroma/upload/filter/exiftool.ex new file mode 100644 index 000000000..869ac8b2b --- /dev/null +++ b/lib/pleroma/upload/filter/exiftool.ex @@ -0,0 +1,14 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2026 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Upload.Filter.Exiftool do + @moduledoc """ + Deprecated: use `Pleroma.Upload.Filter.Exif.StripLocation`. This module delegates to it. + """ + + @behaviour Pleroma.Upload.Filter + + @impl true + defdelegate filter(upload), to: Pleroma.Upload.Filter.Exif.StripLocation +end diff --git a/lib/pleroma/upload/filter/exiftool/read_description.ex b/lib/pleroma/upload/filter/exiftool/read_description.ex index 8283b8643..c1c60dfdc 100644 --- a/lib/pleroma/upload/filter/exiftool/read_description.ex +++ b/lib/pleroma/upload/filter/exiftool/read_description.ex @@ -4,51 +4,11 @@ defmodule Pleroma.Upload.Filter.Exiftool.ReadDescription do @moduledoc """ - Gets a valid description from the related EXIF tags and provides them in the response if no description is provided yet. - It will first check ImageDescription, when that doesn't probide a valid description, it will check iptc:Caption-Abstract. - A valid description means the fields are filled in and not too long (see `:instance, :description_limit`). + Deprecated: use `Pleroma.Upload.Filter.Exif.ReadDescription`. This module delegates to it. """ + @behaviour Pleroma.Upload.Filter - def filter(%Pleroma.Upload{description: description}) - when is_binary(description), - do: {:ok, :noop} - - def filter(%Pleroma.Upload{tempfile: file} = upload), - do: {:ok, :filtered, upload |> Map.put(:description, read_description_from_exif_data(file))} - - def filter(_, _), do: {:ok, :noop} - - defp read_description_from_exif_data(file) do - nil - |> read_when_empty(file, "-ImageDescription") - |> read_when_empty(file, "-iptc:Caption-Abstract") - end - - defp read_when_empty(current_description, _, _) when is_binary(current_description), - do: current_description - - defp read_when_empty(_, file, tag) do - if File.exists?(file) do - try do - {tag_content, 0} = - System.cmd("exiftool", ["-m", "-b", "-s3", tag, file], - stderr_to_stdout: false, - parallelism: true - ) - - tag_content = String.trim(tag_content) - - if tag_content != "" and - String.length(tag_content) <= - Pleroma.Config.get([:instance, :description_limit]), - do: tag_content, - else: nil - rescue - _ in ErlangError -> nil - end - else - nil - end - end + @impl true + defdelegate filter(upload), to: Pleroma.Upload.Filter.Exif.ReadDescription end diff --git a/lib/pleroma/upload/filter/exiftool/strip_location.ex b/lib/pleroma/upload/filter/exiftool/strip_location.ex index 23346d234..3e64adca8 100644 --- a/lib/pleroma/upload/filter/exiftool/strip_location.ex +++ b/lib/pleroma/upload/filter/exiftool/strip_location.ex @@ -4,30 +4,11 @@ defmodule Pleroma.Upload.Filter.Exiftool.StripLocation do @moduledoc """ - Strips GPS related EXIF tags and overwrites the file in place. - Also strips or replaces filesystem metadata e.g., timestamps. + Deprecated: use `Pleroma.Upload.Filter.Exif.StripLocation`. This module delegates to it. """ + @behaviour Pleroma.Upload.Filter - # Formats not compatible with exiftool at this time - def filter(%Pleroma.Upload{content_type: "image/heic"}), do: {:ok, :noop} - def filter(%Pleroma.Upload{content_type: "image/webp"}), do: {:ok, :noop} - def filter(%Pleroma.Upload{content_type: "image/svg" <> _}), do: {:ok, :noop} - - def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do - try do - case System.cmd("exiftool", ["-m", "-overwrite_original", "-gps:all=", "-png:all=", file], - stderr_to_stdout: true, - parallelism: true - ) do - {_response, 0} -> {:ok, :filtered} - {error, _} -> {:error, error} - end - rescue - e in ErlangError -> - {:error, "#{__MODULE__}: #{inspect(e)}"} - end - end - - def filter(_), do: {:ok, :noop} + @impl true + defdelegate filter(upload), to: Pleroma.Upload.Filter.Exif.StripLocation end diff --git a/test/pleroma/config/deprecation_warnings_test.exs b/test/pleroma/config/deprecation_warnings_test.exs index 62f5bdeea..65b6a3d98 100644 --- a/test/pleroma/config/deprecation_warnings_test.exs +++ b/test/pleroma/config/deprecation_warnings_test.exs @@ -13,40 +13,39 @@ defmodule Pleroma.Config.DeprecationWarningsTest do describe "filter exiftool" do test "gives warning when still used" do - clear_config( - [Pleroma.Upload, :filters], - [Pleroma.Upload.Filter.Exiftool] - ) - - assert capture_log(fn -> DeprecationWarnings.check_exiftool_filter() end) =~ - """ - !!!DEPRECATION WARNING!!! - Your config is using Exiftool as a filter instead of Exiftool.StripLocation. This should work for now, but you are advised to change to the new configuration to prevent possible issues later: - - ``` - config :pleroma, Pleroma.Upload, - filters: [Pleroma.Upload.Filter.Exiftool] - ``` - - Is now - - - ``` - config :pleroma, Pleroma.Upload, - filters: [Pleroma.Upload.Filter.Exiftool.StripLocation] - ``` - """ - end - - test "changes setting to exiftool strip location" do clear_config( [Pleroma.Upload, :filters], [Pleroma.Upload.Filter.Exiftool, Pleroma.Upload.Filter.Exiftool.ReadDescription] ) + log = capture_log(fn -> DeprecationWarnings.check_exiftool_filter() end) + + assert log =~ "!!!DEPRECATION WARNING!!!" + + assert log =~ + "* `Pleroma.Upload.Filter.Exiftool` is now `Pleroma.Upload.Filter.Exif.StripLocation`" + + assert log =~ + "* `Pleroma.Upload.Filter.Exiftool.ReadDescription` is now `Pleroma.Upload.Filter.Exif.ReadDescription`" + + refute log =~ "Exiftool.StripLocation`" + end + + test "changes setting to the Exif filters" do + clear_config( + [Pleroma.Upload, :filters], + [ + Pleroma.Upload.Filter.Dedupe, + Pleroma.Upload.Filter.Exiftool, + Pleroma.Upload.Filter.Exiftool.StripLocation, + Pleroma.Upload.Filter.Exiftool.ReadDescription + ] + ) + expected_config = [ - Pleroma.Upload.Filter.Exiftool.StripLocation, - Pleroma.Upload.Filter.Exiftool.ReadDescription + Pleroma.Upload.Filter.Dedupe, + Pleroma.Upload.Filter.Exif.StripLocation, + Pleroma.Upload.Filter.Exif.ReadDescription ] capture_log(fn -> DeprecationWarnings.warn() end) @@ -58,8 +57,8 @@ defmodule Pleroma.Config.DeprecationWarningsTest do clear_config( [Pleroma.Upload, :filters], [ - Pleroma.Upload.Filter.Exiftool.StripLocation, - Pleroma.Upload.Filter.Exiftool.ReadDescription + Pleroma.Upload.Filter.Exif.StripLocation, + Pleroma.Upload.Filter.Exif.ReadDescription ] ) diff --git a/test/pleroma/upload/filter/exiftool/strip_location_test.exs b/test/pleroma/upload/filter/exiftool/strip_location_test.exs deleted file mode 100644 index 485060215..000000000 --- a/test/pleroma/upload/filter/exiftool/strip_location_test.exs +++ /dev/null @@ -1,51 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2022 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Upload.Filter.Exiftool.StripLocationTest do - use Pleroma.DataCase, async: true - alias Pleroma.Upload.Filter - - test "apply exiftool filter" do - assert Pleroma.Utils.command_available?("exiftool") - - ~w{jpg png} - |> Enum.map(fn type -> - File.cp!( - "test/fixtures/DSCN0010.#{type}", - "test/fixtures/DSCN0010_tmp.#{type}" - ) - - upload = %Pleroma.Upload{ - name: "image_with_GPS_data.#{type}", - content_type: "image/jpeg", - path: Path.absname("test/fixtures/DSCN0010.#{type}"), - tempfile: Path.absname("test/fixtures/DSCN0010_tmp.#{type}") - } - - assert Filter.Exiftool.StripLocation.filter(upload) == {:ok, :filtered} - - {exif_original, 0} = System.cmd("exiftool", ["-m", "test/fixtures/DSCN0010.#{type}"]) - {exif_filtered, 0} = System.cmd("exiftool", ["-m", "test/fixtures/DSCN0010_tmp.#{type}"]) - - assert String.match?(exif_original, ~r/GPS/) - refute String.match?(exif_filtered, ~r/GPS/) - end) - end - - test "verify webp, heic, svg files are skipped" do - uploads = - ~w{webp heic svg svg+xml} - |> Enum.map(fn type -> - %Pleroma.Upload{ - name: "sample.#{type}", - content_type: "image/#{type}" - } - end) - - uploads - |> Enum.each(fn upload -> - assert Filter.Exiftool.StripLocation.filter(upload) == {:ok, :noop} - end) - end -end diff --git a/test/pleroma/upload/filter/exiftool_test.exs b/test/pleroma/upload/filter/exiftool_test.exs new file mode 100644 index 000000000..c3ee3c59c --- /dev/null +++ b/test/pleroma/upload/filter/exiftool_test.exs @@ -0,0 +1,31 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2026 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Upload.Filter.ExiftoolTest do + # Instances may still configure the deprecated Exiftool filter names. + # They must keep working, including for formats exiftool used to skip. + use Pleroma.DataCase, async: true + + import Pleroma.Tests.ExifHelpers + + for filter <- [Pleroma.Upload.Filter.Exiftool, Pleroma.Upload.Filter.Exiftool.StripLocation] do + test "#{inspect(filter)} strips GPS data from HEIC uploads" do + upload = upload_copy("strip_location_gps.heic", "image/heic") + assert gps_tags(upload.tempfile) =~ "GPS Latitude" + + assert Pleroma.Upload.Filter.filter([unquote(filter)], upload) == {:ok, upload} + assert gps_tags(upload.tempfile) == "" + end + end + + test "Exiftool.ReadDescription reads the description" do + upload = upload_copy("strip_location_gps.heic", "image/heic", description: nil) + + assert {:ok, %{description: "a heic with a location"}} = + Pleroma.Upload.Filter.filter( + [Pleroma.Upload.Filter.Exiftool.ReadDescription], + upload + ) + end +end From df280b1d6b1bb76777b6f4092ca4e5fc373e25fa Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 27 Oct 2026 17:29:44 +0400 Subject: [PATCH 06/51] feat(instance): enable Exif upload filters without exiftool check --- docs/administration/CLI_tasks/instance.md | 4 +-- lib/mix/tasks/pleroma/instance.ex | 30 +++++------------------ test/mix/tasks/pleroma/instance_test.exs | 2 +- 3 files changed, 9 insertions(+), 27 deletions(-) diff --git a/docs/administration/CLI_tasks/instance.md b/docs/administration/CLI_tasks/instance.md index 88509cf5b..622299629 100644 --- a/docs/administration/CLI_tasks/instance.md +++ b/docs/administration/CLI_tasks/instance.md @@ -37,8 +37,8 @@ If any of the options are left unspecified, you will be prompted interactively. - `--static-dir ` - the directory custom public files should be read from (custom emojis, frontend bundle overrides, robots.txt, etc.) - `--listen-ip ` - the ip the app should listen to, defaults to 127.0.0.1 - `--listen-port ` - the port the app should listen to, defaults to 4000 -- `--strip-uploads-location ` - use ExifTool to strip uploads of sensitive location data -- `--read-uploads-description ` - use ExifTool to read image descriptions from uploads +- `--strip-uploads-location ` - strip sensitive location data from uploaded images +- `--read-uploads-description ` - read image descriptions from upload metadata - `--anonymize-uploads ` - randomize uploaded filenames - `--dedupe-uploads ` - store files based on their hash to reduce data storage requirements if duplicates are uploaded with different filenames - `--skip-release-env` - skip generation the release environment file diff --git a/lib/mix/tasks/pleroma/instance.ex b/lib/mix/tasks/pleroma/instance.ex index 143af5cdd..1a196264a 100644 --- a/lib/mix/tasks/pleroma/instance.ex +++ b/lib/mix/tasks/pleroma/instance.ex @@ -162,38 +162,20 @@ defmodule Mix.Tasks.Pleroma.Instance do ) |> Path.expand() - {strip_uploads_location_message, strip_uploads_location_default} = - if Pleroma.Utils.command_available?("exiftool") do - {"Do you want to strip location (GPS) data from uploaded images? This requires exiftool, it was detected as installed. (y/n)", - "y"} - else - {"Do you want to strip location (GPS) data from uploaded images? This requires exiftool, it was detected as not installed, please install it if you answer yes. (y/n)", - "n"} - end - strip_uploads_location = get_option( options, :strip_uploads_location, - strip_uploads_location_message, - strip_uploads_location_default + "Do you want to strip location (GPS) data from uploaded images? (y/n)", + "y" ) === "y" - {read_uploads_description_message, read_uploads_description_default} = - if Pleroma.Utils.command_available?("exiftool") do - {"Do you want to read data from uploaded files so clients can use it to prefill fields like image description? This requires exiftool, it was detected as installed. (y/n)", - "y"} - else - {"Do you want to read data from uploaded files so clients can use it to prefill fields like image description? This requires exiftool, it was detected as not installed, please install it if you answer yes. (y/n)", - "n"} - end - read_uploads_description = get_option( options, :read_uploads_description, - read_uploads_description_message, - read_uploads_description_default + "Do you want to read metadata from uploaded images so clients can prefill the media description field? (y/n)", + "y" ) === "y" anonymize_uploads = @@ -324,14 +306,14 @@ defmodule Mix.Tasks.Pleroma.Instance do defp upload_filters(filters) when is_map(filters) do enabled_filters = if filters.strip_location do - [Pleroma.Upload.Filter.Exiftool.StripLocation] + [Pleroma.Upload.Filter.Exif.StripLocation] else [] end enabled_filters = if filters.read_description do - enabled_filters ++ [Pleroma.Upload.Filter.Exiftool.ReadDescription] + enabled_filters ++ [Pleroma.Upload.Filter.Exif.ReadDescription] else enabled_filters end diff --git a/test/mix/tasks/pleroma/instance_test.exs b/test/mix/tasks/pleroma/instance_test.exs index 5ecb6e445..0a8c3d107 100644 --- a/test/mix/tasks/pleroma/instance_test.exs +++ b/test/mix/tasks/pleroma/instance_test.exs @@ -95,7 +95,7 @@ defmodule Mix.Tasks.Pleroma.InstanceTest do assert generated_config =~ "http: [ip: {127, 0, 0, 1}, port: 4000]" assert generated_config =~ - "filters: [Pleroma.Upload.Filter.Exiftool.StripLocation, Pleroma.Upload.Filter.Exiftool.ReadDescription]" + "filters: [Pleroma.Upload.Filter.Exif.StripLocation, Pleroma.Upload.Filter.Exif.ReadDescription]" assert File.read!(tmp_path() <> "setup.psql") == generated_setup_psql() assert File.exists?(Path.expand("./test/instance/static/robots.txt")) From 114d683683837c25cd0fbd2c789378583ff65478 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 27 Oct 2026 17:30:07 +0400 Subject: [PATCH 07/51] docs: drop exiftool from install docs and Docker images --- Dockerfile | 2 +- Dockerfile-e2e | 2 +- docs/configuration/cheatsheet.md | 12 ++++++++---- docs/installation/alpine_linux_en.md | 2 +- docs/installation/debian_based_en.md | 2 +- docs/installation/debian_based_jp.md | 3 +-- docs/installation/freebsd_en.md | 2 +- docs/installation/generic_dependencies.include | 1 - docs/installation/gentoo_en.md | 3 +-- docs/installation/gentoo_otp_en.md | 3 +-- docs/installation/openbsd_en.md | 3 +-- docs/installation/openbsd_fi.md | 3 +-- .../installation/optional/media_graphics_packages.md | 12 +----------- docs/installation/otp_en.md | 5 ++--- installation/release-to-docker/Dockerfile | 2 +- 15 files changed, 22 insertions(+), 35 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4018d9bc9..1067ef1c8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -42,7 +42,7 @@ ARG HOME=/opt/pleroma ARG DATA=/var/lib/pleroma RUN apk update &&\ - apk add exiftool ffmpeg vips libmagic ncurses postgresql-client &&\ + apk add ffmpeg vips libmagic ncurses postgresql-client &&\ adduser --system --shell /bin/false --home ${HOME} pleroma &&\ mkdir -p ${DATA}/uploads &&\ mkdir -p ${DATA}/static &&\ diff --git a/Dockerfile-e2e b/Dockerfile-e2e index df8bb39aa..59a285a67 100644 --- a/Dockerfile-e2e +++ b/Dockerfile-e2e @@ -41,7 +41,7 @@ ARG HOME=/opt/pleroma ARG DATA=/var/lib/pleroma RUN apk update &&\ - apk add exiftool ffmpeg vips libmagic ncurses postgresql-client &&\ + apk add ffmpeg vips libmagic ncurses postgresql-client &&\ adduser --system --shell /bin/false --home ${HOME} pleroma &&\ mkdir -p ${DATA}/uploads &&\ mkdir -p ${DATA}/static &&\ diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index 4865ff0aa..2d525ad63 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -702,15 +702,19 @@ This filter replaces the filename (not the path) of an upload. For complete obfu No specific configuration. -#### Pleroma.Upload.Filter.Exiftool.StripLocation +#### Pleroma.Upload.Filter.Exif.StripLocation -This filter only strips the GPS and location metadata with Exiftool leaving color profiles and attributes intact. +This filter strips GPS and location metadata from JPEG, WebP, HEIC and AVIF images, leaving color profiles and attributes intact. For PNG images it removes all text and EXIF metadata chunks. Images are not re-encoded, and other formats are left unchanged. + +It replaces the deprecated `Pleroma.Upload.Filter.Exiftool.StripLocation`, which no longer needs exiftool. No specific configuration. -#### Pleroma.Upload.Filter.Exiftool.ReadDescription +#### Pleroma.Upload.Filter.Exif.ReadDescription -This filter reads the ImageDescription and iptc:Caption-Abstract fields with Exiftool so clients can prefill the media description field. +This filter reads the ImageDescription and iptc:Caption-Abstract fields so clients can prefill the media description field. + +It replaces the deprecated `Pleroma.Upload.Filter.Exiftool.ReadDescription`, which no longer needs exiftool. No specific configuration. diff --git a/docs/installation/alpine_linux_en.md b/docs/installation/alpine_linux_en.md index 7154bca48..777a94dfc 100644 --- a/docs/installation/alpine_linux_en.md +++ b/docs/installation/alpine_linux_en.md @@ -68,7 +68,7 @@ sudo rc-update add postgresql ### Install media / graphics packages (optional, see [`docs/installation/optional/media_graphics_packages.md`](../installation/optional/media_graphics_packages.md)) ```shell -sudo apk add ffmpeg imagemagick exiftool +sudo apk add ffmpeg imagemagick ``` ### Install PleromaBE diff --git a/docs/installation/debian_based_en.md b/docs/installation/debian_based_en.md index f9086d2f9..2c85e52aa 100644 --- a/docs/installation/debian_based_en.md +++ b/docs/installation/debian_based_en.md @@ -36,7 +36,7 @@ sudo apt install elixir erlang-dev erlang-nox ### Optional packages: [`docs/installation/optional/media_graphics_packages.md`](../installation/optional/media_graphics_packages.md) ```shell -sudo apt install imagemagick ffmpeg libimage-exiftool-perl +sudo apt install imagemagick ffmpeg ``` ### Install PleromaBE diff --git a/docs/installation/debian_based_jp.md b/docs/installation/debian_based_jp.md index c853840c2..e374900e5 100644 --- a/docs/installation/debian_based_jp.md +++ b/docs/installation/debian_based_jp.md @@ -28,7 +28,6 @@ Note: This article is potentially outdated because at this time we may not have - `certbot` (または何らかのLet's Encrypt向けACMEクライアント) - `ImageMagick` - `ffmpeg` -- `exiftool` ### システムを準備する @@ -60,7 +59,7 @@ sudo apt install elixir erlang-dev erlang-nox ### オプションパッケージ: [`docs/installation/optional/media_graphics_packages.md`](../installation/optional/media_graphics_packages.md) ```shell -sudo apt install imagemagick ffmpeg libimage-exiftool-perl +sudo apt install imagemagick ffmpeg ``` ### Pleroma BE (バックエンド) をインストールします diff --git a/docs/installation/freebsd_en.md b/docs/installation/freebsd_en.md index 920bc7d35..2b857b9a9 100644 --- a/docs/installation/freebsd_en.md +++ b/docs/installation/freebsd_en.md @@ -31,7 +31,7 @@ Setup the required services to automatically start at boot, using `sysrc(8)`. ### Install media / graphics packages (optional, see [`docs/installation/optional/media_graphics_packages.md`](../installation/optional/media_graphics_packages.md)) ```shell -# pkg install imagemagick ffmpeg p5-Image-ExifTool vips +# pkg install imagemagick ffmpeg vips ``` ## Configuring Pleroma diff --git a/docs/installation/generic_dependencies.include b/docs/installation/generic_dependencies.include index 86dbc6838..da7e50848 100644 --- a/docs/installation/generic_dependencies.include +++ b/docs/installation/generic_dependencies.include @@ -12,7 +12,6 @@ * ImageMagick * FFmpeg -* exiftool !!!info **Source installs only** diff --git a/docs/installation/gentoo_en.md b/docs/installation/gentoo_en.md index dc47d27f8..066497b85 100644 --- a/docs/installation/gentoo_en.md +++ b/docs/installation/gentoo_en.md @@ -41,7 +41,6 @@ Gentoo quite pointedly does not come with a cron daemon installed, and as such i * `app-crypt/certbot-nginx` (nginx certbot plugin that allows use of the all-powerful `--nginx` flag on certbot) * `media-gfx/imagemagick` * `media-video/ffmpeg` -* `media-libs/exiftool` ### Prepare the system @@ -97,7 +96,7 @@ Not only does this make it much easier to deploy changes you make, as you can co ### Install media / graphics packages (optional, see [`docs/installation/optional/media_graphics_packages.md`](docs/installation/optional/media_graphics_packages.md)) ```shell -# emerge --ask media-video/ffmpeg media-gfx/imagemagick media-libs/exiftool +# emerge --ask media-video/ffmpeg media-gfx/imagemagick ``` ### Install PleromaBE diff --git a/docs/installation/gentoo_otp_en.md b/docs/installation/gentoo_otp_en.md index 20d8835da..352d9b807 100644 --- a/docs/installation/gentoo_otp_en.md +++ b/docs/installation/gentoo_otp_en.md @@ -25,7 +25,6 @@ Gentoo quite pointedly does not come with a cron daemon installed, and as such i * `app-crypt/certbot-nginx` (nginx certbot plugin that allows use of the all-powerful `--nginx` flag on certbot) * `media-gfx/imagemagick` * `media-video/ffmpeg` -* `media-libs/exiftool` ### Prepare the system @@ -68,7 +67,7 @@ The output from emerging postgresql should give you a command for initializing t See [Optional software packages needed for specific functionality](optional/media_graphics_packages.md) for details. ```shell -# emerge --ask media-video/ffmpeg media-gfx/imagemagick media-libs/exiftool +# emerge --ask media-video/ffmpeg media-gfx/imagemagick ``` ### Setup PleromaBE diff --git a/docs/installation/openbsd_en.md b/docs/installation/openbsd_en.md index 82d137e96..1cc638ae0 100644 --- a/docs/installation/openbsd_en.md +++ b/docs/installation/openbsd_en.md @@ -26,12 +26,11 @@ Independently of the reverse proxy, [acme-client(1)](https://man.openbsd.org/acm * ImageMagick * ffmpeg - * exiftool To install the above: ``` -# pkg_add ImageMagick ffmpeg p5-Image-ExifTool +# pkg_add ImageMagick ffmpeg ``` For more information read [`docs/installation/optional/media_graphics_packages.md`](../installation/optional/media_graphics_packages.md): diff --git a/docs/installation/openbsd_fi.md b/docs/installation/openbsd_fi.md index 858e64020..9ed553280 100644 --- a/docs/installation/openbsd_fi.md +++ b/docs/installation/openbsd_fi.md @@ -25,11 +25,10 @@ Asenna tarvittava ohjelmisto: [`docs/installation/optional/media_graphics_packages.md`](../installation/optional/media_graphics_packages.md): * ImageMagick * ffmpeg - * exiftool Asenna tarvittava ohjelmisto: -`# pkg_add ImageMagick ffmpeg p5-Image-ExifTool` +`# pkg_add ImageMagick ffmpeg` Luo postgresql-tietokanta: diff --git a/docs/installation/optional/media_graphics_packages.md b/docs/installation/optional/media_graphics_packages.md index 89125d1c6..e7e6283f8 100644 --- a/docs/installation/optional/media_graphics_packages.md +++ b/docs/installation/optional/media_graphics_packages.md @@ -2,9 +2,8 @@ For specific Pleroma functionality (which is disabled by default) some or all of the below packages are required: -* `ImageMagic` +* `ImageMagick` * `ffmpeg` -* `exiftool` Please refer to documentation in `docs/installation` on how to install them on specific OS. @@ -26,12 +25,3 @@ It is required for the following Pleroma features: It is required for the following Pleroma features: * Media preview proxy for videos (related config: `media_preview_proxy/enabled` in `config/config.exs`) - -## `exiftool` - -`exiftool` is media files metadata reader/writer. - -It is required for the following Pleroma features: - -* `Pleroma.Upload.Filters.Exiftool.StripLocation` upload filter (related config: `Pleroma.Upload/filters` in `config/config.exs`) -* `Pleroma.Upload.Filters.Exiftool.ReadDescription` upload filter (related config: `Pleroma.Upload/filters` in `config/config.exs`) diff --git a/docs/installation/otp_en.md b/docs/installation/otp_en.md index 0f296a9f6..7132ef5a5 100644 --- a/docs/installation/otp_en.md +++ b/docs/installation/otp_en.md @@ -61,17 +61,16 @@ Other than things bundled in the OTP release Pleroma depends on: Per [`docs/installation/optional/media_graphics_packages.md`](optional/media_graphics_packages.md): * ImageMagick * ffmpeg - * exiftool === "Alpine" ``` apk update - apk add imagemagick ffmpeg exiftool + apk add imagemagick ffmpeg ``` === "Debian/Ubuntu" ``` - apt install imagemagick ffmpeg libimage-exiftool-perl + apt install imagemagick ffmpeg ``` ## Setup diff --git a/installation/release-to-docker/Dockerfile b/installation/release-to-docker/Dockerfile index 0ffeb5d5b..ac9118aba 100644 --- a/installation/release-to-docker/Dockerfile +++ b/installation/release-to-docker/Dockerfile @@ -12,7 +12,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ openssl libssl3 \ libmagic1t64 file \ postgresql-client \ - ffmpeg imagemagick libimage-exiftool-perl \ + ffmpeg imagemagick \ libvips42t64 \ unzip \ curl \ From bd6602f4ce939b5f7d3817cc276e10f73812467e Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 27 Oct 2026 17:35:32 +0400 Subject: [PATCH 08/51] feat(config): migrate Exiftool upload filters in ConfigDB to Exif names --- ...7000000_upload_filter_exiftool_to_exif.exs | 41 ++++++++++++ .../upload_filter_exiftool_to_exif_test.exs | 64 +++++++++++++++++++ 2 files changed, 105 insertions(+) create mode 100644 priv/repo/migrations/20261027000000_upload_filter_exiftool_to_exif.exs create mode 100644 test/pleroma/repo/migrations/upload_filter_exiftool_to_exif_test.exs diff --git a/priv/repo/migrations/20261027000000_upload_filter_exiftool_to_exif.exs b/priv/repo/migrations/20261027000000_upload_filter_exiftool_to_exif.exs new file mode 100644 index 000000000..62392f957 --- /dev/null +++ b/priv/repo/migrations/20261027000000_upload_filter_exiftool_to_exif.exs @@ -0,0 +1,41 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2026 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Repo.Migrations.UploadFilterExiftoolToExif do + use Ecto.Migration + + alias Pleroma.ConfigDB + + @up %{ + Pleroma.Upload.Filter.Exiftool => Pleroma.Upload.Filter.Exif.StripLocation, + Pleroma.Upload.Filter.Exiftool.StripLocation => Pleroma.Upload.Filter.Exif.StripLocation, + Pleroma.Upload.Filter.Exiftool.ReadDescription => Pleroma.Upload.Filter.Exif.ReadDescription + } + + @down %{ + Pleroma.Upload.Filter.Exif.StripLocation => Pleroma.Upload.Filter.Exiftool.StripLocation, + Pleroma.Upload.Filter.Exif.ReadDescription => Pleroma.Upload.Filter.Exiftool.ReadDescription + } + + def up, do: rename_filters(@up) + + def down, do: rename_filters(@down) + + defp rename_filters(renames) do + case ConfigDB.get_by_params(%{group: :pleroma, key: Pleroma.Upload}) do + %{value: value} -> + new_value = + Keyword.update(value, :filters, [], fn filters -> + filters + |> Enum.map(&Map.get(renames, &1, &1)) + |> Enum.uniq() + end) + + ConfigDB.update_or_create(%{group: :pleroma, key: Pleroma.Upload, value: new_value}) + + _ -> + nil + end + end +end diff --git a/test/pleroma/repo/migrations/upload_filter_exiftool_to_exif_test.exs b/test/pleroma/repo/migrations/upload_filter_exiftool_to_exif_test.exs new file mode 100644 index 000000000..9d39c6ea6 --- /dev/null +++ b/test/pleroma/repo/migrations/upload_filter_exiftool_to_exif_test.exs @@ -0,0 +1,64 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2026 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Repo.Migrations.UploadFilterExiftoolToExifTest do + use Pleroma.DataCase + import Pleroma.Factory + import Pleroma.Tests.Helpers + alias Pleroma.ConfigDB + + setup_all do: require_migration("20261027000000_upload_filter_exiftool_to_exif") + + defp filters do + ConfigDB.get_by_params(%{group: :pleroma, key: Pleroma.Upload}).value[:filters] + end + + test "up/0 renames the Exiftool filters", %{migration: migration} do + insert(:config, + group: :pleroma, + key: Pleroma.Upload, + value: [ + filters: [ + Pleroma.Upload.Filter.Dedupe, + Pleroma.Upload.Filter.Exiftool, + Pleroma.Upload.Filter.Exiftool.StripLocation, + Pleroma.Upload.Filter.Exiftool.ReadDescription + ] + ] + ) + + migration.up() + + assert filters() == [ + Pleroma.Upload.Filter.Dedupe, + Pleroma.Upload.Filter.Exif.StripLocation, + Pleroma.Upload.Filter.Exif.ReadDescription + ] + end + + test "down/0 restores the Exiftool filters", %{migration: migration} do + insert(:config, + group: :pleroma, + key: Pleroma.Upload, + value: [ + filters: [ + Pleroma.Upload.Filter.Exif.StripLocation, + Pleroma.Upload.Filter.Exif.ReadDescription + ] + ] + ) + + migration.down() + + assert filters() == [ + Pleroma.Upload.Filter.Exiftool.StripLocation, + Pleroma.Upload.Filter.Exiftool.ReadDescription + ] + end + + test "up/0 does nothing without a database config", %{migration: migration} do + assert migration.up() == nil + assert ConfigDB.get_by_params(%{group: :pleroma, key: Pleroma.Upload}) == nil + end +end From d1deb9da4546ef27c426c0cd98e9dad6adbaaa7f Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 27 Oct 2026 18:07:45 +0400 Subject: [PATCH 09/51] build: bump exif_stripper to 0.1.2 0.1.2 handles gridded HEIC photos (as iPhones write them), PNG files with data after IEND and TIFF files, and bounds HEIC parsing by the input size. Add tests for these files. --- mix.exs | 2 +- mix.lock | 2 +- test/fixtures/strip_location_gps.tiff | Bin 0 -> 9038 bytes test/fixtures/strip_location_gps_grid.heic | Bin 0 -> 12530 bytes .../upload/filter/exif/strip_location_test.exs | 12 ++++++++++-- 5 files changed, 12 insertions(+), 4 deletions(-) create mode 100644 test/fixtures/strip_location_gps.tiff create mode 100644 test/fixtures/strip_location_gps_grid.heic diff --git a/mix.exs b/mix.exs index fc2d70b48..6527f902b 100644 --- a/mix.exs +++ b/mix.exs @@ -205,7 +205,7 @@ defmodule Pleroma.Mixfile do {:majic, "~> 1.2"}, {:open_api_spex, "~> 3.22"}, {:ecto_psql_extras, "~> 0.8"}, - {:exif_stripper, "~> 0.1.1"}, + {:exif_stripper, "~> 0.1.2"}, {:vix, "~> 0.36"}, {:elixir_make, "~> 0.7.8", override: true}, {:blurhash, "~> 0.1.0", hex: :rinpatch_blurhash}, diff --git a/mix.lock b/mix.lock index aaa944985..4fbcc257f 100644 --- a/mix.lock +++ b/mix.lock @@ -47,7 +47,7 @@ "ex_doc": {:hex, :ex_doc, "0.38.2", "504d25eef296b4dec3b8e33e810bc8b5344d565998cd83914ffe1b8503737c02", [:mix], [{:earmark_parser, "~> 1.4.44", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "732f2d972e42c116a70802f9898c51b54916e542cc50968ac6980512ec90f42b"}, "ex_machina": {:hex, :ex_machina, "2.8.0", "a0e847b5712065055ec3255840e2c78ef9366634d62390839d4880483be38abe", [:mix], [{:ecto, "~> 2.2 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:ecto_sql, "~> 3.0", [hex: :ecto_sql, repo: "hexpm", optional: true]}], "hexpm", "79fe1a9c64c0c1c1fab6c4fa5d871682cb90de5885320c187d117004627a7729"}, "ex_syslogger": {:hex, :ex_syslogger, "1.5.2", "72b6aa2d47a236e999171f2e1ec18698740f40af0bd02c8c650bf5f1fd1bac79", [:mix], [{:poison, ">= 1.5.0", [hex: :poison, repo: "hexpm", optional: true]}, {:syslog, "~> 1.1.0", [hex: :syslog, repo: "hexpm", optional: false]}], "hexpm", "ab9fab4136dbc62651ec6f16fa4842f10cf02ab4433fa3d0976c01be99398399"}, - "exif_stripper": {:hex, :exif_stripper, "0.1.1", "4e9d682fecee3532974126894bc41629d7346702b51ee6d969c98ae78313328d", [:mix], [], "hexpm", "bd62115d7eb0d0458b2c92aeb4e54df8946b136113556cda7e2b572b638c1535"}, + "exif_stripper": {:hex, :exif_stripper, "0.1.2", "fa176be937201366545c8c93b5d89b6e2da1ee00534c3f1fb931402e03262755", [:mix], [], "hexpm", "5c524f7cc061826a6108b7b6f85d1d2481d0cff4ce0a678aaedd6edaa0d86dcb"}, "exile": {:hex, :exile, "0.10.0", "b69e2d27a9af670b0f0a0898addca0eda78f6f5ba95ccfbc9bc6ccdd04925436", [:make, :mix], [{:elixir_make, "~> 0.6", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm", "c62ee8fee565b5ac4a898d0dcd58d2b04fb5eec1655af1ddcc9eb582c6732c33"}, "expo": {:hex, :expo, "0.5.1", "249e826a897cac48f591deba863b26c16682b43711dd15ee86b92f25eafd96d9", [:mix], [], "hexpm", "68a4233b0658a3d12ee00d27d37d856b1ba48607e7ce20fd376958d0ba6ce92b"}, "fast_html": {:hex, :fast_html, "2.5.0", "23578c1c1fa03db6a3786d78218b2d944df34b0fc3729e72de912f390913c80f", [:make, :mix], [{:elixir_make, "~> 0.4", [hex: :elixir_make, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 1.1", [hex: :nimble_pool, repo: "hexpm", optional: false]}], "hexpm", "69eb46ed98a5d9cca1ccd4a5ac94ce5dd626fc29513fbaa0a16cd8b2da67ae3e"}, diff --git a/test/fixtures/strip_location_gps.tiff b/test/fixtures/strip_location_gps.tiff new file mode 100644 index 0000000000000000000000000000000000000000..afbc4ac5834b21da059f761f2d5eaf4a7bbcbe13 GIT binary patch literal 9038 zcmeHLcUV-{w%_N>nL-l{qz=6+SjS<2p$y}|ARsnkP{G(m6l`cz)S%Ib zK&+@iqp|mPF@^|A493`v-nC~yhkNtg``-8ddmm@Mea>&MUDn#awe~t^c1#R%LP)Cx zk()?FA%qAhftzTbG1L)aLcuK*Ns}WI?qn?75Q*q;$I`iQm&?atIrT|i3kZ+J1tEr9 zj`3c39U(_};`%VflNAWh3@DL`x_nM(*Zi(u4i7|p9J+=W7-C?EfguKl7#Lz;h=Cyn zh8P%PV2FVs28I|IV&MNh14tK+1Qcfxfp8o^C}FZ6vp`Pk$IzKzC}CA4to`Y%qlM5> zlItsBgMN&484mQ zEG`fW@pJI?^19+Y=87wxYjE3tt5^*0mouk+p!jMqFU!MDT>E)LY-bL&Wpvm0s)^E7RuKPbq1>_J2?X6 z9)OdJq{( zt5BSR`Ggo@-bbkf9|=(bng(|P5~Bx*g9?!d70Xc;;-h>h9I(IzN;vcqLyg zEWll#+HoC^z++sI1lkHvCiKmNkpprLoF@wVN8(uupx|+M=5G1Sf_{x|pfT9uYb1mA zY}5edXv&}B%C$+MM=5fZ%SX#)16n_?9IvhClh)~AtMclV z85>~vIPg^jGv~ny#r^ygf;I2K?%e*-pV^daTxKG3lt^lpB=9D01AAfR71Ajp!G)!L z9p#6N5a;^hV`mU&w^r+E`Fb!joNH9e#4G9@<^RE}pS?L?+hDK8DaVaGk~01+Za^3P zGW{a`KK&}}HrhGFfm$2wIIUHVcGC{Q@7qt~2JZ+3$AfnSj!!}V4#2A~>W@L? zy`r2Ius`rT*c-V20EapKJ6kzcG2_ZHgt`2mS)|PM;0&VhzI>OmZP@N?CfkYa$qr%@ zY%BPAumb_vu&vqAY(r>`U^}ow*ruQO81PsgNN@z<$om|NioBNwRt`iOUSBqNUnl!(OR*5s9NO_+T8nBZUniM%?K<5Cjat{y6pedk9gl2#pQs5P52QlOx&Y(Ux zcjK%D5l>8z`|xjKl;Ov4V1zOpV5}lraTpXDFtrJ(L7X%s?jpE8m&Vc(N@%mn$SLU z5FJ9t(GTb>x`?i!JMaeTLQha1dWqf<1i>UU2wlRMuq5mVXTpQZ-dTDPc2Bk}JC|B@J1eHPwsOi*fYB9Br+Ceo_C#fsc1FDbu zK-Zv~(VgjjbRK;?J&QhrUP-T^Z=)ZepMtF2MSsa)Fbp6rydhrF7&(lYjKz%gj6IBF zj7yA%j2BEM)0pYZ^k>E~`OG3_C37uvH}fd-GP8^Mnx)3FWO=Z{Sm`V=tAbU-+Q~Y? zy3BgSdZ(hLVyEJ#5~q@(QmV34Wvfb?$_14!m3OMzst&6Ds>!N3s`FH9Rhv{#soqt6 zuBN7Dt>&wipe9nAt5&PFSM9Xg1GP8mI_gg9!Riy$W$H`Rx2qphzoq_MLqo$}BTyq< zqex?kMx(|Fjr$s}*}7~uHjkabp2MzVH?uFZpK7XT+Gz%9PSz~btkK-Bc|o&ROGV3G zD_D!KHCwArt5xfU)(dT2ZFlWB?R@RU+B>z+Xm{%{b?kLQbux73>uk|Eq4Q9erfZ`c ztSiv{LU*g~DcvqThMt2SPcKKWTJIaZ^Ll;yI{G8^Q}s*q>+}!ncN)+P>^ zCNoSnn4B`{HPtg6WhyYOGTmo-+l(}GGD|d@X|~1eoY@O=3-fSuiTPUd6Xv}Z1{Q%9 zIToudj#%_q>ROJm6j`paJZkxvqt6NAyhaGONv2=dIpZJ6NY!S6J_{ zzGtIq<71O)v&yF3rq9;WHpaHxc9-pKI~6-0yKK8Pb{%%F>>cdK*)Om^Xy5H%Wjh~lak>5VQ-$%KQ${n?HRQG6y(ZbPNM?dtp_UHR= z_U|0S88davrZJrXRssBghJc>~Z2~g_w*@{58Wxljv^(hOSngQq*aKr<1^Wfh3O*J> z3keUY3OOID6PgmbF7$qwby!x|uCTsv&+wVy$0FzvQ4z}{Zt%=`0$wApFVZ`5R^-Vj z^{Aw%bx}V@J4H*Q+hWL=$e5KecVq2h#jywDP#iCAW!$}ZhxqC7-zP8<;uGo;eob^w zEKlr6(oLF@v@_{t^4R31$#+s5Q)DUashX)1QyWuXjSCsKVqE9=;p1nGKbvNfCQ55b zXQrp7Z%cnMA#}p(30)ICCst0pKFMKH$)q!rO(y3}K0HNp3V+J}sr0GirtX~jo*&0= z;J=v0n^r%qPY^6vBY2V#n6WCON9ZqHA$*kSpSdElJIg<7WmZpiKz2=buP9iwPV_7% zB4=aHtK7KU?YSTG#^vqFXXQ`LZx!o^bHyhMEDA~rE=imvm6Fcse$!V^?~_JJw-pkF zlL}jk^x#kFIhmvEOWA|sF~xPouVs>QuU@^if$@+1E3vnSz<^i3HUi+}ve{j1uqJ}${za(=1T z(hbWP%f!pBFArG0V}d)0vR-dUES+jAC>KfUa&f4hOmbJEP zm#uxjE_>azy0LY8>dopG)xTPwvHtRgzzt0s%{NwWe7i}s>E`C}&CLz=4K-UBTZ*^* zx;1rc$JahzH*Pc8R=w@7?c(j7I}&%CZ1iqy+-bIR=`M1YY*)|jiMua;6Y@<$ZWrHvcV{kq*jL@tx>s}G z?0&;PhW%qtr&s6U2O$s6{hajkori*lPkt%<<$c%uUp0QMd1U#hvD>5jP)}IT#m5sK zcRi6jdDmO{oAz(@PaU6r_j};)=laI?bv=_l`}oJ==f=;s|2gu{_80Ll?!U}``R-NK lYvb2D-uS%fcsuUxqjxjjGvC+#HSDjp4^bcPe$4;)@t=LG`q%&f literal 0 HcmV?d00001 diff --git a/test/fixtures/strip_location_gps_grid.heic b/test/fixtures/strip_location_gps_grid.heic new file mode 100644 index 0000000000000000000000000000000000000000..3a9d5bf97b904a1c99af9cf1bf88f71ce18ea9eb GIT binary patch literal 12530 zcmeHMcU)A*x1YQBZUL5EdM``wu%IY*dJzyrED)ovTVP>X)}@GOY%y4457yXX*QhbE z#)4g=v6mzoe;8vFH3`O8q9(pGcXkny`+4vEO5Pvu!@%8h&Uf0GGiT17J&O?X*Hn%& z6sxsL1f@)?iO$yM8q2f_O|~{Yk>DQkGIgZ_*F*6sR_Tm{MaZC4R+@P;^;c>28W22G zMm46mQ1FPWj5?W#ml(8_Wxyj8jSv&8B|Q*|0mKmmH4pcJUd7c)Xwe#kn!kcH7szmd zd@fMH1q!)95f^B|1zK`}R$QQ%3$*3}C0wAC3$)<^ZMi@@F3_F}bl?IVxj-i_(3uN# z;R0Q`KsPSXoeT8f0zJ7vFD}rV3zTtzK3t$L7wE?Y`g4H+Twov<7{mnzbAcgTU?>+D z#s!9Rfe~C_Bo`RP1;6ZHuTq1i2sOYn3Y3NzOLa#`$H3jB7ZBBdYjalk*U)(+?cd^|!te?w(r9`L<^S5=kvPXHd=mbXP?OzjVR zPvE~Pu1d#zM}%mX+T#3yz(Y`>vr5Y|abGyNv_?O?PY&=f7d=H;kyHpg94Y#6u{s0q z85~*qzOg(Hw}Au1NL1?N6yQM*CQe(Kk#FMbDysWo9Q0%^*A^#XJz*}sxJ1!A7kD_( z{6MuX4bKbX@Usk+xp+*F`PF(|A3Rq)@F&$3?0P{Kh$@To(*chGKEKkKkNbi?f;x>h zH3Rrm;5QZ<(=iU~6*L=kte=5RgpmqkN*dr`6Jd?9Di7NM)+k)9*7wKu1RoLJL<1-V zQloM-96psuk8Z)~s6bj&&2mLZfy&^Kfd=95q(Lh^JVt0yfl2`{0{%`*J(h6~+$RDV zpspNgpsfyinsFIY!3d-$9;F-}Ea8#wu_G%@Ge*LUNnUGE71S4_hfrS(TsCTCks8pj z4CEw0tDvq3_|~(;fG5SETvl761FpxjG?`{8haM_+F47OLljxQNI_ps_)M4G&wKL9) zoaq2hHztee!N}R!jL>2fieT~XEM}hd&C2oGnp$QZ0lKzceQS%Et$YaVRS6^OV1;^< zeJVhkSD<&PsrMUevNO(hs|q$4>Sy-N7_t`hg_Tb<4%I5p&1ya|W}gd^U7gWz3duIx z>^aG<2U|vVh1os$lK8}!|7KU;?50__w!50s+HY7-QvJnmU@rbe{(1gQ{$(Ts`aS;& z{~mDH`8VM2>lS_9>;d*EE#Sl_h_@*znr(@_Faa#1V{6O68hEGF>^`MIKLyyg0<2dB zeuwuu0z9DgUbOWPJX-5|>=9)4A{duQZ#-VfBG^Ol{McLmYM+1RgtR&z8vSfHgJH

?Sm$U%eToN%2&%b%OA^cvQ}XCPkv4Qqx_0|CDc5i=g~Xp{q!Ds zFMS5dK-xnF!nXY1Y1G=*Z6gTTefE?}{3Ov5p(IoiD@l-05+C@Bk;DV%C-IeJ zNt~fJT@oZombkar!@OfUK(Gg4&wJyGB<>Y}mS#`G>nn!cH?mPqVOsB>*C=bVN*n>T zt9HG!pbMf?aWR&_nEzwP}N6H;!81@zh)F9&-4o zpsOHAkRk|zz9d@71j&MQK`N37qA@Or67&YX3vR`DG0{v0M7y3NZ1|z za2zByieP#{ZBHhdiNi6n^=d&AVp|+7xmViy6vADjQe6Z06@-$?4Wo?OqT)(f0^E$$ zvJAa4GF%oTkA^oQzGvY!2tDr0-k&I^Gs-GsHNhB6_;4?j!adLx?tOmnR*wK(JELy! zo=t&!Lk`ME1!xFpR}6b;1lx{56HpzRj%K3|&|xjEbS+sGd{`l||)I1=KLAh|*J4)EMerYC1KS zT1>5^Hc+2Xd#FRyDe40CHFb+>qJE)X@MxZx=fLyi1@fYJal8axCNH1&Hm`^`f>+C% z#GA=m$XmtR%-hL3#5>Kq!fWI`;60%cZAClLK6C^fM<>xabRn&wjr3T08eLDXqPIX4 zI6+^c8|jDiZ;XJkXJkwS>`gk;pHVU+n6b6QNM( zDhw0$5atLK!b)MCaFKAc@SyOb@Q(1g$V%ibiV>xX3PpO+L{YtHqv(L>qUf&ZrGCXGY-MHTYt_Z7 zua(wnywyUhZB{3(Zdv^%wiAbnlf;9?RpOcAb>ar`*W$<4*4Ba6@z#abmDV$@H&`FB zzG?kjVlRo5WJpw!agrsHosx@^pQIMj0BM4Bh_qH(Fa1<{PWsTs!Y0rr$wpx_)@F&# z9-C`6Pi*aNqiu6+%WP-ZZnQmRd(Td27igDar?Q)5_mSNZyW94(y`O!Oz0&?&`_=Zx z?C&@T9D*D&97-IfJACYL)}h(a&atE80LL1~C64o_VtM9jdmVR-5 z8o&8|hy9xUz5TQO$M|pZ|0cj9ATFRdU_ro%fM50dM=C?)+tOI_F>qCaDI4~aDDiy@M{s` zh=hphh)ofD&=_UR zl9-bfyeQb8@4&SoriT^+WA_X zLtJj$jJT6s_+1jajO((m%Wqvfb*=2Wz3b22BD$4!+t}@1_n_|T?yI}s?%~@*(PL$g z#-6gC!+Ngh+1Sgcm!j9IUf;$0#jE1i#@|f{NhnF!obXekJkglABk@^M*Q9rn8j|Vc zwB%{YXH)D_`ll>TxtZ#lT9mpewK**|ZFE{gIzK%#eNOtd49^T@#)gdM-kp2D)B9+q zII~~o;>_=|LbHrnd$W1jy|d?K-{=$2N8e{x4$8^MnV0izZcy%s+|T<8`sVgs()WkF z=)5s`C;Hj<8`^JUzo+?$`Lpt`_Ydx0)&KAS>43olHVk++FnQqIfsKQr2E8-rbb(ue zwqSRmuy8=(+QKJqrM&gQTXzP>4z3$~_3fayYu-LJ#BGRf$o`?yp^BlO4&x6SIBesv zSBf0PD#erG8N-(jZ&oHL7bqX7;#KoiP3j)%x$3)`9-6tDdqq8p<`p#+_bRS0eyB~- zF4jIONiSJm@~kwcbY1Cd-9X*eGEv#EvORh`eX0IvxmS5j`2|Cmq0Z1aqU(qcMl>5U zjcY5Yinl9vS31C#(lb@TRqs{Zs_t36r26^D0V6-Hv8~b9oEa55YU-%FwW+mhMl++8 zqYsbq88czbt#=aMSvi&(JACZnaem__jk_~GZT$KPmJ>=Rd^u4*aqh%l-yQhw-btR5 z#!tHaUfO$`>Lhi>x@(iWOCfDO*QR{>GJ9I(_hU{&iLZ}==T@C z|9Ylo=Gj@DW-Xg7n600EbxyB2>*w0et(|*&UiQ45^JVj=&Hwd-p&y*8?^wTlf#rg# z1&s@{7Vci;zi9TNSBr}mUs;m4Wb0CorBjza`%v}a#bxo!wk-EtK7IL%71|YFtxQ|F zb5-D~`i}%3Re$uu>H(`ytchE*VXfQR>1%&qr(f5&K5zZe4V^b^*yyow)+XMjs!dIs z2X8+Aaq`Ezw?u4Nxz%aw^lfNc)wcVe4EyBj_RQ^vKaKlz%VzQtMh372JAL?5Grg&uYQ0293d&a0xlrV@syZ){Ter*aSIy%7lvD`ggRLw+!NYXtYCO( ze8K6t$*ULG4->6|Mz+aRgW|g^hbCKF{|vk2Iic~8@+j6C$`mjeF42f$-krdkjCh35 zgfN8g2@wz?Bt%4r1tFG%SP>#7#F`KZAyPtY2(cx^ju3l790+kF#EB4RLR<)OCB%&o zcS1Y}@g&5H5N|?cg!mBRONbvK{)7Y&5=cl8A;E-%5E4pA7$M<=L=X~5NE9J*LZbN- zc#PmZu&s?&rGRbc3#AR%4QRPmSFWU>A{oJ@5i_tE2zk~3)e!Gt<(sSwdkqEZy`VYN zv$s!H-pI;)T#aRK8&(#y$oRUdfg~m_xc&w!V}E01St8I9NN}3v&A@2jT5FPh>{!{V zMJD=*TV$di{ImcC#`h)qNm^v0pVTb-5&iHd+ujfBR{{xDv)l-b2GXl0*&k9+Aj6Lb zP+)w2qMw6V4j}r$PX?qufanLm7MNtBAN){2XeF)J{)*@s5N|29P4#nv>4Ne?F zf$>APu(F?74kP;cx5z}lfEKwCI1Nq)LV^C_M8BXGndld6mLsr!C6J)T1=>g86s{g8 z5KZ!ZRt_`Ek%L${yhT3F%8pvT|gLyo!~h%(5Ksyq-;gv+d<$L9WD+2Pl4U z5y6ST1!NBu5WHrpj6FXXc31`<0+PYc&SrHLAwJ~>U&qK8y(#w=K935G$-8J!0*59egz>|#AIdJ#TZ z9Os3hzZt&|oOvEtiCI?)x5WkT4z9<$0#pH^>rZ)9^2N346?ixo--a%p7SL0Apn4eWet;A3hb`&X3neG z#8J~tj9{sWvB`y*+GhcCwOj0O^)U_hT-~a#Io4NhW zw4a&wGt+)%+P^ime*^sUH;w<~cjo`Q_e{y(mIJ7B^?p4`(AzoHGai4D65#AZ`)vP8 ZHl^K?|Eatbmn98kvQ=ZH_`(MK6F5dtE literal 0 HcmV?d00001 diff --git a/test/pleroma/upload/filter/exif/strip_location_test.exs b/test/pleroma/upload/filter/exif/strip_location_test.exs index 01ca37711..3291e7aa6 100644 --- a/test/pleroma/upload/filter/exif/strip_location_test.exs +++ b/test/pleroma/upload/filter/exif/strip_location_test.exs @@ -13,9 +13,11 @@ defmodule Pleroma.Upload.Filter.Exif.StripLocationTest do {"DSCN0010.jpg", "image/jpeg"}, {"DSCN0010.png", "image/png"}, {"strip_location_gps.webp", "image/webp"}, - {"strip_location_gps.heic", "image/heic"} + {"strip_location_gps.heic", "image/heic"}, + {"strip_location_gps_grid.heic", "image/heic"}, + {"strip_location_gps.tiff", "image/tiff"} ] do - test "strips GPS data from #{content_type}" do + test "strips GPS data from #{fixture} (#{content_type})" do upload = upload_copy(unquote(fixture), unquote(content_type)) assert gps_tags(upload.tempfile) =~ "GPS Latitude" @@ -40,6 +42,12 @@ defmodule Pleroma.Upload.Filter.Exif.StripLocationTest do assert File.read!(upload.tempfile) == original end + test "accepts PNG files with data after IEND" do + upload = upload_copy("image_with_stray_data_after.png", "image/png") + + assert {:ok, _} = StripLocation.filter(upload) + end + test "ignores non-image uploads" do upload = upload_copy("DSCN0010.jpg", "application/octet-stream") From ce8b2276c290240205a074fdc0771e8bf4b3db92 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 27 Oct 2026 18:14:34 +0400 Subject: [PATCH 10/51] fix(migration): leave ConfigDB upload configs without filters alone Keyword.update/4 added filters: [] to a Pleroma.Upload ConfigDB entry without a :filters key, which would hide the filters from the config file, GPS stripping included. Only rewrite lists that change. --- ...7000000_upload_filter_exiftool_to_exif.exs | 26 +++++++++---------- .../upload_filter_exiftool_to_exif_test.exs | 9 +++++++ 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/priv/repo/migrations/20261027000000_upload_filter_exiftool_to_exif.exs b/priv/repo/migrations/20261027000000_upload_filter_exiftool_to_exif.exs index 62392f957..4e737e859 100644 --- a/priv/repo/migrations/20261027000000_upload_filter_exiftool_to_exif.exs +++ b/priv/repo/migrations/20261027000000_upload_filter_exiftool_to_exif.exs @@ -22,20 +22,20 @@ defmodule Pleroma.Repo.Migrations.UploadFilterExiftoolToExif do def down, do: rename_filters(@down) + # Only touch configs that list filters: adding an empty list would hide the + # filters from the config file. defp rename_filters(renames) do - case ConfigDB.get_by_params(%{group: :pleroma, key: Pleroma.Upload}) do - %{value: value} -> - new_value = - Keyword.update(value, :filters, [], fn filters -> - filters - |> Enum.map(&Map.get(renames, &1, &1)) - |> Enum.uniq() - end) - - ConfigDB.update_or_create(%{group: :pleroma, key: Pleroma.Upload, value: new_value}) - - _ -> - nil + with %{value: value} <- ConfigDB.get_by_params(%{group: :pleroma, key: Pleroma.Upload}), + filters when is_list(filters) <- value[:filters], + new_filters when new_filters != filters <- + filters |> Enum.map(&Map.get(renames, &1, &1)) |> Enum.uniq() do + ConfigDB.update_or_create(%{ + group: :pleroma, + key: Pleroma.Upload, + value: Keyword.put(value, :filters, new_filters) + }) + else + _ -> nil end end end diff --git a/test/pleroma/repo/migrations/upload_filter_exiftool_to_exif_test.exs b/test/pleroma/repo/migrations/upload_filter_exiftool_to_exif_test.exs index 9d39c6ea6..0fdf26961 100644 --- a/test/pleroma/repo/migrations/upload_filter_exiftool_to_exif_test.exs +++ b/test/pleroma/repo/migrations/upload_filter_exiftool_to_exif_test.exs @@ -61,4 +61,13 @@ defmodule Pleroma.Repo.Migrations.UploadFilterExiftoolToExifTest do assert migration.up() == nil assert ConfigDB.get_by_params(%{group: :pleroma, key: Pleroma.Upload}) == nil end + + test "up/0 does not add filters to a database config without them", %{migration: migration} do + insert(:config, group: :pleroma, key: Pleroma.Upload, value: [uploader: Pleroma.Uploaders.S3]) + + migration.up() + + value = ConfigDB.get_by_params(%{group: :pleroma, key: Pleroma.Upload}).value + assert value == [uploader: Pleroma.Uploaders.S3] + end end From e083a299c340096299184c71ec3056437690cd9c Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 27 Oct 2026 18:14:34 +0400 Subject: [PATCH 11/51] docs(upload): mention TIFF and the XMP limit in the Exif filter docs --- docs/configuration/cheatsheet.md | 2 +- lib/pleroma/upload/filter/exif/read_description.ex | 2 +- lib/pleroma/upload/filter/exif/strip_location.ex | 2 +- test/pleroma/upload/filter/exif/read_description_test.exs | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index 2d525ad63..30e3e113b 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -704,7 +704,7 @@ No specific configuration. #### Pleroma.Upload.Filter.Exif.StripLocation -This filter strips GPS and location metadata from JPEG, WebP, HEIC and AVIF images, leaving color profiles and attributes intact. For PNG images it removes all text and EXIF metadata chunks. Images are not re-encoded, and other formats are left unchanged. +This filter strips GPS and location metadata from JPEG, WebP, HEIC, AVIF and TIFF images, leaving color profiles and attributes intact. For PNG images it removes all text and EXIF metadata chunks. Images are not re-encoded, and other formats are left unchanged. It replaces the deprecated `Pleroma.Upload.Filter.Exiftool.StripLocation`, which no longer needs exiftool. diff --git a/lib/pleroma/upload/filter/exif/read_description.ex b/lib/pleroma/upload/filter/exif/read_description.ex index 654422b1b..96b219042 100644 --- a/lib/pleroma/upload/filter/exif/read_description.ex +++ b/lib/pleroma/upload/filter/exif/read_description.ex @@ -6,7 +6,7 @@ defmodule Pleroma.Upload.Filter.Exif.ReadDescription do @moduledoc """ Reads a description from image metadata when the upload has none yet. - Checks EXIF ImageDescription, then IPTC Caption-Abstract, then XMP. Blank + Checks EXIF ImageDescription, then IPTC Caption-Abstract (and XMP for GIF). Blank values and values longer than `[:instance, :description_limit]` are ignored. """ diff --git a/lib/pleroma/upload/filter/exif/strip_location.ex b/lib/pleroma/upload/filter/exif/strip_location.ex index 2e2b79f4c..622f8e335 100644 --- a/lib/pleroma/upload/filter/exif/strip_location.ex +++ b/lib/pleroma/upload/filter/exif/strip_location.ex @@ -6,7 +6,7 @@ defmodule Pleroma.Upload.Filter.Exif.StripLocation do @moduledoc """ Strips GPS metadata from uploaded images and overwrites the file in place. - Supports JPEG, PNG, WebP and ISO-BMFF images (HEIC, AVIF). This is lossless: + Supports JPEG, PNG, WebP, TIFF and ISO-BMFF images (HEIC, AVIF). This is lossless: the image data is not re-encoded. Other image types are left untouched. """ diff --git a/test/pleroma/upload/filter/exif/read_description_test.exs b/test/pleroma/upload/filter/exif/read_description_test.exs index 15d2944cd..07e3ce710 100644 --- a/test/pleroma/upload/filter/exif/read_description_test.exs +++ b/test/pleroma/upload/filter/exif/read_description_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2021 Pleroma Authors +# Copyright © 2017-2026 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Upload.Filter.Exif.ReadDescriptionTest do From 5a5f4794d3efdc13efed1954a7cd88293f1d955c Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 27 Oct 2026 20:45:32 +0400 Subject: [PATCH 12/51] perf(federation): always prefer the shared inbox Followers-only posts and DMs went to each recipient's personal inbox, so a followers-only post was sent once per follower on the same server. Mastodon, GoToSocial and Iceshrimp.NET always use the shared inbox. Only activities with bcc or bto still go to personal inboxes. Based on Akkoma a80444041. Co-authored-by: Oneric --- changelog.d/shared-inbox.change | 1 + lib/pleroma/web/activity_pub/publisher.ex | 38 +++++------------ .../web/activity_pub/publisher_test.exs | 42 ++++++++++++++++++- 3 files changed, 52 insertions(+), 29 deletions(-) create mode 100644 changelog.d/shared-inbox.change diff --git a/changelog.d/shared-inbox.change b/changelog.d/shared-inbox.change new file mode 100644 index 000000000..0cb74711e --- /dev/null +++ b/changelog.d/shared-inbox.change @@ -0,0 +1 @@ +Deliver to shared inboxes whenever possible, including followers-only posts and DMs, so each server gets one copy diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex index 17fae2a86..89c8168d4 100644 --- a/lib/pleroma/web/activity_pub/publisher.ex +++ b/lib/pleroma/web/activity_pub/publisher.ex @@ -281,36 +281,18 @@ defmodule Pleroma.Web.ActivityPub.Publisher do defp maybe_use_sharedinbox(%User{shared_inbox: shared_inbox}), do: shared_inbox @doc """ - Determine a user inbox to use based on heuristics. These heuristics - are based on an approximation of the ``sharedInbox`` rules in the - [ActivityPub specification][ap-sharedinbox]. + Determines the inbox to deliver an activity to. - Please do not edit this function (or its children) without reading - the spec, as editing the code is likely to introduce some breakage - without some familiarity. - - [ap-sharedinbox]: https://www.w3.org/TR/activitypub/#shared-inbox-delivery + Like Mastodon, GoToSocial and Akkoma, this always prefers the shared inbox, + which saves duplicate deliveries to the same server. Only activities with + `bcc` or `bto` go to personal inboxes, because the receiver rebuilds that + addressing from the inbox the activity arrived at. """ - def determine_inbox( - %Activity{data: activity_data}, - %User{inbox: inbox} = user - ) do - to = activity_data["to"] || [] - cc = activity_data["cc"] || [] - type = activity_data["type"] - - cond do - type == "Delete" -> - maybe_use_sharedinbox(user) - - Pleroma.Constants.as_public() in to || Pleroma.Constants.as_public() in cc -> - maybe_use_sharedinbox(user) - - length(to) + length(cc) > 1 -> - maybe_use_sharedinbox(user) - - true -> - inbox + def determine_inbox(%Activity{data: activity_data}, %User{inbox: inbox} = user) do + if Enum.empty?(activity_data["bcc"] || []) and Enum.empty?(activity_data["bto"] || []) do + maybe_use_sharedinbox(user) + else + inbox end end diff --git a/test/pleroma/web/activity_pub/publisher_test.exs b/test/pleroma/web/activity_pub/publisher_test.exs index e8b6ecf38..0c9035b4e 100644 --- a/test/pleroma/web/activity_pub/publisher_test.exs +++ b/test/pleroma/web/activity_pub/publisher_test.exs @@ -137,7 +137,7 @@ defmodule Pleroma.Web.ActivityPub.PublisherTest do assert Publisher.determine_inbox(activity, user) == "http://example.com/inbox" end - test "it returns inbox for messages involving single recipients in total" do + test "it returns sharedInbox for messages to a single recipient" do user = insert(:user, %{ shared_inbox: "http://example.com/inbox", @@ -148,6 +148,46 @@ defmodule Pleroma.Web.ActivityPub.PublisherTest do data: %{"to" => [user.ap_id], "cc" => []} } + assert Publisher.determine_inbox(activity, user) == "http://example.com/inbox" + end + + test "it returns sharedInbox for followers-only messages" do + user = + insert(:user, %{ + shared_inbox: "http://example.com/inbox", + inbox: "http://example.com/personal-inbox" + }) + + author = insert(:user) + + activity = %Activity{ + data: %{"to" => [author.follower_address], "cc" => []} + } + + assert Publisher.determine_inbox(activity, user) == "http://example.com/inbox" + end + + test "it returns inbox for messages with bcc or bto, which needs personal inboxes" do + user = + insert(:user, %{ + shared_inbox: "http://example.com/inbox", + inbox: "http://example.com/personal-inbox" + }) + + for field <- ["bcc", "bto"] do + activity = %Activity{ + data: %{"to" => [@as_public], "cc" => [], field => [user.ap_id]} + } + + assert Publisher.determine_inbox(activity, user) == "http://example.com/personal-inbox" + end + end + + test "it returns inbox when the user has no sharedInbox" do + user = insert(:user, %{shared_inbox: nil, inbox: "http://example.com/personal-inbox"}) + + activity = %Activity{data: %{"to" => [@as_public], "cc" => []}} + assert Publisher.determine_inbox(activity, user) == "http://example.com/personal-inbox" end end From f57e97bf18a24799bb8924218838b52ea51cec8c Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 27 Oct 2026 21:46:40 +0400 Subject: [PATCH 13/51] feat(signature): add a freshness and coverage policy for incoming signatures Signature.check_signature_policy/2 requires date or (created) to be signed and within signature_max_age (2 h) / signature_max_clock_skew (1 h), and requires a signed, hashed digest for requests with a body. Based on Akkoma's http_signatures fork (required headers, max_sig_age) and Mastodon's SignedRequest (clock skew margin). --- config/config.exs | 3 + config/description.exs | 21 +++++ docs/configuration/cheatsheet.md | 3 + lib/pleroma/signature.ex | 78 +++++++++++++++++++ test/pleroma/signature_test.exs | 129 +++++++++++++++++++++++++++++++ 5 files changed, 234 insertions(+) diff --git a/config/config.exs b/config/config.exs index 1b8baed1f..617b4b8e5 100644 --- a/config/config.exs +++ b/config/config.exs @@ -373,6 +373,9 @@ config :pleroma, :activitypub, note_replies_output_limit: 5, sign_object_fetches: true, authorized_fetch_mode: false, + signature_policy: :enforce, + signature_max_age: 7200, + signature_max_clock_skew: 3600, client_api_enabled: false, anonymize_reporter: false, anonymize_reporter_local_nickname: "" diff --git a/config/description.exs b/config/description.exs index f435628aa..1b065fe6c 100644 --- a/config/description.exs +++ b/config/description.exs @@ -1788,6 +1788,27 @@ config :pleroma, :config_description, [ description: "List of IPs (CIDR format accepted) to exempt from HTTP Signatures requirement (for example to allow debugging, you shouldn't otherwise need this)" }, + %{ + key: :signature_policy, + type: {:dropdown, :atom}, + description: + "What to do with incoming HTTP signatures that do not sign the date (or `(created)`), are outside the allowed time window, or do not sign the digest of a request body. `:enforce` treats them as invalid, `:log` only logs them", + suggestions: [:enforce, :log] + }, + %{ + key: :signature_max_age, + type: :integer, + description: + "How old, in seconds, the signed date of an incoming HTTP signature can be. Default: 7200 (2 hours)", + suggestions: [7200] + }, + %{ + key: :signature_max_clock_skew, + type: :integer, + description: + "How far, in seconds, the signed date of an incoming HTTP signature can be in the future. Default: 3600 (1 hour). A `(created)` time in the future is always rejected", + suggestions: [3600] + }, %{ key: :note_replies_output_limit, type: :integer, diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index 4865ff0aa..d49aa51f7 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -286,6 +286,9 @@ Notes: * `sign_object_fetches`: Sign object fetches with HTTP signatures * `authorized_fetch_mode`: Require HTTP signatures for AP fetches * `authorized_fetch_mode_exceptions`: List of IPs (CIDR format accepted) to exempt from HTTP Signatures requirement (for example to allow debugging, you shouldn't otherwise need this) +* `signature_policy`: What to do with incoming HTTP signatures that do not sign the `date` header (or `(created)`), whose signed date is outside the window below, or that do not sign the `digest` header of a request with a body (or whose body Pleroma did not hash). `:enforce` (default) treats them as invalid, `:log` only logs a warning. +* `signature_max_age`: How old, in seconds, the signed date of an incoming HTTP signature can be. Default: `7200` (2 hours). +* `signature_max_clock_skew`: How far, in seconds, the signed `date` of an incoming HTTP signature can be in the future. Default: `3600` (1 hour). A `(created)` time in the future is always rejected. ## Pleroma.User diff --git a/lib/pleroma/signature.ex b/lib/pleroma/signature.ex index 382cf9db3..e85c2aa5e 100644 --- a/lib/pleroma/signature.ex +++ b/lib/pleroma/signature.ex @@ -95,6 +95,84 @@ defmodule Pleroma.Signature do Timex.format!(date, "{WDshort}, {0D} {Mshort} {YYYY} {h24}:{m}:{s} GMT") end + @doc """ + Checks the parts of an incoming signature that the cryptographic check does + not cover, so that a captured request cannot be replayed later or with a + different body. + + The signature must cover `date` or `(created)`, and the signed time must be + within the configured window. Requests other than GET and HEAD must also + sign `digest`, and DigestPlug must have hashed their body. Based on Akkoma's http_signatures fork and Mastodon's + SignedRequest. + """ + @spec check_signature_policy(Plug.Conn.t(), DateTime.t()) :: :ok | {:error, atom()} + def check_signature_policy(%Plug.Conn{} = conn, now \\ DateTime.utc_now()) do + signature = HTTPSignatures.signature_for_conn(conn) + signed = Map.get(signature, "headers", []) + + cond do + "date" not in signed and "(created)" not in signed -> + {:error, :date_not_signed} + + conn.method not in ["GET", "HEAD"] and "digest" not in signed -> + {:error, :digest_not_signed} + + # The plug puts the digest computed by DigestPlug into the header. With + # no computed digest (e.g. multipart), nothing checked the body. + conn.method not in ["GET", "HEAD"] and is_nil(conn.assigns[:digest]) -> + {:error, :body_not_hashed} + + true -> + with :ok <- check_signed_date(conn, signed, now) do + check_signed_created(signature, signed, now) + end + end + end + + defp check_signed_date(conn, signed, now) do + if "date" in signed do + # Exactly one header: a second one could make this check and the + # signature check see different dates. + with [date] <- Plug.Conn.get_req_header(conn, "date"), + {:ok, %DateTime{} = date} <- Timex.parse(date, "{RFC1123}") do + check_time_window(date, now) + else + _ -> {:error, :invalid_date} + end + else + :ok + end + end + + defp check_signed_created(signature, signed, now) do + if "(created)" in signed do + with created when is_binary(created) <- signature["created"], + {created, ""} <- Integer.parse(created), + {:ok, created} <- DateTime.from_unix(created) do + check_time_window(created, now) + else + _ -> {:error, :invalid_date} + end + else + :ok + end + end + + defp check_time_window(signed_at, now) do + age = DateTime.diff(now, signed_at) + + cond do + age > Pleroma.Config.get([:activitypub, :signature_max_age], 7200) -> + {:error, :date_too_old} + + -age > Pleroma.Config.get([:activitypub, :signature_max_clock_skew], 3600) -> + {:error, :date_in_future} + + true -> + :ok + end + end + @spec validate_signature(Plug.Conn.t(), String.t()) :: boolean() def validate_signature(%Plug.Conn{} = conn, request_target) do # Newer drafts for HTTP signatures now use @request-target instead of the diff --git a/test/pleroma/signature_test.exs b/test/pleroma/signature_test.exs index 0c7c4c840..18a2a1ddc 100644 --- a/test/pleroma/signature_test.exs +++ b/test/pleroma/signature_test.exs @@ -116,6 +116,135 @@ defmodule Pleroma.SignatureTest do end end + describe "check_signature_policy/2" do + @now ~U[2026-10-27 12:00:00Z] + + defp policy_conn(method, signed_headers, extra_headers, extra_params \\ "") do + signature = + "keyId=\"https://remote.example/users/a#main-key\",headers=\"#{signed_headers}\"" <> + extra_params <> ",signature=\"fake\"" + + %Plug.Conn{ + method: method, + req_headers: [{"signature", signature}, {"host", "pleroma.example"} | extra_headers], + assigns: %{digest: "SHA-256=abc"} + } + end + + defp date_header(seconds_ago) do + date = DateTime.add(@now, -seconds_ago) |> DateTime.to_naive() + {"date", Signature.signed_date(date)} + end + + test "accepts a fresh signed GET (Pleroma, Mastodon, Misskey style)" do + for headers <- ["(request-target) host date", "(request-target) host date accept"] do + conn = policy_conn("GET", headers, [date_header(10)]) + assert Signature.check_signature_policy(conn, @now) == :ok + end + end + + test "accepts a fresh signed POST covering the digest" do + conn = + policy_conn("POST", "(request-target) host content-length digest date", [ + date_header(10), + {"digest", "SHA-256=abc"} + ]) + + assert Signature.check_signature_policy(conn, @now) == :ok + end + + test "accepts (created) instead of date (GoToSocial style)" do + created = DateTime.to_unix(@now) - 10 + + conn = + policy_conn( + "POST", + "(request-target) (created) host digest", + [{"digest", "SHA-256=abc"}], + ",algorithm=\"hs2019\",created=#{created}" + ) + + assert Signature.check_signature_policy(conn, @now) == :ok + end + + test "rejects signatures that cover neither date nor (created)" do + conn = policy_conn("GET", "(request-target) host", [date_header(10)]) + assert Signature.check_signature_policy(conn, @now) == {:error, :date_not_signed} + end + + test "rejects a body request whose digest is not signed" do + conn = + policy_conn("POST", "(request-target) host date", [ + date_header(10), + {"digest", "SHA-256=abc"} + ]) + + assert Signature.check_signature_policy(conn, @now) == {:error, :digest_not_signed} + end + + test "rejects a signed date that is missing or unparsable" do + conn = policy_conn("GET", "(request-target) host date", []) + assert Signature.check_signature_policy(conn, @now) == {:error, :invalid_date} + + conn = policy_conn("GET", "(request-target) host date", [{"date", "yesterday"}]) + assert Signature.check_signature_policy(conn, @now) == {:error, :invalid_date} + end + + test "rejects more than one date header" do + conn = + policy_conn("GET", "(request-target) host date", [date_header(10), date_header(9000)]) + + assert Signature.check_signature_policy(conn, @now) == {:error, :invalid_date} + end + + test "rejects a signed digest when the body was not hashed" do + conn = + policy_conn("POST", "(request-target) host date digest", [ + date_header(10), + {"digest", "SHA-256=abc"} + ]) + |> Map.put(:assigns, %{}) + + assert Signature.check_signature_policy(conn, @now) == {:error, :body_not_hashed} + end + + test "rejects a signed date that is too old or too far in the future" do + clear_config([:activitypub, :signature_max_age], 7200) + clear_config([:activitypub, :signature_max_clock_skew], 3600) + + conn = policy_conn("GET", "(request-target) host date", [date_header(7200)]) + assert Signature.check_signature_policy(conn, @now) == :ok + + conn = policy_conn("GET", "(request-target) host date", [date_header(7201)]) + assert Signature.check_signature_policy(conn, @now) == {:error, :date_too_old} + + conn = policy_conn("GET", "(request-target) host date", [date_header(-3600)]) + assert Signature.check_signature_policy(conn, @now) == :ok + + conn = policy_conn("GET", "(request-target) host date", [date_header(-3601)]) + assert Signature.check_signature_policy(conn, @now) == {:error, :date_in_future} + end + + test "rejects a (created) timestamp that is too old" do + clear_config([:activitypub, :signature_max_age], 60) + created = DateTime.to_unix(@now) - 61 + + conn = + policy_conn("GET", "(request-target) (created) host", [], ",created=#{created}") + + assert Signature.check_signature_policy(conn, @now) == {:error, :date_too_old} + end + + test "uses the cavage default of a signed date when headers are not given" do + conn = %Plug.Conn{ + method: "GET", + req_headers: [{"signature", "keyId=\"https://remote.example/users/a#main-key\""}] + } + + assert Signature.check_signature_policy(conn, @now) == {:error, :invalid_date} + end + end + describe "key_id_to_actor_id/1" do test "it properly deduces the actor id for misskey" do assert Signature.key_id_to_actor_id("https://example.com/users/1234/publickey") == From e5481b4c28497a9d655feb706406dc06d11aa991 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 27 Oct 2026 21:46:40 +0400 Subject: [PATCH 14/51] fix(federation): enforce the signature policy on incoming requests HTTPSignaturePlug checks the policy before the cryptographic check. A signed GET that fails it counts as unsigned; the inbox answers 401 to a POST that fails it instead of queueing a failed-signature retry, which only re-checks the signature. signature_policy: :log only logs. Based on Akkoma's HTTP signature hardening. --- changelog.d/signature-freshness.security | 1 + .../activity_pub/activity_pub_controller.ex | 29 +++-- lib/pleroma/web/plugs/http_signature_plug.ex | 46 +++++-- .../activity_pub_controller_test.exs | 75 ++++++++--- .../plugs/http_signature_plug_mrf_test.exs | 6 +- .../plugs/http_signature_plug_policy_test.exs | 119 ++++++++++++++++++ .../web/plugs/http_signature_plug_test.exs | 112 +++++++++++++---- 7 files changed, 329 insertions(+), 59 deletions(-) create mode 100644 changelog.d/signature-freshness.security create mode 100644 test/pleroma/web/plugs/http_signature_plug_policy_test.exs diff --git a/changelog.d/signature-freshness.security b/changelog.d/signature-freshness.security new file mode 100644 index 000000000..a314c403e --- /dev/null +++ b/changelog.d/signature-freshness.security @@ -0,0 +1 @@ +Reject incoming HTTP signatures that do not sign the date, are older than 2 hours (or more than 1 hour in the future), or do not sign the digest of a request body, so captured requests cannot be replayed. Set `config :pleroma, :activitypub, signature_policy: :log` to only log these diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index e9b0a0d6d..dd5f6028f 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -386,18 +386,25 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do end def inbox(%{assigns: %{valid_signature: false}} = conn, params) do - if unsupported_http_message_signature?(conn) do - reject_unsupported_http_message_signature(conn) - else - Federator.incoming_failed_signature_ap_doc(%{ - method: conn.method, - req_headers: conn.req_headers, - request_path: conn.request_path, - params: params, - query_string: conn.query_string - }) + cond do + unsupported_http_message_signature?(conn) -> + reject_unsupported_http_message_signature(conn) - json(conn, "ok") + reason = conn.assigns[:signature_error] -> + conn + |> put_status(:unauthorized) + |> json("error, invalid HTTP signature (#{reason})") + + true -> + Federator.incoming_failed_signature_ap_doc(%{ + method: conn.method, + req_headers: conn.req_headers, + request_path: conn.request_path, + params: params, + query_string: conn.query_string + }) + + json(conn, "ok") end end diff --git a/lib/pleroma/web/plugs/http_signature_plug.ex b/lib/pleroma/web/plugs/http_signature_plug.ex index 1f7cb51f3..be1c9c2ee 100644 --- a/lib/pleroma/web/plugs/http_signature_plug.ex +++ b/lib/pleroma/web/plugs/http_signature_plug.ex @@ -45,20 +45,52 @@ defmodule Pleroma.Web.Plugs.HTTPSignaturePlug do defp maybe_assign_valid_signature(conn) do if has_signature_header?(conn) do - # we replace the digest header with the one we computed in DigestPlug - conn = - case conn do - %{assigns: %{digest: digest}} = conn -> put_req_header(conn, "digest", digest) - conn -> conn - end + conn = put_computed_digest(conn) - assign(conn, :valid_signature, Signature.validate_signature(conn)) + case check_signature_policy(conn) do + :ok -> + assign(conn, :valid_signature, Signature.validate_signature(conn)) + + {:error, reason} -> + reject_signature(conn, reason) + end else Logger.debug("No signature header!") conn end end + # we replace the digest header with the one we computed in DigestPlug + defp put_computed_digest(%{assigns: %{digest: digest}} = conn), + do: put_req_header(conn, "digest", digest) + + defp put_computed_digest(conn), do: conn + + defp check_signature_policy(conn) do + with {:error, reason} <- Signature.check_signature_policy(conn) do + if Pleroma.Config.get([:activitypub, :signature_policy], :enforce) == :log do + Logger.warning( + "HTTP signature policy violation (#{reason}) for #{conn.method} #{conn.request_path}, allowed by config" + ) + + :ok + else + {:error, reason} + end + end + end + + # The request is treated as badly signed. The reason is kept so the inbox + # can refuse it instead of queueing a failed-signature retry, which only + # checks the cryptographic signature. + defp reject_signature(conn, reason) do + Logger.debug("Rejecting HTTP signature: #{reason}") + + conn + |> assign(:valid_signature, false) + |> assign(:signature_error, reason) + end + defp maybe_assign_actor_id(%{assigns: %{valid_signature: true}} = conn) do adapter = Application.get_env(:http_signatures, :adapter) diff --git a/test/pleroma/web/activity_pub/activity_pub_controller_test.exs b/test/pleroma/web/activity_pub/activity_pub_controller_test.exs index 13a4ee108..606d47c8d 100644 --- a/test/pleroma/web/activity_pub/activity_pub_controller_test.exs +++ b/test/pleroma/web/activity_pub/activity_pub_controller_test.exs @@ -9,6 +9,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do alias Pleroma.Activity alias Pleroma.Delivery alias Pleroma.Object + alias Pleroma.Signature alias Pleroma.Tests.ObanHelpers alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub @@ -49,6 +50,16 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do |> put_req_header("signature", "keyId=\"#{actor_id}#main-key\"") end + defp put_fake_signature(conn, actor_id, date \\ Signature.signed_date()) do + conn + |> put_req_header("date", date) + |> put_req_header("digest", "SHA-256=fake-digest") + |> put_req_header( + "signature", + "keyId=\"#{actor_id}#main-key\",algorithm=\"rsa-sha256\",headers=\"(request-target) host date digest content-type\",signature=\"fake-signature\"" + ) + end + defp expect_signature_retry_from(%User{} = signer) do signer_json = UserView.render("user.json", %{user: signer}) |> Map.delete("featured") @@ -792,12 +803,12 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do } } - Mox.expect(Pleroma.StubbedHTTPSignaturesMock, :validate_conn, fn _conn -> false end) + Mox.stub(Pleroma.StubbedHTTPSignaturesMock, :validate_conn, fn _conn -> false end) conn = conn |> put_req_header("content-type", "application/activity+json") - |> put_req_header("date", "Fri, 29 May 2026 13:33:31 GMT") + |> put_req_header("date", Signature.signed_date()) |> put_req_header("content-digest", "sha-256=:fake-digest:") |> put_req_header( "signature-input", @@ -832,12 +843,12 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do legacy_signature = "keyId=\"https://activitypubbot.example/user/ok#main-key\",algorithm=\"rsa-sha256\",headers=\"(request-target) host date digest content-type\",signature=\"fake-signature\"" - Mox.expect(Pleroma.StubbedHTTPSignaturesMock, :validate_conn, fn _conn -> false end) + Mox.stub(Pleroma.StubbedHTTPSignaturesMock, :validate_conn, fn _conn -> false end) conn = conn |> put_req_header("content-type", "application/activity+json") - |> put_req_header("date", "Fri, 29 May 2026 13:33:31 GMT") + |> put_req_header("date", Signature.signed_date()) |> put_req_header("digest", "SHA-256=fake-digest") |> put_req_header("signature", "sig1=:fake-signature:") |> Map.update!(:req_headers, &[{"signature", legacy_signature} | &1]) @@ -871,7 +882,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do conn = conn |> put_req_header("content-type", "application/activity+json") - |> put_req_header("date", "Fri, 29 May 2026 13:33:31 GMT") + |> put_req_header("date", Signature.signed_date()) |> put_req_header("digest", "SHA-256=fake-digest") |> put_req_header( "signature-input", @@ -881,12 +892,44 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do "signature", "keyId=\"https://activitypubbot.example/user/ok#main-key\",algorithm=\"rsa-sha256\",headers=\"(request-target) host date digest content-type\",signature=\"fake-signature\"" ) - |> post("/inbox", data) + |> post("/inbox", Jason.encode!(data)) assert "ok" == json_response(conn, 200) assert [_] = all_enqueued(worker: SignatureRetryWorker) end + test "refuses a replayed signed POST instead of queueing a retry", %{conn: conn} do + data = File.read!("test/fixtures/mastodon-post-activity.json") |> Jason.decode!() + stale = NaiveDateTime.add(NaiveDateTime.utc_now(), -3 * 3600) + + conn = + conn + |> put_req_header("content-type", "application/activity+json") + |> put_fake_signature(data["actor"], Signature.signed_date(stale)) + |> post("/inbox", Jason.encode!(data)) + + assert "error, invalid HTTP signature (date_too_old)" == json_response(conn, 401) + assert [] == all_enqueued(worker: SignatureRetryWorker) + assert [] == all_enqueued(worker: ReceiverWorker) + end + + test "refuses a signed POST that does not cover the digest", %{conn: conn} do + data = File.read!("test/fixtures/mastodon-post-activity.json") |> Jason.decode!() + + conn = + conn + |> put_req_header("content-type", "application/activity+json") + |> put_req_header("date", Signature.signed_date()) + |> put_req_header( + "signature", + "keyId=\"#{data["actor"]}#main-key\",headers=\"(request-target) host date\",signature=\"fake\"" + ) + |> post("/inbox", Jason.encode!(data)) + + assert "error, invalid HTTP signature (digest_not_signed)" == json_response(conn, 401) + assert [] == all_enqueued(worker: SignatureRetryWorker) + end + test "does not create a forged post after failed signature retry", %{conn: conn} do alice = insert(:user, local: false, ap_id: "https://one.com/users/alice") bob = insert(:user, local: false, ap_id: "https://two.com/users/bob") @@ -918,8 +961,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do conn |> assign(:valid_signature, false) |> put_req_header("content-type", "application/activity+json") - |> put_req_header("signature", "keyId=\"https://one.com/users/alice#main-key\"") - |> post("/inbox", data) + |> put_fake_signature(alice.ap_id) + |> post("/inbox", Jason.encode!(data)) assert "ok" == json_response(conn, 200) @@ -950,8 +993,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do conn |> assign(:valid_signature, false) |> put_req_header("content-type", "application/activity+json") - |> put_req_header("signature", "keyId=\"https://one.com/users/alice#main-key\"") - |> post("/inbox", data) + |> put_fake_signature(alice.ap_id) + |> post("/inbox", Jason.encode!(data)) assert "ok" == json_response(conn, 200) @@ -982,8 +1025,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do conn |> assign(:valid_signature, false) |> put_req_header("content-type", "application/activity+json") - |> put_req_header("signature", "keyId=\"https://one.com/users/alice#main-key\"") - |> post("/inbox", data) + |> put_fake_signature(alice.ap_id) + |> post("/inbox", Jason.encode!(data)) assert "ok" == json_response(conn, 200) @@ -1025,13 +1068,13 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do conn = conn |> put_req_header("content-type", "application/activity+json") - |> put_req_header("date", "Thu, 25 Jul 2024 13:33:31 GMT") + |> put_req_header("date", Signature.signed_date()) |> put_req_header("digest", "SHA-256=fake-digest") |> put_req_header( "signature", "keyId=\"#{alice.ap_id}#main-key\",algorithm=\"rsa-sha256\",headers=\"(request-target) host date digest content-type\",signature=\"fake-signature\"" ) - |> post("/inbox", data) + |> post("/inbox", Jason.encode!(data)) assert conn.assigns.valid_signature == false assert "ok" == json_response(conn, 200) @@ -1063,13 +1106,13 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do conn = conn |> put_req_header("content-type", "application/activity+json") - |> put_req_header("date", "Thu, 25 Jul 2024 13:33:31 GMT") + |> put_req_header("date", Signature.signed_date()) |> put_req_header("digest", "SHA-256=fake-digest") |> put_req_header( "signature", "keyId=\"#{alice.ap_id}#main-key\",algorithm=\"rsa-sha256\",headers=\"(request-target) host date digest content-type\",signature=\"fake-signature\"" ) - |> post("/inbox", data) + |> post("/inbox", Jason.encode!(data)) assert conn.assigns.valid_signature == false assert "ok" == json_response(conn, 200) diff --git a/test/pleroma/web/plugs/http_signature_plug_mrf_test.exs b/test/pleroma/web/plugs/http_signature_plug_mrf_test.exs index 1ee81b1a0..0101e4ab6 100644 --- a/test/pleroma/web/plugs/http_signature_plug_mrf_test.exs +++ b/test/pleroma/web/plugs/http_signature_plug_mrf_test.exs @@ -26,7 +26,11 @@ defmodule Pleroma.Web.Plugs.HTTPSignaturePlugMRFTest do |> expect(:validate_conn, fn _ -> true end) build_conn(:get, "/doesntmattter", %{"actor" => actor}) - |> put_req_header("signature", "keyId=\"#{actor}#main-key") + |> put_req_header("date", Pleroma.Signature.signed_date()) + |> put_req_header( + "signature", + "keyId=\"#{actor}#main-key\",headers=\"(request-target) host date\",signature=\"fake\"" + ) |> put_format("activity+json") |> HTTPSignaturePlug.call(%{}) end diff --git a/test/pleroma/web/plugs/http_signature_plug_policy_test.exs b/test/pleroma/web/plugs/http_signature_plug_policy_test.exs new file mode 100644 index 000000000..14279d4bc --- /dev/null +++ b/test/pleroma/web/plugs/http_signature_plug_policy_test.exs @@ -0,0 +1,119 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.Plugs.HTTPSignaturePlugPolicyTest do + # Not async: changes global config and uses global HTTP mocks. + use Pleroma.Web.ConnCase + + alias Pleroma.Keys + alias Pleroma.Object.Fetcher + alias Pleroma.Repo + alias Pleroma.StubbedHTTPSignaturesMock, as: HTTPSignaturesMock + alias Pleroma.User + alias Pleroma.Web.ActivityPub.InternalFetchActor + alias Pleroma.Web.ActivityPub.Publisher + alias Pleroma.Web.Plugs.DigestPlug + alias Pleroma.Web.Plugs.HTTPSignaturePlug + + import ExUnit.CaptureLog + import Mox + import Phoenix.Controller, only: [put_format: 2] + import Pleroma.Factory + import Plug.Conn + + defp incoming_conn(method, url, headers, body \\ "") do + %URI{host: host, path: path} = URI.parse(url) + + headers = Enum.map(headers, fn {name, value} -> {String.downcase(name), value} end) + + Plug.Test.conn(method, path, body) + |> Map.put(:req_headers, [{"host", host} | headers]) + |> put_format("activity+json") + end + + # Local users have no public_key column set; publish it like a remote copy would. + defp store_public_key(%User{} = user) do + {:ok, _, public_key} = Keys.keys_from_pem(user.keys) + entry = :public_key.pem_entry_encode(:SubjectPublicKeyInfo, public_key) + + user + |> Ecto.Changeset.change(public_key: :public_key.pem_encode([entry])) + |> Repo.update!() + |> User.set_cache() + end + + defp capture_request(method) do + test_pid = self() + + Tesla.Mock.mock_global(fn %{method: ^method, headers: headers, body: body} = env -> + send(test_pid, {:request, headers, body}) + %{env | status: 200, body: ""} + end) + end + + test "accepts deliveries signed by Pleroma's publisher" do + stub_with(Pleroma.Web.ActivityPub.TransmogrifierMock, Pleroma.Web.ActivityPub.Transmogrifier) + capture_request(:post) + + activity = insert(:note_activity) + store_public_key(User.get_by_ap_id(activity.data["actor"])) + inbox = "https://remote.example/inbox" + + {:ok, _} = + %{inbox: inbox, activity_id: activity.id} + |> Publisher.prepare_one() + |> Publisher.publish_one() + + assert_received {:request, headers, body} + + {:ok, _, conn} = + incoming_conn(:post, inbox, [{"content-length", "#{byte_size(body)}"} | headers], body) + |> DigestPlug.read_body([]) + + conn = HTTPSignaturePlug.call(conn, %{}) + + assert conn.assigns.valid_signature == true + refute conn.halted + end + + test "accepts object fetches signed by Pleroma's fetcher" do + clear_config([:activitypub, :sign_object_fetches], true) + capture_request(:get) + store_public_key(InternalFetchActor.get_actor()) + + url = "https://remote.example/objects/1" + Fetcher.fetch_and_contain_remote_object_from_id(url) + + assert_received {:request, headers, _body} + + conn = + incoming_conn(:get, url, headers) + |> HTTPSignaturePlug.call(%{}) + + assert conn.assigns.valid_signature == true + end + + test "only logs policy violations when the policy is set to :log" do + clear_config([:activitypub, :signature_policy], :log) + + HTTPSignaturesMock + |> expect(:validate_conn, fn _ -> true end) + + log = + capture_log(fn -> + conn = + incoming_conn(:post, "https://pleroma.example/inbox", [ + {"date", Pleroma.Signature.signed_date()}, + {"signature", + "keyId=\"https://remote.example/users/a#main-key\",headers=\"(request-target) host date\",signature=\"fake\""} + ]) + |> HTTPSignaturePlug.call(%{}) + + assert conn.assigns.valid_signature == true + refute conn.halted + end) + + assert log =~ "digest_not_signed" + end +end diff --git a/test/pleroma/web/plugs/http_signature_plug_test.exs b/test/pleroma/web/plugs/http_signature_plug_test.exs index 9d07270bb..bdab88939 100644 --- a/test/pleroma/web/plugs/http_signature_plug_test.exs +++ b/test/pleroma/web/plugs/http_signature_plug_test.exs @@ -13,6 +13,15 @@ defmodule Pleroma.Web.Plugs.HTTPSignaturePlugTest do import Phoenix.Controller, only: [put_format: 2] import Plug.Conn + defp put_signature(conn, actor, signed_headers \\ "(request-target) host date") do + conn + |> put_req_header("date", Pleroma.Signature.signed_date()) + |> put_req_header( + "signature", + "keyId=\"#{actor}#main-key\",headers=\"#{signed_headers}\",signature=\"fake\"" + ) + end + test "it calls HTTPSignatures to check validity if the actor signed it" do params = %{"actor" => "http://mastodon.example.org/users/admin"} conn = build_conn(:get, "/doesntmattter", params) @@ -22,10 +31,7 @@ defmodule Pleroma.Web.Plugs.HTTPSignaturePlugTest do conn = conn - |> put_req_header( - "signature", - "keyId=\"http://mastodon.example.org/users/admin#main-key" - ) + |> put_signature("http://mastodon.example.org/users/admin") |> put_format("activity+json") |> HTTPSignaturePlug.call(%{}) @@ -51,10 +57,7 @@ defmodule Pleroma.Web.Plugs.HTTPSignaturePlugTest do conn = orig_conn - |> put_req_header( - "signature", - "keyId=\"http://mastodon.example.org/users/admin#main-key" - ) + |> put_signature("http://mastodon.example.org/users/admin") |> HTTPSignaturePlug.call(%{}) assert conn.assigns.valid_signature == false @@ -71,10 +74,7 @@ defmodule Pleroma.Web.Plugs.HTTPSignaturePlugTest do conn = orig_conn - |> put_req_header( - "signature", - "keyId=\"http://mastodon.example.org/users/admin#main-key" - ) + |> put_signature("http://mastodon.example.org/users/admin") |> HTTPSignaturePlug.call(%{}) assert conn.assigns.valid_signature == true @@ -108,10 +108,7 @@ defmodule Pleroma.Web.Plugs.HTTPSignaturePlugTest do conn = conn |> Map.put(:remote_ip, {192, 168, 0, 1}) - |> put_req_header( - "signature", - "keyId=\"http://mastodon.example.org/users/admin#main-key" - ) + |> put_signature("http://mastodon.example.org/users/admin") |> HTTPSignaturePlug.call(%{}) assert conn.remote_ip == {192, 168, 0, 1} @@ -131,10 +128,7 @@ defmodule Pleroma.Web.Plugs.HTTPSignaturePlugTest do conn = build_conn(:get, "/doesntmattter", %{"actor" => "http://mastodon.example.org/users/admin"}) - |> put_req_header( - "signature", - "keyId=\"http://mastodon.example.org/users/admin#main-key" - ) + |> put_signature("http://mastodon.example.org/users/admin") |> put_format("activity+json") |> HTTPSignaturePlug.call(%{}) @@ -152,14 +146,84 @@ defmodule Pleroma.Web.Plugs.HTTPSignaturePlugTest do conn = build_conn(:get, "/doesntmattter", %{"actor" => "http://allowed.example.org/users/admin"}) - |> put_req_header( - "signature", - "keyId=\"http://allowed.example.org/users/admin#main-key" - ) + |> put_signature("http://allowed.example.org/users/admin") |> put_format("activity+json") |> HTTPSignaturePlug.call(%{}) assert conn.assigns.valid_signature == true assert conn.halted == false end + + describe "signature policy" do + test "treats a signed GET with a stale date as unsigned" do + HTTPSignaturesMock + |> expect(:validate_conn, 0, fn _ -> true end) + + stale = NaiveDateTime.add(NaiveDateTime.utc_now(), -3 * 3600) + + conn = + build_conn(:get, "/doesntmattter") + |> put_signature("http://mastodon.example.org/users/admin") + |> put_req_header("date", Pleroma.Signature.signed_date(stale)) + |> put_format("activity+json") + |> HTTPSignaturePlug.call(%{}) + + assert conn.assigns.valid_signature == false + assert conn.assigns.signature_error == :date_too_old + refute conn.halted + end + + test "marks a signed POST that does not cover the digest as invalid" do + HTTPSignaturesMock + |> expect(:validate_conn, 0, fn _ -> true end) + + conn = + build_conn(:post, "/inbox") + |> put_signature("http://mastodon.example.org/users/admin") + |> put_format("activity+json") + |> HTTPSignaturePlug.call(%{}) + + assert conn.assigns.valid_signature == false + assert conn.assigns.signature_error == :digest_not_signed + refute conn.halted + end + + test "uses the digest computed from the body for signed POSTs" do + HTTPSignaturesMock + |> expect(:validate_conn, fn conn -> + get_req_header(conn, "digest") == ["SHA-256=computed"] + end) + + conn = + build_conn(:post, "/inbox") + |> put_signature( + "http://mastodon.example.org/users/admin", + "(request-target) host date digest" + ) + |> put_req_header("digest", "SHA-256=claimed") + |> assign(:digest, "SHA-256=computed") + |> put_format("activity+json") + |> HTTPSignaturePlug.call(%{}) + + assert conn.assigns.valid_signature == true + end + + test "marks a signed POST whose body was not hashed as invalid" do + HTTPSignaturesMock + |> expect(:validate_conn, 0, fn _ -> true end) + + conn = + build_conn(:post, "/inbox") + |> put_signature( + "http://mastodon.example.org/users/admin", + "(request-target) host date digest" + ) + |> put_req_header("digest", "SHA-256=claimed") + |> put_format("activity+json") + |> HTTPSignaturePlug.call(%{}) + + assert conn.assigns.valid_signature == false + assert conn.assigns.signature_error == :body_not_hashed + end + end end From 55bf6b851d702979d40fa74a2356ae59e10faabc Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Wed, 28 Oct 2026 11:35:47 +0400 Subject: [PATCH 15/51] build: bump exif_stripper to 0.1.3 0.1.3 strips GPS data from XMP (including DJI drone positions), reads TIFF descriptions, and no longer reads whole images into memory. --- changelog.d/exif-heic-gps.security | 2 +- mix.exs | 2 +- mix.lock | 2 +- test/fixtures/dji_gps.jpg | Bin 0 -> 4213 bytes test/fixtures/xmp_gps.jpg | Bin 0 -> 3917 bytes .../upload/filter/exif/strip_location_test.exs | 15 +++++++++++++++ 6 files changed, 18 insertions(+), 3 deletions(-) create mode 100644 test/fixtures/dji_gps.jpg create mode 100644 test/fixtures/xmp_gps.jpg diff --git a/changelog.d/exif-heic-gps.security b/changelog.d/exif-heic-gps.security index 0ca439d0c..0b6593f5f 100644 --- a/changelog.d/exif-heic-gps.security +++ b/changelog.d/exif-heic-gps.security @@ -1 +1 @@ -Strip GPS data from WebP and HEIC/AVIF uploads; the Exiftool.StripLocation filter used to skip them +Strip GPS data from WebP and HEIC/AVIF uploads; the Exiftool.StripLocation filter used to skip them; GPS data in XMP (including DJI drone positions) is stripped too diff --git a/mix.exs b/mix.exs index 6527f902b..6c2cfe9c9 100644 --- a/mix.exs +++ b/mix.exs @@ -205,7 +205,7 @@ defmodule Pleroma.Mixfile do {:majic, "~> 1.2"}, {:open_api_spex, "~> 3.22"}, {:ecto_psql_extras, "~> 0.8"}, - {:exif_stripper, "~> 0.1.2"}, + {:exif_stripper, "~> 0.1.3"}, {:vix, "~> 0.36"}, {:elixir_make, "~> 0.7.8", override: true}, {:blurhash, "~> 0.1.0", hex: :rinpatch_blurhash}, diff --git a/mix.lock b/mix.lock index 4fbcc257f..8904e6aa6 100644 --- a/mix.lock +++ b/mix.lock @@ -47,7 +47,7 @@ "ex_doc": {:hex, :ex_doc, "0.38.2", "504d25eef296b4dec3b8e33e810bc8b5344d565998cd83914ffe1b8503737c02", [:mix], [{:earmark_parser, "~> 1.4.44", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "732f2d972e42c116a70802f9898c51b54916e542cc50968ac6980512ec90f42b"}, "ex_machina": {:hex, :ex_machina, "2.8.0", "a0e847b5712065055ec3255840e2c78ef9366634d62390839d4880483be38abe", [:mix], [{:ecto, "~> 2.2 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:ecto_sql, "~> 3.0", [hex: :ecto_sql, repo: "hexpm", optional: true]}], "hexpm", "79fe1a9c64c0c1c1fab6c4fa5d871682cb90de5885320c187d117004627a7729"}, "ex_syslogger": {:hex, :ex_syslogger, "1.5.2", "72b6aa2d47a236e999171f2e1ec18698740f40af0bd02c8c650bf5f1fd1bac79", [:mix], [{:poison, ">= 1.5.0", [hex: :poison, repo: "hexpm", optional: true]}, {:syslog, "~> 1.1.0", [hex: :syslog, repo: "hexpm", optional: false]}], "hexpm", "ab9fab4136dbc62651ec6f16fa4842f10cf02ab4433fa3d0976c01be99398399"}, - "exif_stripper": {:hex, :exif_stripper, "0.1.2", "fa176be937201366545c8c93b5d89b6e2da1ee00534c3f1fb931402e03262755", [:mix], [], "hexpm", "5c524f7cc061826a6108b7b6f85d1d2481d0cff4ce0a678aaedd6edaa0d86dcb"}, + "exif_stripper": {:hex, :exif_stripper, "0.1.3", "b3d89a623bb49c09fbe45fe5d4f1becee0ffe011bcade174c3bb91cdae59745e", [:mix], [], "hexpm", "e1ab26d766205ae54be57c19ec70d408b020f3c31855b49d53a55460de81c6e1"}, "exile": {:hex, :exile, "0.10.0", "b69e2d27a9af670b0f0a0898addca0eda78f6f5ba95ccfbc9bc6ccdd04925436", [:make, :mix], [{:elixir_make, "~> 0.6", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm", "c62ee8fee565b5ac4a898d0dcd58d2b04fb5eec1655af1ddcc9eb582c6732c33"}, "expo": {:hex, :expo, "0.5.1", "249e826a897cac48f591deba863b26c16682b43711dd15ee86b92f25eafd96d9", [:mix], [], "hexpm", "68a4233b0658a3d12ee00d27d37d856b1ba48607e7ce20fd376958d0ba6ce92b"}, "fast_html": {:hex, :fast_html, "2.5.0", "23578c1c1fa03db6a3786d78218b2d944df34b0fc3729e72de912f390913c80f", [:make, :mix], [{:elixir_make, "~> 0.4", [hex: :elixir_make, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 1.1", [hex: :nimble_pool, repo: "hexpm", optional: false]}], "hexpm", "69eb46ed98a5d9cca1ccd4a5ac94ce5dd626fc29513fbaa0a16cd8b2da67ae3e"}, diff --git a/test/fixtures/dji_gps.jpg b/test/fixtures/dji_gps.jpg new file mode 100644 index 0000000000000000000000000000000000000000..7249228cf16b6afe6e96111147a6032fb4a70260 GIT binary patch literal 4213 zcmeHKTW}Lq7(ScLz3E8Wf~6De_JUiK>}EHqG&@N;r6o`VN7^dl+h+GPS-RN`$%cf^ zI3sxX4e#;>cz;91>w}_reZwo@i}kKAf;W_LCZ63a+a!e<;|xCKWHOU;{{Q>V|9$72 z%>0#;l`p}v&B@6mz%UH(=mRPX?SE%9EyvNcQlNNIO+iXfvviry(F_%$L2R^~ z5h<9KlsNhG>u->RB*w|@!&^gJbK@{0ZJN)+>G>OGg!wrkDw3lKZ>-Ffv$-tPc%qz@ zl>%3elLk45e(TFLNf;^GT%4TD@@dF%8_Lq`jH=266Q+ivBqGU+v)t4~(xgDMIBC{e zDwU{Gm{Rj;nu$iEbcm%{Hi$BUg*}SKmxD@S05Nz8gJS{~guIl~Bvm1FPClg;wKz$7 z36okBEaqG>FB`m~Ktma3p`sO#6AX>$2qGp5oVduQFIBe~lQm;foyn3;%*nixj+5n} z2xs}CtdWW9ANFBB&wiKl*>8fOw3OAyx*hOLHb{EH) z!Fu2&OSA$F`Bni9n{2TH^?)fTBNukpoMSjdv9+?6WHbR+rHGMK77Q{tv3J1U;L}I*^3RKU$kK<#EfdmR(Ib&{T5kcD3wk%7LbmTequa zS5po&mHhwQuEkCcDyV%cp+2qh1sDfzr_<$hxLq!n$K&?;y4rnhZN8NqomX_N>gl~| zRZk#53{oV~zjh!H7+SYxZHS3PBE4iZKElSRaD>%KFptOMYx8xtw|BFBfj+kBQF#;i z-2ej^j`ahZAH)4vQk#-ie;_gy;&-@*C-gWmq_uhB^0}np*@FR~t z_V|HAho5-zsi&WL_POU@c=6~A^ zT>P%ies-_BYyJL%M~-<0!c%9Kr}8JfD+VJAtIz7v3}ydL*uMXy%o4V!>ooA;7&<)M Q4@SYc^FxPjAFZ7F2g4kKKL7v# literal 0 HcmV?d00001 diff --git a/test/fixtures/xmp_gps.jpg b/test/fixtures/xmp_gps.jpg new file mode 100644 index 0000000000000000000000000000000000000000..dd76e84f79ac7ba2a153c9e0cd681e8b5715e64f GIT binary patch literal 3917 zcmex=s)|yBtNcQetFn_V zQ`GJ4xNIt{Dsl^QQ%e#RDspr3imfVa)xm~a0ofp7eRTz}l9Fs&bg%gf94%8m8%i_-NCEiEne4UF`SjC6r2 zbc-wVN)jt{^NN*$CU7Z$&2ULAPAI02aP_Ri!wn{;m1`>i;;h0kb=0JEknIMC$auV~>ZPhDu zQ&Q6sOLI!p?YvV{3lwrwZS+B|$jO9g2XmnMZS*lLfG9@t0`cBT1%@cxV<^!Jk^#pr z#8)7$m3u%i&=r{_r75X)#-{p)Hu@OCpb&rw`XoZtnpo(VSm+t}A=JaAP}Jw=r9%ui z(lG?8cSTYUmjXr@I5^1iE-=Mm_bn_jLp%#qX60Oznpl!w1WIm@NGSobZ1mBkusR3g zLSVWCrcix&I)}tQFgef^fYKYy!Kfu91@UNbjRqGTqK16Ns32P*JyB& z6apkErcdfZ^iWdsfZdXEVE^O)Z3brs4mLJ+HdYRHc6Lrq4lW)MULJ049!Vi#ei2zI zc{y1r85so?J#_^o9c39A4RcK$149!N6M1z@TMHu_J!2CikRgnmoSZz|JQBRT5=M$L zibf=Z{|6WZIT#oj7?~NB7?=bZnFSgDA7PMZU|?hg_Q62iFeqSTVrF4wW9Q)H;sz?% zD!{d!pzFb!U9xX3zTPI5o8roG<0MW4oqZMDikqloVbuf*=gfJ(V&YTRE(2~ znmD<{#3dx9RMpfqG__1j&CD$`oZ7B5-4Z25|nt2S-kvUS_`9Xod&I(+2lvEwIBp1O4T z%GGPvZ`{1~@X_NZPoF)1@$%KjPoKYh{r3IG&tD*aF)}biyao3en!f~r{$gTbVP;_m z`HPXM9Eh0(Sy&Yf*@PSe*%J$em5drWM4Tor+<1^v**NHfXj0KdE-@98hpHbzUIY7# kIFB`v Date: Thu, 29 Oct 2026 09:24:08 +0400 Subject: [PATCH 16/51] Sec: Update bandit to version 1.14.1 --- changelog.d/bandit-1.14.1.security | 1 + mix.lock | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/bandit-1.14.1.security diff --git a/changelog.d/bandit-1.14.1.security b/changelog.d/bandit-1.14.1.security new file mode 100644 index 000000000..5c4703b31 --- /dev/null +++ b/changelog.d/bandit-1.14.1.security @@ -0,0 +1 @@ +Update Bandit to version 1.14.1, which fixes an HTTP/2 request body memory amplification diff --git a/mix.lock b/mix.lock index f8ce53f4c..400ade34d 100644 --- a/mix.lock +++ b/mix.lock @@ -1,7 +1,7 @@ %{ "accept": {:hex, :accept, "0.3.5", "b33b127abca7cc948bbe6caa4c263369abf1347cfa9d8e699c6d214660f10cd1", [:rebar3], [], "hexpm", "11b18c220bcc2eab63b5470c038ef10eb6783bcb1fcdb11aa4137defa5ac1bb8"}, "argon2_elixir": {:hex, :argon2_elixir, "4.1.3", "4f28318286f89453364d7fbb53e03d4563fd7ed2438a60237eba5e426e97785f", [:make, :mix], [{:comeonin, "~> 5.3", [hex: :comeonin, repo: "hexpm", optional: false]}, {:elixir_make, "~> 0.6", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm", "7c295b8d8e0eaf6f43641698f962526cdf87c6feb7d14bd21e599271b510608c"}, - "bandit": {:hex, :bandit, "1.14.0", "555af34f126edb98d8b6b95bbeba4d5f4c3af21e37535d791f32db2365c532fb", [:mix], [{:hpax, "~> 1.1", [hex: :hpax, repo: "hexpm", optional: false]}, {:plug, "~> 1.18", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:thousand_island, "~> 1.6", [hex: :thousand_island, repo: "hexpm", optional: false]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "2d07630e4bf77b55240881386f7629bcc2f87426a299860b2d31c4ad77a48746"}, + "bandit": {:hex, :bandit, "1.14.1", "5fb450e2360faf03d9b9fb45cfe52e14197f62d376c34781219bec0f9bc7ca06", [:mix], [{:hpax, "~> 1.1", [hex: :hpax, repo: "hexpm", optional: false]}, {:plug, "~> 1.18", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:thousand_island, "~> 1.6", [hex: :thousand_island, repo: "hexpm", optional: false]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "87c7ee5e08fb5f22d4e072e681e1101266a21432889b08ef8d0cb0c1450f8bba"}, "base62": {:hex, :base62, "1.2.2", "85c6627eb609317b70f555294045895ffaaeb1758666ab9ef9ca38865b11e629", [:mix], [{:custom_base, "~> 0.2.1", [hex: :custom_base, repo: "hexpm", optional: false]}], "hexpm", "d41336bda8eaa5be197f1e4592400513ee60518e5b9f4dcf38f4b4dae6f377bb"}, "bbcode_pleroma": {:hex, :bbcode_pleroma, "0.2.0", "d36f5bca6e2f62261c45be30fa9b92725c0655ad45c99025cb1c3e28e25803ef", [:mix], [{:nimble_parsec, "~> 0.5", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "19851074419a5fedb4ef49e1f01b30df504bb5dbb6d6adfc135238063bebd1c3"}, "bcrypt_elixir": {:hex, :bcrypt_elixir, "2.3.1", "5114d780459a04f2b4aeef52307de23de961b69e13a5cd98a911e39fda13f420", [:make, :mix], [{:comeonin, "~> 5.3", [hex: :comeonin, repo: "hexpm", optional: false]}, {:elixir_make, "~> 0.6", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm", "42182d5f46764def15bf9af83739e3bf4ad22661b1c34fc3e88558efced07279"}, From 385af2a4d16f75ff6349707e0cd2af79282d27ab Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Thu, 29 Oct 2026 09:24:14 +0400 Subject: [PATCH 17/51] Sec: Update jose to version 1.11.13 --- changelog.d/jose-1.11.13.security | 1 + mix.lock | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/jose-1.11.13.security diff --git a/changelog.d/jose-1.11.13.security b/changelog.d/jose-1.11.13.security new file mode 100644 index 000000000..e672d07e7 --- /dev/null +++ b/changelog.d/jose-1.11.13.security @@ -0,0 +1 @@ +Update jose to version 1.11.13, which fixes JWE, key parsing and signature validation vulnerabilities diff --git a/mix.lock b/mix.lock index 400ade34d..7ab59ee46 100644 --- a/mix.lock +++ b/mix.lock @@ -68,7 +68,7 @@ "inet_cidr": {:hex, :inet_cidr, "1.0.9", "e0ef72a2942529da78c8e4147d53f2ef5f6f5293335c3637b0fdf83c012cc816", [:mix], [], "hexpm", "172da15ff7cf635b1feaf14f5818be28c811b37cc5fb7c5f7c01058c1c1066cc"}, "jason": {:hex, :jason, "1.4.5", "2e3a008590b0b8d7388c20293e9dcc9cf3e5d642fd2a114e4cbbb52e595d940a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0 or ~> 3.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "b0c823996102bcd0239b3c2444eb00409b72f6a140c1950bc8b457d836b30684"}, "joken": {:hex, :joken, "2.6.2", "5daaf82259ca603af4f0b065475099ada1b2b849ff140ccd37f4b6828ca6892a", [:mix], [{:jose, "~> 1.11.10", [hex: :jose, repo: "hexpm", optional: false]}], "hexpm", "5134b5b0a6e37494e46dbf9e4dad53808e5e787904b7c73972651b51cce3d72b"}, - "jose": {:hex, :jose, "1.11.10", "a903f5227417bd2a08c8a00a0cbcc458118be84480955e8d251297a425723f83", [:mix, :rebar3], [], "hexpm", "0d6cd36ff8ba174db29148fc112b5842186b68a90ce9fc2b3ec3afe76593e614"}, + "jose": {:hex, :jose, "1.11.13", "94abcd8bf98eb9f007272918bd3f5ffb613e8d250567de233e6c2b2456319005", [:mix, :rebar3], [], "hexpm", "15d8bd0f01ab29d1771e8da1f60f9c98725b23a17f7312cdd40e8e2757bd98c7"}, "jumper": {:hex, :jumper, "1.0.2", "68cdcd84472a00ac596b4e6459a41b3062d4427cbd4f1e8c8793c5b54f1406a7", [:mix], [], "hexpm", "9b7782409021e01ab3c08270e26f36eb62976a38c1aa64b2eaf6348422f165e1"}, "linkify": {:hex, :linkify, "0.5.3", "5f8143d8f61f5ff08d3aeeff47ef6509492b4948d8f08007fbf66e4d2246a7f2", [:mix], [], "hexpm", "3ef35a1377d47c25506e07c1c005ea9d38d700699d92ee92825f024434258177"}, "logger_backends": {:hex, :logger_backends, "1.0.0", "09c4fad6202e08cb0fbd37f328282f16539aca380f512523ce9472b28edc6bdf", [:mix], [], "hexpm", "1faceb3e7ec3ef66a8f5746c5afd020e63996df6fd4eb8cdb789e5665ae6c9ce"}, From e417cd13830e2d33f2a67654f17e9984890cb778 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Thu, 29 Oct 2026 10:41:18 +0400 Subject: [PATCH 18/51] test(signature): build test signatures with sigils for credo --- test/pleroma/signature_test.exs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/pleroma/signature_test.exs b/test/pleroma/signature_test.exs index 18a2a1ddc..3c0ec598f 100644 --- a/test/pleroma/signature_test.exs +++ b/test/pleroma/signature_test.exs @@ -121,8 +121,8 @@ defmodule Pleroma.SignatureTest do defp policy_conn(method, signed_headers, extra_headers, extra_params \\ "") do signature = - "keyId=\"https://remote.example/users/a#main-key\",headers=\"#{signed_headers}\"" <> - extra_params <> ",signature=\"fake\"" + ~s(keyId="https://remote.example/users/a#main-key",headers="#{signed_headers}") <> + extra_params <> ~s(,signature="fake") %Plug.Conn{ method: method, From 8e3404a239415e95be53f8f38e4e3f331ca56789 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 27 Oct 2026 20:48:02 +0400 Subject: [PATCH 19/51] feat(instances): record the instance of new remote users The instances table only got rows from fetches, reachability changes and favicon lookups, so most hosts we know users from had none. Insert a row when a remote user is created, keeping existing rows. --- lib/pleroma/instances/instance.ex | 7 +++++++ lib/pleroma/web/activity_pub/activity_pub.ex | 10 ++++++---- .../web/activity_pub/activity_pub_test.exs | 19 +++++++++++++++++++ 3 files changed, 32 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex index 3695e0b75..3d3ef5774 100644 --- a/lib/pleroma/instances/instance.ex +++ b/lib/pleroma/instances/instance.ex @@ -91,6 +91,13 @@ defmodule Pleroma.Instances.Instance do def reachable?(url_or_host) when is_binary(url_or_host), do: true + @doc "Creates a record for the instance unless one exists. Existing records are kept." + def ensure_exists(url_or_host) when is_binary(url_or_host) do + %Instance{host: host(url_or_host)} + |> changeset(%{}) + |> Repo.insert(on_conflict: :nothing, conflict_target: :host) + end + def set_reachable(url_or_host) when is_binary(url_or_host) do host = host(url_or_host) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index b997e7831..a5325d8ec 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -11,6 +11,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do alias Pleroma.Conversation.Participation alias Pleroma.Filter alias Pleroma.Hashtag + alias Pleroma.Instances.Instance alias Pleroma.Maps alias Pleroma.Notification alias Pleroma.Object @@ -2068,10 +2069,11 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do else maybe_handle_clashing_nickname(data) - data - |> User.remote_user_changeset() - |> Repo.insert() - |> User.set_cache() + with {:ok, user} <- data |> User.remote_user_changeset() |> Repo.insert() do + # Keeps the instances table complete, so it can list our peers. + Instance.ensure_exists(user.ap_id) + User.set_cache({:ok, user}) + end end end end diff --git a/test/pleroma/web/activity_pub/activity_pub_test.exs b/test/pleroma/web/activity_pub/activity_pub_test.exs index c118cd8f1..fcebdffb1 100644 --- a/test/pleroma/web/activity_pub/activity_pub_test.exs +++ b/test/pleroma/web/activity_pub/activity_pub_test.exs @@ -184,6 +184,25 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do assert user.follower_address == "http://mastodon.example.org/users/admin/followers" end + test "it records the user's instance" do + refute Repo.get_by(Pleroma.Instances.Instance, host: "mastodon.example.org") + + {:ok, _user} = ActivityPub.make_user_from_ap_id("http://mastodon.example.org/users/admin") + + assert %{unreachable_since: nil} = + Repo.get_by(Pleroma.Instances.Instance, host: "mastodon.example.org") + end + + test "it keeps an existing instance record" do + unreachable_since = ~N[2020-01-01 00:00:00.000000] + insert(:instance, host: "mastodon.example.org", unreachable_since: unreachable_since) + + {:ok, _user} = ActivityPub.make_user_from_ap_id("http://mastodon.example.org/users/admin") + + assert %{unreachable_since: ^unreachable_since} = + Repo.get_by(Pleroma.Instances.Instance, host: "mastodon.example.org") + end + test "it returns a user that is invisible" do user_id = "http://mastodon.example.org/users/relay" {:ok, user} = ActivityPub.make_user_from_ap_id(user_id) From db11e5e790eb255742155a806652e2732cf841ca Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 27 Oct 2026 20:49:07 +0400 Subject: [PATCH 20/51] feat(migration): add the hosts of existing remote users to instances --- ...28000000_backfill_instances_from_users.exs | 29 +++++++++++++++++ .../backfill_instances_from_users_test.exs | 31 +++++++++++++++++++ 2 files changed, 60 insertions(+) create mode 100644 priv/repo/migrations/20261028000000_backfill_instances_from_users.exs create mode 100644 test/pleroma/repo/migrations/backfill_instances_from_users_test.exs diff --git a/priv/repo/migrations/20261028000000_backfill_instances_from_users.exs b/priv/repo/migrations/20261028000000_backfill_instances_from_users.exs new file mode 100644 index 000000000..13d805c30 --- /dev/null +++ b/priv/repo/migrations/20261028000000_backfill_instances_from_users.exs @@ -0,0 +1,29 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2026 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Repo.Migrations.BackfillInstancesFromUsers do + use Ecto.Migration + + # Remote users now create an instances row; add the hosts of existing ones, + # so the peer list can come from the instances table. + def up do + Pleroma.Repo.query!( + """ + INSERT INTO instances (host, inserted_at, updated_at) + SELECT DISTINCT host, now(), now() + FROM ( + SELECT substring(ap_id from '^[A-Za-z][A-Za-z0-9+.-]*://([^/:?#]+)') AS host + FROM users + WHERE local = false AND ap_id IS NOT NULL + ) AS hosts + WHERE host IS NOT NULL + ON CONFLICT (host) DO NOTHING + """, + [], + timeout: :infinity + ) + end + + def down, do: :ok +end diff --git a/test/pleroma/repo/migrations/backfill_instances_from_users_test.exs b/test/pleroma/repo/migrations/backfill_instances_from_users_test.exs new file mode 100644 index 000000000..733c6644b --- /dev/null +++ b/test/pleroma/repo/migrations/backfill_instances_from_users_test.exs @@ -0,0 +1,31 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2026 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Repo.Migrations.BackfillInstancesFromUsersTest do + use Pleroma.DataCase + import Pleroma.Factory + import Pleroma.Tests.Helpers + + alias Pleroma.Instances.Instance + + setup_all do: require_migration("20261028000000_backfill_instances_from_users") + + test "up/0 adds the hosts of remote users and keeps existing records", %{migration: migration} do + unreachable_since = ~N[2020-01-01 00:00:00.000000] + insert(:instance, host: "known.example", unreachable_since: unreachable_since) + + insert(:user, local: false, ap_id: "https://known.example/users/a") + insert(:user, local: false, ap_id: "https://new.example/users/a") + insert(:user, local: false, ap_id: "https://new.example/users/b") + insert(:user, local: false, ap_id: "https://ported.example:8443/users/c") + insert(:user, local: true) + + migration.up() + + hosts = Instance |> Repo.all() |> Enum.map(& &1.host) |> Enum.sort() + assert hosts == ["known.example", "new.example", "ported.example"] + + assert %{unreachable_since: ^unreachable_since} = Repo.get_by(Instance, host: "known.example") + end +end From e8f238359f6a35a34094dfd5f02a653048c22c6e Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 27 Oct 2026 20:49:52 +0400 Subject: [PATCH 21/51] perf(stats): read peers from instances, update every 5 minutes The peers query read the nickname of every remote user every minute (2.7 s and 1.2M buffer hits on a large instance). The instances table now has a row for every host we know users from, so read it instead, and recompute the stats every 5 minutes. Based on Akkoma 138b1aea2 and 7b76fdeed. Co-authored-by: Oneric Co-authored-by: FloatingGhost --- changelog.d/stats-peers-from-instances.change | 1 + lib/pleroma/stats.ex | 14 +++++--------- .../controllers/instance_controller_test.exs | 9 +++++---- 3 files changed, 11 insertions(+), 13 deletions(-) create mode 100644 changelog.d/stats-peers-from-instances.change diff --git a/changelog.d/stats-peers-from-instances.change b/changelog.d/stats-peers-from-instances.change new file mode 100644 index 000000000..c493b4550 --- /dev/null +++ b/changelog.d/stats-peers-from-instances.change @@ -0,0 +1 @@ +Read the peer list from the instances table and update instance stats every 5 minutes instead of every minute; the old peers query was one of the most expensive regular queries diff --git a/lib/pleroma/stats.ex b/lib/pleroma/stats.ex index 47b30b951..328aa110a 100644 --- a/lib/pleroma/stats.ex +++ b/lib/pleroma/stats.ex @@ -8,10 +8,11 @@ defmodule Pleroma.Stats do import Ecto.Query alias Pleroma.CounterCache + alias Pleroma.Instances.Instance alias Pleroma.Repo alias Pleroma.User - @interval :timer.seconds(60) + @interval :timer.seconds(300) def start_link(_) do GenServer.start_link( @@ -64,14 +65,9 @@ defmodule Pleroma.Stats do } } def calculate_stat_data do - peers = - from( - u in User, - select: fragment("distinct split_part(?, '@', 2)", u.nickname), - where: u.local != ^true - ) - |> Repo.all() - |> Enum.filter(& &1) + # Hosts are unique in instances, and remote users add theirs. Scanning + # all remote users for their domains was one of the costliest queries. + peers = Repo.all(from(i in Instance, select: i.host)) domain_count = Enum.count(peers) diff --git a/test/pleroma/web/mastodon_api/controllers/instance_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/instance_controller_test.exs index 461b46066..404d276db 100644 --- a/test/pleroma/web/mastodon_api/controllers/instance_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/instance_controller_test.exs @@ -63,8 +63,8 @@ defmodule Pleroma.Web.MastodonAPI.InstanceControllerTest do user2 = insert(:user, %{local: true}) {:ok, _user2} = User.set_activation(user2, false) - insert(:user, %{local: false, nickname: "u@peer1.com"}) - insert(:user, %{local: false, nickname: "u@peer2.com"}) + insert(:instance, host: "peer1.com") + insert(:instance, host: "peer2.com") {:ok, _} = Pleroma.Web.CommonAPI.post(user, %{status: "cofe"}) @@ -83,8 +83,9 @@ defmodule Pleroma.Web.MastodonAPI.InstanceControllerTest do end test "get peers", %{conn: conn} do - insert(:user, %{local: false, nickname: "u@peer1.com"}) - insert(:user, %{local: false, nickname: "u@peer2.com"}) + # Peers come from the instances table, which remote users add to. + insert(:instance, host: "peer1.com") + insert(:instance, host: "peer2.com") Pleroma.Stats.force_update() From 958166589ef5a77f0274e2f5b5cb76273ed940ac Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 27 Oct 2026 20:53:56 +0400 Subject: [PATCH 22/51] fix(stats): leave our own host out of the peers Favicon lookups for local accounts can add our own host to instances. Also skip userinfo and IPv6 literals in the backfill migration. --- changelog.d/stats-peers-from-instances.change | 2 +- lib/pleroma/stats.ex | 8 +++++++- .../20261028000000_backfill_instances_from_users.exs | 3 ++- .../migrations/backfill_instances_from_users_test.exs | 4 +++- .../mastodon_api/controllers/instance_controller_test.exs | 2 ++ 5 files changed, 15 insertions(+), 4 deletions(-) diff --git a/changelog.d/stats-peers-from-instances.change b/changelog.d/stats-peers-from-instances.change index c493b4550..a3dfe2e0c 100644 --- a/changelog.d/stats-peers-from-instances.change +++ b/changelog.d/stats-peers-from-instances.change @@ -1 +1 @@ -Read the peer list from the instances table and update instance stats every 5 minutes instead of every minute; the old peers query was one of the most expensive regular queries +Read the peer list from the instances table (hosts of remote actors, without our own host) and update instance stats every 5 minutes; the old peers query scanned all remote users every minute diff --git a/lib/pleroma/stats.ex b/lib/pleroma/stats.ex index 328aa110a..079f6f7dd 100644 --- a/lib/pleroma/stats.ex +++ b/lib/pleroma/stats.ex @@ -67,7 +67,10 @@ defmodule Pleroma.Stats do def calculate_stat_data do # Hosts are unique in instances, and remote users add theirs. Scanning # all remote users for their domains was one of the costliest queries. - peers = Repo.all(from(i in Instance, select: i.host)) + # Favicon lookups for local accounts can add our own host, so skip it. + peers = + from(i in Instance, where: i.host != ^local_host(), select: i.host) + |> Repo.all() domain_count = Enum.count(peers) @@ -93,6 +96,9 @@ defmodule Pleroma.Stats do } end + # Read from the config: this also runs before the endpoint has started. + defp local_host, do: Pleroma.Config.get([Pleroma.Web.Endpoint, :url, :host]) + @spec get_status_visibility_count(String.t() | nil) :: map() def get_status_visibility_count(instance \\ nil) do if is_nil(instance) do diff --git a/priv/repo/migrations/20261028000000_backfill_instances_from_users.exs b/priv/repo/migrations/20261028000000_backfill_instances_from_users.exs index 13d805c30..88dfe9ec5 100644 --- a/priv/repo/migrations/20261028000000_backfill_instances_from_users.exs +++ b/priv/repo/migrations/20261028000000_backfill_instances_from_users.exs @@ -13,7 +13,8 @@ defmodule Pleroma.Repo.Migrations.BackfillInstancesFromUsers do INSERT INTO instances (host, inserted_at, updated_at) SELECT DISTINCT host, now(), now() FROM ( - SELECT substring(ap_id from '^[A-Za-z][A-Za-z0-9+.-]*://([^/:?#]+)') AS host + -- Skips userinfo; IPv6 literals give NULL and are left out. + SELECT substring(ap_id from '^[A-Za-z][A-Za-z0-9+.-]*://(?:[^/?#@]*@)?([^/:?#\\[]+)') AS host FROM users WHERE local = false AND ap_id IS NOT NULL ) AS hosts diff --git a/test/pleroma/repo/migrations/backfill_instances_from_users_test.exs b/test/pleroma/repo/migrations/backfill_instances_from_users_test.exs index 733c6644b..811e336bd 100644 --- a/test/pleroma/repo/migrations/backfill_instances_from_users_test.exs +++ b/test/pleroma/repo/migrations/backfill_instances_from_users_test.exs @@ -19,12 +19,14 @@ defmodule Pleroma.Repo.Migrations.BackfillInstancesFromUsersTest do insert(:user, local: false, ap_id: "https://new.example/users/a") insert(:user, local: false, ap_id: "https://new.example/users/b") insert(:user, local: false, ap_id: "https://ported.example:8443/users/c") + insert(:user, local: false, ap_id: "https://user@userinfo.example/users/d") + insert(:user, local: false, ap_id: "https://[2001:db8::1]/users/e") insert(:user, local: true) migration.up() hosts = Instance |> Repo.all() |> Enum.map(& &1.host) |> Enum.sort() - assert hosts == ["known.example", "new.example", "ported.example"] + assert hosts == ["known.example", "new.example", "ported.example", "userinfo.example"] assert %{unreachable_since: ^unreachable_since} = Repo.get_by(Instance, host: "known.example") end diff --git a/test/pleroma/web/mastodon_api/controllers/instance_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/instance_controller_test.exs index 404d276db..f9d42d981 100644 --- a/test/pleroma/web/mastodon_api/controllers/instance_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/instance_controller_test.exs @@ -86,6 +86,8 @@ defmodule Pleroma.Web.MastodonAPI.InstanceControllerTest do # Peers come from the instances table, which remote users add to. insert(:instance, host: "peer1.com") insert(:instance, host: "peer2.com") + # Favicon lookups for local accounts can add our own host. + insert(:instance, host: Pleroma.Web.Endpoint.host()) Pleroma.Stats.force_update() From 1849356abe698cee3c9590944d3883e1758aafc1 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Thu, 29 Oct 2026 10:42:46 +0400 Subject: [PATCH 23/51] fix(instances): record the instance when refreshing known users too Hosts without a row (IPv6 literals skipped by the backfill, rows deleted later) were never added again, because known users take the update path. --- lib/pleroma/web/activity_pub/activity_pub.ex | 11 ++++++----- test/pleroma/web/activity_pub/activity_pub_test.exs | 9 +++++++++ 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index a5325d8ec..f6c74bdac 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -2058,6 +2058,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do def make_user_from_ap_id(ap_id, additional \\ []) do with {:ok, data} <- fetch_and_prepare_user_from_ap_id(ap_id, additional) do enqueue_pin_fetches(data) + # Keeps the instances table complete, so it can list our peers. + Instance.ensure_exists(data.ap_id) # the fetched actor's id differs from ap_id when it was fetched from a display URL user = User.get_cached_by_ap_id(data.ap_id) @@ -2069,11 +2071,10 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do else maybe_handle_clashing_nickname(data) - with {:ok, user} <- data |> User.remote_user_changeset() |> Repo.insert() do - # Keeps the instances table complete, so it can list our peers. - Instance.ensure_exists(user.ap_id) - User.set_cache({:ok, user}) - end + data + |> User.remote_user_changeset() + |> Repo.insert() + |> User.set_cache() end end end diff --git a/test/pleroma/web/activity_pub/activity_pub_test.exs b/test/pleroma/web/activity_pub/activity_pub_test.exs index fcebdffb1..bd7c97820 100644 --- a/test/pleroma/web/activity_pub/activity_pub_test.exs +++ b/test/pleroma/web/activity_pub/activity_pub_test.exs @@ -193,6 +193,15 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do Repo.get_by(Pleroma.Instances.Instance, host: "mastodon.example.org") end + test "it records the instance of a known user when refreshing it" do + insert(:user, local: false, ap_id: "http://mastodon.example.org/users/admin") + refute Repo.get_by(Pleroma.Instances.Instance, host: "mastodon.example.org") + + {:ok, _user} = ActivityPub.make_user_from_ap_id("http://mastodon.example.org/users/admin") + + assert Repo.get_by(Pleroma.Instances.Instance, host: "mastodon.example.org") + end + test "it keeps an existing instance record" do unreachable_since = ~N[2020-01-01 00:00:00.000000] insert(:instance, host: "mastodon.example.org", unreachable_since: unreachable_since) From fa8541309c7fa0e45dd63ee71d054f8ab1da0369 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Thu, 29 Oct 2026 11:26:02 +0400 Subject: [PATCH 24/51] fix(search): set gin_fuzzy_search_limit only for search queries The limit was a connection parameter, so it applied to every GIN index scan, not only full-text search: recipient lookups and Oban job queries could return incomplete, random results. Set it with SET LOCAL around the search query only, configurable under Pleroma.Search.DatabaseSearch. The default stays 500 for search: measured on a 74M-object database, higher limits or none make searches for common words exceed the query timeout. Remote fetches for URL queries run outside the transaction. Based on Akkoma 811f0a468. Co-authored-by: Oneric --- changelog.d/gin-fuzzy-search-limit.fix | 1 + config/config.exs | 4 +- config/description.exs | 15 +++++ docs/configuration/search.md | 8 +++ lib/pleroma/search/database_search.ex | 60 ++++++++++++++------ test/pleroma/search/database_search_test.exs | 27 +++++++++ 6 files changed, 98 insertions(+), 17 deletions(-) create mode 100644 changelog.d/gin-fuzzy-search-limit.fix diff --git a/changelog.d/gin-fuzzy-search-limit.fix b/changelog.d/gin-fuzzy-search-limit.fix new file mode 100644 index 000000000..276c3daa9 --- /dev/null +++ b/changelog.d/gin-fuzzy-search-limit.fix @@ -0,0 +1 @@ +gin_fuzzy_search_limit is now only set for search queries (configurable under Pleroma.Search.DatabaseSearch), so other GIN index scans such as recipient lookups and Oban job queries are exact again diff --git a/config/config.exs b/config/config.exs index 1b8baed1f..32f5b6c20 100644 --- a/config/config.exs +++ b/config/config.exs @@ -843,7 +843,7 @@ config :pleroma, :modules, runtime_dir: "instance/modules" config :pleroma, configurable_from_database: false config :pleroma, Pleroma.Repo, - parameters: [gin_fuzzy_search_limit: "500", jit: "off"], + parameters: [jit: "off"], prepare: :unnamed config :pleroma, :connections_pool, @@ -948,6 +948,8 @@ config :pleroma, Pleroma.Web.WebFinger, domain: nil, update_nickname_on_user_fet config :pleroma, Pleroma.Search, module: Pleroma.Search.DatabaseSearch +config :pleroma, Pleroma.Search.DatabaseSearch, gin_fuzzy_search_limit: 500 + config :pleroma, Pleroma.Search.Meilisearch, url: "http://127.0.0.1:7700/", private_key: nil, diff --git a/config/description.exs b/config/description.exs index f435628aa..67ae4a585 100644 --- a/config/description.exs +++ b/config/description.exs @@ -3565,6 +3565,21 @@ config :pleroma, :config_description, [ } ] }, + %{ + group: :pleroma, + key: Pleroma.Search.DatabaseSearch, + type: :group, + description: "Built-in database search settings.", + children: [ + %{ + key: :gin_fuzzy_search_limit, + type: :integer, + description: + "Limits how many matches PostgreSQL considers in GIN full-text search queries (see `gin_fuzzy_search_limit` in the PostgreSQL docs). Only set for search queries. Keeps search fast on large databases, but results become incomplete and change between requests. Set to `nil` for exact results, which is fine on small instances. Default: 500.", + suggestions: [500, nil] + } + ] + }, %{ group: :pleroma, key: Pleroma.Search.Meilisearch, diff --git a/docs/configuration/search.md b/docs/configuration/search.md index 5820cdbd9..c37629184 100644 --- a/docs/configuration/search.md +++ b/docs/configuration/search.md @@ -10,6 +10,14 @@ To use built-in search that has no external dependencies, set the search module While it has no external dependencies, it has problems with performance and relevancy. +With a GIN index (the default), search queries limit how many matches PostgreSQL considers (`gin_fuzzy_search_limit`, see [PostgreSQL's docs](https://www.postgresql.org/docs/current/gin.html#GIN-TIPS)). This keeps searches for common words fast on large databases, but the results are incomplete and can change between requests. On smaller instances you can turn it off for exact results: + +> config :pleroma, Pleroma.Search.DatabaseSearch, gin_fuzzy_search_limit: nil + +Without a limit, searches for common words can take longer than the database timeout on large databases, and then return nothing. + +Earlier versions of Pleroma set this limit for every database connection, which also made other GIN index scans (for example recipient lookups) incomplete. It is now only set for search queries. + ## QdrantSearch This uses the vector search engine [Qdrant](https://qdrant.tech) to search the posts in a vector space. This needs a way to generate embeddings and uses the [OpenAI API](https://platform.openai.com/docs/guides/embeddings/what-are-embeddings). This is implemented by several project besides OpenAI itself, including the python-based fastembed-server found in `supplemental/search/fastembed-api`. diff --git a/lib/pleroma/search/database_search.ex b/lib/pleroma/search/database_search.ex index e88d632cb..c66723d1d 100644 --- a/lib/pleroma/search/database_search.ex +++ b/lib/pleroma/search/database_search.ex @@ -7,6 +7,7 @@ defmodule Pleroma.Search.DatabaseSearch do alias Pleroma.Config alias Pleroma.Object.Fetcher alias Pleroma.Pagination + alias Pleroma.Repo alias Pleroma.User alias Pleroma.Web.ActivityPub.Visibility @@ -23,22 +24,49 @@ defmodule Pleroma.Search.DatabaseSearch do offset = Keyword.get(options, :offset, 0) author = Keyword.get(options, :author) - try do - Activity - |> Activity.with_preloaded_object() - |> Activity.restrict_deactivated_users() - |> restrict_public(user) - |> query_with(index_type, search_query) - |> maybe_restrict_local(user) - |> maybe_restrict_author(author) - |> maybe_restrict_blocked(user) - |> Pagination.fetch_paginated( - %{"offset" => offset, "limit" => limit, "skip_order" => index_type == :rum}, - :offset - ) - |> maybe_fetch(user, search_query) - rescue - _ -> maybe_fetch([], user, search_query) + activities = + try do + with_fuzzy_limit(fn -> + Activity + |> Activity.with_preloaded_object() + |> Activity.restrict_deactivated_users() + |> restrict_public(user) + |> query_with(index_type, search_query) + |> maybe_restrict_local(user) + |> maybe_restrict_author(author) + |> maybe_restrict_blocked(user) + |> Pagination.fetch_paginated( + %{"offset" => offset, "limit" => limit, "skip_order" => index_type == :rum}, + :offset + ) + end) + rescue + _ -> [] + end + + # Outside the query transaction, as it may fetch over the network. + maybe_fetch(activities, user, search_query) + end + + @doc """ + Runs `fun` with `gin_fuzzy_search_limit` set, if configured under + `Pleroma.Search.DatabaseSearch`. The limit makes GIN scans faster but their + results incomplete and random, so it is only set for search queries. + """ + def with_fuzzy_limit(fun) do + case Config.get([__MODULE__, :gin_fuzzy_search_limit]) do + limit when is_integer(limit) and limit > 0 -> + {:ok, result} = + Repo.transaction(fn -> + # SET LOCAL takes no parameters; limit is an integer. + Repo.query!("SET LOCAL gin_fuzzy_search_limit TO #{limit}") + fun.() + end) + + result + + _ -> + fun.() end end diff --git a/test/pleroma/search/database_search_test.exs b/test/pleroma/search/database_search_test.exs index d8dd09063..cce810e99 100644 --- a/test/pleroma/search/database_search_test.exs +++ b/test/pleroma/search/database_search_test.exs @@ -44,4 +44,31 @@ defmodule Pleroma.Search.DatabaseSearchTest do assert result.id == other_post.id end + + describe "gin_fuzzy_search_limit" do + defp fuzzy_limit do + %{rows: [[value]]} = Pleroma.Repo.query!("SHOW gin_fuzzy_search_limit") + value + end + + test "is not set on normal database connections" do + assert fuzzy_limit() == "0" + end + + test "is only set for the search query when configured" do + clear_config([Search, :gin_fuzzy_search_limit], nil) + assert Search.with_fuzzy_limit(&fuzzy_limit/0) == "0" + + clear_config([Search, :gin_fuzzy_search_limit], 10_000) + assert Search.with_fuzzy_limit(&fuzzy_limit/0) == "10000" + end + + test "search still works with a limit" do + clear_config([Search, :gin_fuzzy_search_limit], 10_000) + {:ok, post} = CommonAPI.post(insert(:user), %{status: "it's wednesday my dudes"}) + + assert [%{id: id}] = Search.search(nil, "wednesday") + assert id == post.id + end + end end From f26a760f86b9353ab27c1c1f692e246d47c1603d Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Thu, 29 Oct 2026 12:37:02 +0400 Subject: [PATCH 25/51] ci: fetch only the target branch tip trees in check-changelog The check fetched the whole history of develop with all tags for every pull request (about 250 MB), which was slow and loaded the git server. Listing added files only needs the trees of the tip. --- changelog.d/ci-cheap-changelog-check.skip | 0 tools/check-changelog | 4 +++- 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 changelog.d/ci-cheap-changelog-check.skip diff --git a/changelog.d/ci-cheap-changelog-check.skip b/changelog.d/ci-cheap-changelog-check.skip new file mode 100644 index 000000000..e69de29bb diff --git a/tools/check-changelog b/tools/check-changelog index d09a68895..f2a81d427 100644 --- a/tools/check-changelog +++ b/tools/check-changelog @@ -6,7 +6,9 @@ git config --global --add safe.directory "$(pwd)" echo "looking for change log" git remote add upstream https://git.pleroma.social/pleroma/pleroma.git -git fetch upstream ${CI_COMMIT_TARGET_BRANCH}:refs/remotes/upstream/${CI_COMMIT_TARGET_BRANCH} +# Only the tip's trees are needed to list added files: no history, tags or file contents. +git fetch --depth=1 --no-tags --filter=blob:none upstream \ + ${CI_COMMIT_TARGET_BRANCH}:refs/remotes/upstream/${CI_COMMIT_TARGET_BRANCH} git diff --raw --no-renames upstream/${CI_COMMIT_TARGET_BRANCH} HEAD -- changelog.d | \ grep ' A\t' | grep '\.\(skip\|add\|remove\|fix\|security\|change\)$' From 53b79781da17606afde107f46f552d63d6bb3470 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Fri, 30 Oct 2026 10:36:05 +0400 Subject: [PATCH 26/51] fix(mastodon_api): rate limit status actions per status id StatusController keeps string param keys (replace_params: false, since b8db67daf), so params: [:id] never matched and status_id_action became one bucket per user across all statuses (3 favourites/reblogs a minute). --- changelog.d/status-id-action-rate-limit.fix | 1 + .../controllers/status_controller.ex | 4 ++-- .../controllers/status_controller_test.exs | 23 +++++++++++++++++++ 3 files changed, 26 insertions(+), 2 deletions(-) create mode 100644 changelog.d/status-id-action-rate-limit.fix diff --git a/changelog.d/status-id-action-rate-limit.fix b/changelog.d/status-id-action-rate-limit.fix new file mode 100644 index 000000000..030eeb0df --- /dev/null +++ b/changelog.d/status-id-action-rate-limit.fix @@ -0,0 +1 @@ +Rate limit favourites and reblogs per status, not across all statuses diff --git a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex index 6cd7469f9..c5210d588 100644 --- a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex @@ -92,13 +92,13 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do plug( RateLimiter, - [name: :status_id_action, bucket_name: "status_id_action:reblog_unreblog", params: [:id]] + [name: :status_id_action, bucket_name: "status_id_action:reblog_unreblog", params: ["id"]] when action in ~w(reblog unreblog)a ) plug( RateLimiter, - [name: :status_id_action, bucket_name: "status_id_action:fav_unfav", params: [:id]] + [name: :status_id_action, bucket_name: "status_id_action:fav_unfav", params: ["id"]] when action in ~w(favourite unfavourite)a ) diff --git a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs index 6ee79689d..1b8a3b626 100644 --- a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs @@ -2243,6 +2243,29 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do |> json_response_and_validate_schema(404) == %{"error" => "Record not found"} end + describe "status_id_action rate limit" do + setup do: clear_config([:rate_limit, :status_id_action], {60_000, 2}) + setup do: clear_config([:rate_limit, :statuses_actions], {60_000, 100}) + setup do: oauth_access(["write:statuses", "write:favourites"]) + + for {action, undo} <- [{"favourite", "unfavourite"}, {"reblog", "unreblog"}] do + test "#{action} is limited per status, not per user", %{conn: conn} do + conn = put_req_header(conn, "content-type", "application/json") + statuses = insert_list(3, :note_activity) + + assert Enum.map( + statuses, + &post(conn, "/api/v1/statuses/#{&1.id}/#{unquote(action)}").status + ) == + [200, 200, 200] + + status = hd(statuses) + assert post(conn, "/api/v1/statuses/#{status.id}/#{unquote(undo)}").status == 200 + assert post(conn, "/api/v1/statuses/#{status.id}/#{unquote(action)}").status == 429 + end + end + end + describe "pinned statuses" do setup do: oauth_access(["write:accounts"]) From 44adf271c24f354694fc53bd53b298169adb4859 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Fri, 30 Oct 2026 10:20:06 +0400 Subject: [PATCH 27/51] fix(activitypub): rate limit C2S outbox posts POST /users/:nickname/outbox had no rate limiter, so a client could send unlimited activities through the C2S API. Apply :statuses_actions to update_outbox. RateLimiter keys buckets by limiter name and user, so the outbox shares one counter with the Mastodon API status actions and mixing both APIs does not double the limit. --- .../activity_pub/activity_pub_controller.ex | 4 ++ .../activity_pub_controller_test.exs | 37 +++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index dd5f6028f..5ee93eabf 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -26,6 +26,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do alias Pleroma.Web.Plugs.EnsureAuthenticatedPlug alias Pleroma.Web.Plugs.FederatingPlug alias Pleroma.Web.Plugs.OAuthScopesPlug + alias Pleroma.Web.Plugs.RateLimiter require Logger @@ -66,6 +67,9 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do when action in [:followers, :following] ) + # Shares the per-user bucket with Mastodon API status actions and emoji reactions. + plug(RateLimiter, [name: :statuses_actions] when action == :update_outbox) + plug(Majic.Plug, [pool: Pleroma.MajicPool] when action in [:upload_media]) plug( diff --git a/test/pleroma/web/activity_pub/activity_pub_controller_test.exs b/test/pleroma/web/activity_pub/activity_pub_controller_test.exs index 606d47c8d..b72fb6424 100644 --- a/test/pleroma/web/activity_pub/activity_pub_controller_test.exs +++ b/test/pleroma/web/activity_pub/activity_pub_controller_test.exs @@ -2685,6 +2685,43 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do assert result == "Character limit (5 characters) exceeded, contains 11 characters" end + + test "it is rate limited", %{conn: conn, activity: activity} do + clear_config([:rate_limit, :statuses_actions], {60_000, 2}) + user = insert(:user) + + conn = + conn + |> assign(:user, user) + |> put_req_header("content-type", "application/activity+json") + + for _ <- 1..2 do + conn + |> post("/users/#{user.nickname}/outbox", activity) + |> json_response(201) + end + + assert conn + |> post("/users/#{user.nickname}/outbox", activity) + |> json_response(429) == %{"error" => "Throttled"} + end + + test "it shares the rate limit with the Mastodon API", %{activity: activity} do + clear_config([:rate_limit, :statuses_actions], {60_000, 2}) + %{user: user, conn: conn} = oauth_access(["write"]) + + for _ <- 1..2 do + conn + |> put_req_header("content-type", "application/json") + |> post("/api/v1/statuses", %{status: "cofe"}) + |> json_response(200) + end + + assert conn + |> put_req_header("content-type", "application/activity+json") + |> post("/users/#{user.nickname}/outbox", activity) + |> json_response(429) == %{"error" => "Throttled"} + end end describe "/relay/followers" do From a4f1300578ef8a7f2135213ca0bacd9d561cdf4f Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Fri, 30 Oct 2026 10:20:43 +0400 Subject: [PATCH 28/51] fix(pleroma_api): rate limit emoji reactions The emoji reaction create and delete endpoints had no rate limiter. Apply :statuses_actions like the other status actions, plus a per-status :status_id_action bucket ("status_id_action:emoji_react") in the same way favourite and reblog do. --- .../controllers/emoji_reaction_controller.ex | 9 ++++ .../emoji_reaction_controller_test.exs | 49 +++++++++++++++++++ 2 files changed, 58 insertions(+) diff --git a/lib/pleroma/web/pleroma_api/controllers/emoji_reaction_controller.ex b/lib/pleroma/web/pleroma_api/controllers/emoji_reaction_controller.ex index 82abfce7f..57e3a2d5b 100644 --- a/lib/pleroma/web/pleroma_api/controllers/emoji_reaction_controller.ex +++ b/lib/pleroma/web/pleroma_api/controllers/emoji_reaction_controller.ex @@ -12,6 +12,7 @@ defmodule Pleroma.Web.PleromaAPI.EmojiReactionController do alias Pleroma.Web.CommonAPI alias Pleroma.Web.MastodonAPI.StatusView alias Pleroma.Web.Plugs.OAuthScopesPlug + alias Pleroma.Web.Plugs.RateLimiter plug(Pleroma.Web.ApiSpec.CastAndValidate) plug(OAuthScopesPlug, %{scopes: ["write:statuses"]} when action in [:create, :delete]) @@ -22,6 +23,14 @@ defmodule Pleroma.Web.PleromaAPI.EmojiReactionController do when action == :index ) + plug( + RateLimiter, + [name: :status_id_action, bucket_name: "status_id_action:emoji_react", params: [:id]] + when action in [:create, :delete] + ) + + plug(RateLimiter, [name: :statuses_actions] when action in [:create, :delete]) + defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.EmojiReactionOperation action_fallback(Pleroma.Web.MastodonAPI.FallbackController) diff --git a/test/pleroma/web/pleroma_api/controllers/emoji_reaction_controller_test.exs b/test/pleroma/web/pleroma_api/controllers/emoji_reaction_controller_test.exs index 8fcdb233b..12b037a21 100644 --- a/test/pleroma/web/pleroma_api/controllers/emoji_reaction_controller_test.exs +++ b/test/pleroma/web/pleroma_api/controllers/emoji_reaction_controller_test.exs @@ -477,4 +477,53 @@ defmodule Pleroma.Web.PleromaAPI.EmojiReactionControllerTest do |> get("/api/v1/pleroma/statuses/#{activity_id}/reactions/🐈") |> json_response_and_validate_schema(404) == %{"error" => "Record not found"} end + + describe "rate limiting" do + test "PUT and DELETE share a per-status limit", %{conn: conn} do + clear_config([:rate_limit, :status_id_action], {60_000, 2}) + user = insert(:user) + %{id: activity_id} = insert(:note_activity) + %{id: other_activity_id} = insert(:note_activity) + conn = prepare_conn_of_user(conn, user) + + conn + |> put("/api/v1/pleroma/statuses/#{activity_id}/reactions/☕") + |> json_response_and_validate_schema(200) + + conn + |> delete("/api/v1/pleroma/statuses/#{activity_id}/reactions/☕") + |> json_response_and_validate_schema(200) + + assert conn + |> put("/api/v1/pleroma/statuses/#{activity_id}/reactions/🐈") + |> json_response(429) == %{"error" => "Throttled"} + + conn + |> put("/api/v1/pleroma/statuses/#{other_activity_id}/reactions/🐈") + |> json_response_and_validate_schema(200) + end + + test "PUT and DELETE count against statuses_actions", %{conn: conn} do + clear_config([:rate_limit, :statuses_actions], {60_000, 2}) + user = insert(:user) + [first, second, third] = insert_list(3, :note_activity) + conn = prepare_conn_of_user(conn, user) + + conn + |> put("/api/v1/pleroma/statuses/#{first.id}/reactions/☕") + |> json_response_and_validate_schema(200) + + conn + |> delete("/api/v1/pleroma/statuses/#{first.id}/reactions/☕") + |> json_response_and_validate_schema(200) + + assert conn + |> put("/api/v1/pleroma/statuses/#{second.id}/reactions/☕") + |> json_response(429) == %{"error" => "Throttled"} + + assert conn + |> delete("/api/v1/pleroma/statuses/#{third.id}/reactions/☕") + |> json_response(429) == %{"error" => "Throttled"} + end + end end From 924df2a0633bfe5dc5213895e56460dcc95ca0a1 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Fri, 30 Oct 2026 10:20:44 +0400 Subject: [PATCH 29/51] chore: add changelog entry for C2S rate limits --- changelog.d/c2s-rate-limit.security | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/c2s-rate-limit.security diff --git a/changelog.d/c2s-rate-limit.security b/changelog.d/c2s-rate-limit.security new file mode 100644 index 000000000..8cab4a309 --- /dev/null +++ b/changelog.d/c2s-rate-limit.security @@ -0,0 +1 @@ +Rate limit ActivityPub C2S outbox posts and emoji reactions From db43aec78263ed2456c9f2053b1b48866ddf9a75 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 27 Oct 2026 21:31:36 +0400 Subject: [PATCH 30/51] fix(search): fetch remote statuses only when resolving is allowed Status search fetched any http(s) query from its remote server, for anonymous users, on every result page and with resolve=false, so anyone could make the server fetch URLs. As in Mastodon and in Pleroma's user search, fetch only with resolve=true, on the first page, and for anonymous users only if [:instance, :limit_to_local_content] is false. Based on Akkoma ca9e33ed2 and 17efec060. Co-authored-by: Oneric --- changelog.d/search-remote-fetch.security | 1 + lib/pleroma/search/database_search.ex | 24 +++++++++++++-- lib/pleroma/search/meilisearch.ex | 4 +-- lib/pleroma/search/parade_db.ex | 12 ++++---- lib/pleroma/search/qdrant_search.ex | 4 +-- .../controllers/search_controller_test.exs | 30 +++++++++++++++++-- 6 files changed, 61 insertions(+), 14 deletions(-) create mode 100644 changelog.d/search-remote-fetch.security diff --git a/changelog.d/search-remote-fetch.security b/changelog.d/search-remote-fetch.security new file mode 100644 index 000000000..4211615ae --- /dev/null +++ b/changelog.d/search-remote-fetch.security @@ -0,0 +1 @@ +Status search only fetches remote URLs with resolve=true, on the first page, and for anonymous users only if limit_to_local_content allows it diff --git a/lib/pleroma/search/database_search.ex b/lib/pleroma/search/database_search.ex index c66723d1d..dca69260d 100644 --- a/lib/pleroma/search/database_search.ex +++ b/lib/pleroma/search/database_search.ex @@ -45,7 +45,7 @@ defmodule Pleroma.Search.DatabaseSearch do end # Outside the query transaction, as it may fetch over the network. - maybe_fetch(activities, user, search_query) + maybe_fetch(activities, user, search_query, options) end @doc """ @@ -160,7 +160,27 @@ defmodule Pleroma.Search.DatabaseSearch do defp restrict_local(q), do: where(q, local: true) - def maybe_fetch(activities, user, search_query) do + @doc """ + Adds the status a URL query points to, fetching it if needed. As in Mastodon, + this only happens with `resolve: true` and on the first page, and for + anonymous users only if `[:instance, :limit_to_local_content]` is `false`. + """ + def maybe_fetch(activities, user, search_query, options) do + if fetch_allowed?(user, options), + do: fetch(activities, user, search_query), + else: activities + end + + defp fetch_allowed?(user, options) do + options[:resolve] == true and (options[:offset] || 0) == 0 and + case Config.get([:instance, :limit_to_local_content], :unauthenticated) do + :all -> false + :unauthenticated -> match?(%User{}, user) + false -> true + end + end + + defp fetch(activities, user, search_query) do with true <- Regex.match?(~r/https?:/, search_query), {:ok, object} <- Fetcher.fetch_object_from_id(search_query), %Activity{} = activity <- Activity.get_create_by_object_ap_id(object.data["id"]), diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex index dc10076e1..802870b75 100644 --- a/lib/pleroma/search/meilisearch.ex +++ b/lib/pleroma/search/meilisearch.ex @@ -111,11 +111,11 @@ defmodule Pleroma.Search.Meilisearch do |> maybe_restrict_local(user) |> maybe_restrict_author(author) |> maybe_restrict_blocked(user) - |> maybe_fetch(user, query) + |> maybe_fetch(user, query, options) |> order_by([object: obj], desc: obj.data["published"]) |> Pleroma.Repo.all() rescue - _ -> maybe_fetch([], user, query) + _ -> maybe_fetch([], user, query, options) end end end diff --git a/lib/pleroma/search/parade_db.ex b/lib/pleroma/search/parade_db.ex index ee00e74af..78d561505 100644 --- a/lib/pleroma/search/parade_db.ex +++ b/lib/pleroma/search/parade_db.ex @@ -12,7 +12,7 @@ defmodule Pleroma.Search.ParadeDB do import Pleroma.Search.DatabaseSearch, only: [ - maybe_fetch: 3, + maybe_fetch: 4, maybe_restrict_author: 2, maybe_restrict_blocked: 2, maybe_restrict_local: 2 @@ -149,19 +149,19 @@ defmodule Pleroma.Search.ParadeDB do case search_ids(query, limit, offset, author) do {:ok, []} -> - maybe_fetch([], user, query) + maybe_fetch([], user, query, options) {:ok, ids} -> ids - |> fetch_activities(user, query, author) + |> fetch_activities(user, query, author, options) {:error, error} -> Logger.error("ParadeDB search failed: #{inspect(error)}") - maybe_fetch([], user, query) + maybe_fetch([], user, query, options) end end - defp fetch_activities(ids, user, query, author) do + defp fetch_activities(ids, user, query, author, options) do dumped_ids = ids |> Enum.map(&dump_activity_id/1) @@ -183,7 +183,7 @@ defmodule Pleroma.Search.ParadeDB do ) ) |> Pleroma.Repo.all() - |> maybe_fetch(user, query) + |> maybe_fetch(user, query, options) end defp maybe_restrict_public(query, %Pleroma.User{}) do diff --git a/lib/pleroma/search/qdrant_search.ex b/lib/pleroma/search/qdrant_search.ex index a8c5a3eb9..a453d222f 100644 --- a/lib/pleroma/search/qdrant_search.ex +++ b/lib/pleroma/search/qdrant_search.ex @@ -10,7 +10,7 @@ defmodule Pleroma.Search.QdrantSearch do alias __MODULE__.OpenAIClient alias __MODULE__.QdrantClient - import Pleroma.Search.DatabaseSearch, only: [maybe_fetch: 3] + import Pleroma.Search.DatabaseSearch, only: [maybe_fetch: 4] @impl true def create_index do @@ -131,7 +131,7 @@ defmodule Pleroma.Search.QdrantSearch do |> Activity.restrict_deactivated_users() |> Ecto.Query.order_by([a], fragment("array_position(?, ?)", ^ids, a.id)) |> Pleroma.Repo.all() - |> maybe_fetch(user, original_query) + |> maybe_fetch(user, original_query, options) else _ -> [] diff --git a/test/pleroma/web/mastodon_api/controllers/search_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/search_controller_test.exs index f0c9c1901..368c07f35 100644 --- a/test/pleroma/web/mastodon_api/controllers/search_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/search_controller_test.exs @@ -328,7 +328,9 @@ defmodule Pleroma.Web.MastodonAPI.SearchControllerTest do assert status["id"] == to_string(activity.id) end - test "search fetches remote statuses and prefers them over other results", %{conn: conn} do + test "search fetches remote statuses and prefers them over other results" do + %{conn: conn} = oauth_access(["read:search"]) + {:ok, %{id: activity_id}} = CommonAPI.post(insert(:user), %{ status: "check out http://mastodon.example.org/@admin/99541947525187367" @@ -336,7 +338,9 @@ defmodule Pleroma.Web.MastodonAPI.SearchControllerTest do %{"url" => result_url, "id" => result_id} = conn - |> get("/api/v1/search?q=http://mastodon.example.org/@admin/99541947525187367") + |> get( + "/api/v1/search?resolve=true&q=http://mastodon.example.org/@admin/99541947525187367" + ) |> json_response_and_validate_schema(200) |> Map.get("statuses") |> List.first() @@ -345,6 +349,28 @@ defmodule Pleroma.Web.MastodonAPI.SearchControllerTest do assert match?(^result_url, "http://mastodon.example.org/@admin/99541947525187367") end + test "search fetches remote statuses only with resolve, when logged in, on the first page", + %{conn: anonymous} do + %{conn: conn} = oauth_access(["read:search"]) + url = "http://mastodon.example.org/@admin/99541947525187367" + + for {conn, params} <- [ + {conn, ""}, + {conn, "&resolve=false"}, + {conn, "&resolve=true&offset=5"}, + {anonymous, "&resolve=true"} + ] do + assert %{"statuses" => []} = + conn + |> get("/api/v1/search?q=#{url}#{params}") + |> json_response_and_validate_schema(200) + end + + refute Pleroma.Object.get_by_ap_id( + "http://mastodon.example.org/users/admin/statuses/99541947525187367" + ) + end + test "search doesn't show statuses that it shouldn't", %{conn: conn} do {:ok, activity} = CommonAPI.post(insert(:user), %{ From f568536b65bde83ee86a836ceb7f7bcfab647bf0 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Thu, 29 Oct 2026 10:41:51 +0400 Subject: [PATCH 31/51] test(status_view): resolve the remote status as a logged-in user --- test/pleroma/web/mastodon_api/views/status_view_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/pleroma/web/mastodon_api/views/status_view_test.exs b/test/pleroma/web/mastodon_api/views/status_view_test.exs index 6d1bda193..b121cf32c 100644 --- a/test/pleroma/web/mastodon_api/views/status_view_test.exs +++ b/test/pleroma/web/mastodon_api/views/status_view_test.exs @@ -788,7 +788,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do test "put the url advertised in the Activity in to the url attribute" do id = "https://wedistribute.org/wp-json/pterotype/v1/object/85810" - [activity] = Activity.search(nil, id) + [activity] = Activity.search(insert(:user), id, resolve: true) status = StatusView.render("show.json", %{activity: activity}) From 06612c802b561014ed955c8e6999e457e719fa0d Mon Sep 17 00:00:00 2001 From: Floatingghost Date: Sat, 2 Aug 2025 09:47:32 +0100 Subject: [PATCH 32/51] Do not federate undo->block activities the :do_not_federate checks were omitted from the undo pipeline, which could lead to them federating. this commit enforces :outgoing_blocks on undos as well - and any existing value of :do_not_federate is preserved, if one exists Akkoma issue 957. [Pleroma: block/unblock test calls use the Pleroma (blocked, blocker) argument order.] (cherry picked from commit eca7ed572b8a10a0235fb1a381eac4803614dee4) --- .../web/activity_pub/object_validator.ex | 6 ++++++ test/pleroma/web/common_api_test.exs | 20 +++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/lib/pleroma/web/activity_pub/object_validator.ex b/lib/pleroma/web/activity_pub/object_validator.ex index 468298a4d..a89ad94f8 100644 --- a/lib/pleroma/web/activity_pub/object_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validator.ex @@ -78,10 +78,16 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do |> Ecto.Changeset.apply_action(:insert) do object = stringify_keys(object) undone_object = Activity.get_by_ap_id(object["object"]) + outgoing_blocks = Pleroma.Config.get([:activitypub, :outgoing_blocks]) + # if we're undoing a block, and do not permit federating that: + do_not_federate = + Keyword.get(meta, :do_not_federate) || + (Map.get(undone_object.data, "type") == "Block" && !outgoing_blocks) meta = meta |> Keyword.put(:object_data, undone_object.data) + |> Keyword.put(:do_not_federate, do_not_federate) {:ok, object, meta} end diff --git a/test/pleroma/web/common_api_test.exs b/test/pleroma/web/common_api_test.exs index 5d0ad4572..6c2b5ec91 100644 --- a/test/pleroma/web/common_api_test.exs +++ b/test/pleroma/web/common_api_test.exs @@ -338,6 +338,26 @@ defmodule Pleroma.Web.CommonAPITest do assert {:ok, :no_activity} == CommonAPI.unblock(blocked, blocker) refute User.blocks?(blocker, blocked) end + + test "it unblocks and does not federate if outgoing blocks are disabled" do + clear_config([:instance, :federating], true) + clear_config([:activitypub, :outgoing_blocks], false) + + blocked = insert(:user) + blocker = insert(:user) + + with_mock Pleroma.Web.Federator, + publish: fn _ -> nil end do + assert {:ok, block} = CommonAPI.block(blocked, blocker) + assert block.local + assert User.blocks?(blocker, blocked) + + assert {:ok, unblock} = CommonAPI.unblock(blocked, blocker) + assert unblock.local + refute User.blocks?(blocker, blocked) + assert_not_called(Pleroma.Web.Federator.publish(:_)) + end + end end describe "deletion" do From 865bcfb7b1416a00fa6af9a553b1c9c05db4ff79 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sat, 31 Oct 2026 12:36:22 +0400 Subject: [PATCH 33/51] chore: add changelog entry for Undo Block federation --- changelog.d/undo-block-outgoing-blocks.security | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/undo-block-outgoing-blocks.security diff --git a/changelog.d/undo-block-outgoing-blocks.security b/changelog.d/undo-block-outgoing-blocks.security new file mode 100644 index 000000000..8945666bc --- /dev/null +++ b/changelog.d/undo-block-outgoing-blocks.security @@ -0,0 +1 @@ +Do not federate the Undo of a block when `outgoing_blocks` is disabled From 163c49c649f5fb0af053ec2cbfa036c77b8205ff Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sat, 31 Oct 2026 13:12:15 +0400 Subject: [PATCH 34/51] fix(api_spec): pass cast options on the first permissive cast Without them replace_params: false was ignored when strict is off, so conn.params got atom keys and RateLimiter params: ["id"] buckets collapsed to one per user. --- changelog.d/cast-and-validate-cast-opts.fix | 1 + lib/pleroma/web/api_spec/cast_and_validate.ex | 2 +- .../web/mastodon_api/controllers/status_controller_test.exs | 6 ++++-- 3 files changed, 6 insertions(+), 3 deletions(-) create mode 100644 changelog.d/cast-and-validate-cast-opts.fix diff --git a/changelog.d/cast-and-validate-cast-opts.fix b/changelog.d/cast-and-validate-cast-opts.fix new file mode 100644 index 000000000..ba4ce2283 --- /dev/null +++ b/changelog.d/cast-and-validate-cast-opts.fix @@ -0,0 +1 @@ +Fix per-status and per-account rate limits when `Pleroma.Web.ApiSpec.CastAndValidate` `strict` is off (the production default) diff --git a/lib/pleroma/web/api_spec/cast_and_validate.ex b/lib/pleroma/web/api_spec/cast_and_validate.ex index 95bd4d9cf..92f1c5376 100644 --- a/lib/pleroma/web/api_spec/cast_and_validate.ex +++ b/lib/pleroma/web/api_spec/cast_and_validate.ex @@ -114,7 +114,7 @@ defmodule Pleroma.Web.ApiSpec.CastAndValidate do false = _strict, cast_opts ) do - case OpenApiSpex.cast_and_validate(spec, operation, conn, content_type) do + case OpenApiSpex.cast_and_validate(spec, operation, conn, content_type, cast_opts) do {:ok, conn} -> {:ok, conn} diff --git a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs index 1b8a3b626..bc859df48 100644 --- a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs @@ -2248,8 +2248,10 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do setup do: clear_config([:rate_limit, :statuses_actions], {60_000, 100}) setup do: oauth_access(["write:statuses", "write:favourites"]) - for {action, undo} <- [{"favourite", "unfavourite"}, {"reblog", "unreblog"}] do - test "#{action} is limited per status, not per user", %{conn: conn} do + for {action, undo} <- [{"favourite", "unfavourite"}, {"reblog", "unreblog"}], + strict <- [true, false] do + test "#{action} is limited per status, not per user (strict: #{strict})", %{conn: conn} do + clear_config([Pleroma.Web.ApiSpec.CastAndValidate, :strict], unquote(strict)) conn = put_req_header(conn, "content-type", "application/json") statuses = insert_list(3, :note_activity) From 7e157d8cc9f4179dce20ed01d78bbbefe2ac5b11 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sat, 31 Oct 2026 13:17:50 +0400 Subject: [PATCH 35/51] fix(mrf): match domain rules literally Escape the domain so `.` only matches a dot, and use an optional group for the `*.` prefix, which matches the same hosts without exponential backtracking. Bad entries now match nothing instead of raising on every check. --- changelog.d/mrf-domain-regex-escape.security | 1 + lib/pleroma/web/activity_pub/mrf.ex | 15 +++++----- test/pleroma/web/activity_pub/mrf_test.exs | 31 ++++++++++++-------- 3 files changed, 27 insertions(+), 20 deletions(-) create mode 100644 changelog.d/mrf-domain-regex-escape.security diff --git a/changelog.d/mrf-domain-regex-escape.security b/changelog.d/mrf-domain-regex-escape.security new file mode 100644 index 000000000..07274c37b --- /dev/null +++ b/changelog.d/mrf-domain-regex-escape.security @@ -0,0 +1 @@ +MRF domain rules now match dots literally, so a rule for `good.example` no longer also matches `goodxexample`, and wildcard rules no longer backtrack on hostile host names diff --git a/lib/pleroma/web/activity_pub/mrf.ex b/lib/pleroma/web/activity_pub/mrf.ex index 2496d1858..ddcd85624 100644 --- a/lib/pleroma/web/activity_pub/mrf.ex +++ b/lib/pleroma/web/activity_pub/mrf.ex @@ -148,14 +148,13 @@ defmodule Pleroma.Web.ActivityPub.MRF do @spec subdomains_regex([String.t()]) :: [Regex.t()] def subdomains_regex(domains) when is_list(domains) do for domain <- domains do - try do - target = String.replace(domain, "*.", "(.*\\.)*") - ~r<^#{target}$>i - rescue - e -> - Logger.error("MRF: Invalid subdomain Regex: #{domain}") - reraise e, __STACKTRACE__ - end + {prefix, rest} = + case domain do + "*." <> rest -> {"(.*\\.)?", rest} + _ -> {"", domain} + end + + ~r<^#{prefix}#{Regex.escape(rest)}$>i end end diff --git a/test/pleroma/web/activity_pub/mrf_test.exs b/test/pleroma/web/activity_pub/mrf_test.exs index d897e529d..83c0b0ae5 100644 --- a/test/pleroma/web/activity_pub/mrf_test.exs +++ b/test/pleroma/web/activity_pub/mrf_test.exs @@ -6,8 +6,6 @@ defmodule Pleroma.Web.ActivityPub.MRFTest do use ExUnit.Case use Pleroma.Tests.Helpers - import ExUnit.CaptureLog - alias Pleroma.Web.ActivityPub.MRF defp regexes_match!([], []), do: true @@ -21,14 +19,14 @@ defmodule Pleroma.Web.ActivityPub.MRFTest do test "subdomains_regex/1" do regexes = MRF.subdomains_regex(["unsafe.tld", "*.unsafe.tld"]) - assert regexes_match!(regexes, [~r/^unsafe.tld$/i, ~r/^(.*\.)*unsafe.tld$/i]) + assert regexes_match!(regexes, [~r/^unsafe\.tld$/i, ~r/^(.*\.)?unsafe\.tld$/i]) end describe "subdomain_match/2" do test "common domains" do regexes = MRF.subdomains_regex(["unsafe.tld", "unsafe2.tld"]) - assert regexes_match!(regexes, [~r/^unsafe.tld$/i, ~r/^unsafe2.tld$/i]) + assert regexes_match!(regexes, [~r/^unsafe\.tld$/i, ~r/^unsafe2\.tld$/i]) assert MRF.subdomain_match?(regexes, "unsafe.tld") assert MRF.subdomain_match?(regexes, "unsafe2.tld") @@ -39,7 +37,7 @@ defmodule Pleroma.Web.ActivityPub.MRFTest do test "wildcard domains with one subdomain" do regexes = MRF.subdomains_regex(["*.unsafe.tld"]) - assert regexes_match!(regexes, [~r/^(.*\.)*unsafe.tld$/i]) + assert regexes_match!(regexes, [~r/^(.*\.)?unsafe\.tld$/i]) assert MRF.subdomain_match?(regexes, "unsafe.tld") assert MRF.subdomain_match?(regexes, "sub.unsafe.tld") @@ -50,7 +48,7 @@ defmodule Pleroma.Web.ActivityPub.MRFTest do test "wildcard domains with two subdomains" do regexes = MRF.subdomains_regex(["*.unsafe.tld"]) - assert regexes_match!(regexes, [~r/^(.*\.)*unsafe.tld$/i]) + assert regexes_match!(regexes, [~r/^(.*\.)?unsafe\.tld$/i]) assert MRF.subdomain_match?(regexes, "unsafe.tld") assert MRF.subdomain_match?(regexes, "sub.sub.unsafe.tld") @@ -61,7 +59,7 @@ defmodule Pleroma.Web.ActivityPub.MRFTest do test "matches are case-insensitive" do regexes = MRF.subdomains_regex(["UnSafe.TLD", "UnSAFE2.Tld"]) - assert regexes_match!(regexes, [~r/^UnSafe.TLD$/i, ~r/^UnSAFE2.Tld$/i]) + assert regexes_match!(regexes, [~r/^UnSafe\.TLD$/i, ~r/^UnSAFE2\.Tld$/i]) assert MRF.subdomain_match?(regexes, "UNSAFE.TLD") assert MRF.subdomain_match?(regexes, "UNSAFE2.TLD") @@ -72,11 +70,20 @@ defmodule Pleroma.Web.ActivityPub.MRFTest do refute MRF.subdomain_match?(regexes, "example.com") end - test "logs sensible error on accidental wildcard" do - assert_raise Regex.CompileError, fn -> - assert capture_log(MRF.subdomains_regex(["*unsafe.tld"])) =~ - "MRF: Invalid subdomain Regex: *unsafe.tld" - end + test "dots only match dots" do + regexes = MRF.subdomains_regex(["unsafe.tld", "*.other.tld"]) + + refute MRF.subdomain_match?(regexes, "unsafextld") + refute MRF.subdomain_match?(regexes, "otherxtld") + refute MRF.subdomain_match?(regexes, "sub.otherxtld") + end + + test "accidental wildcards and regex syntax match nothing" do + regexes = MRF.subdomains_regex(["*unsafe.tld", "bad(.tld"]) + + refute MRF.subdomain_match?(regexes, "unsafe.tld") + refute MRF.subdomain_match?(regexes, "sub.unsafe.tld") + refute MRF.subdomain_match?(regexes, "bad.tld") end end From b486a84fef3090e70eadb215f62a8474edcde1d6 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sat, 31 Oct 2026 13:43:59 +0400 Subject: [PATCH 36/51] fix(context): show local-only replies to local users Ported from Akkoma 7869030e0. Co-authored-by: Oneric --- changelog.d/context-local-only.fix | 1 + lib/pleroma/web/activity_pub/activity_pub.ex | 19 ++++++++++----- .../controllers/status_controller_test.exs | 24 +++++++++++++++++++ 3 files changed, 38 insertions(+), 6 deletions(-) create mode 100644 changelog.d/context-local-only.fix diff --git a/changelog.d/context-local-only.fix b/changelog.d/context-local-only.fix new file mode 100644 index 000000000..5a1d10f3a --- /dev/null +++ b/changelog.d/context-local-only.fix @@ -0,0 +1 @@ +Show local-only replies in status context to local users who don't follow the replier diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index f6c74bdac..03f8447bb 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -466,21 +466,28 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end def fetch_activities_for_context_query(context, opts) do - public = [Constants.as_public()] + user = opts[:user] recipients = - if opts[:user], - do: [opts[:user].ap_id | User.following(opts[:user])] ++ public, - else: public + cond do + user && user.local -> + [Constants.as_public(), as_local_public(), user.ap_id | User.following(user)] + + user -> + [Constants.as_public(), user.ap_id | User.following(user)] + + true -> + [Constants.as_public()] + end from(activity in Activity) |> maybe_preload_objects(opts) |> maybe_preload_bookmarks(opts) |> maybe_set_thread_muted_field(opts) - |> restrict_unauthenticated(opts[:user]) + |> restrict_unauthenticated(user) |> restrict_blocked(opts) |> restrict_blockers_visibility(opts) - |> restrict_recipients(recipients, opts[:user]) + |> restrict_recipients(recipients, user) |> restrict_filtered(opts) |> where( [activity], diff --git a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs index bc859df48..6574f82f8 100644 --- a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs @@ -3093,6 +3093,30 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do } = response end + test "context shows local-only replies to local users only" do + poster = insert(:user) + replier = insert(:user) + %{conn: conn} = oauth_access(["read:statuses"]) + %{conn: remote_conn} = oauth_access(["read:statuses"], user: insert(:user, local: false)) + + {:ok, %{id: id1}} = CommonAPI.post(poster, %{status: "1"}) + + {:ok, %{id: id2}} = + CommonAPI.post(replier, %{status: "2", visibility: "local", in_reply_to_status_id: id1}) + + assert %{"descendants" => [%{"id" => ^id2}]} = + conn + |> get("/api/v1/statuses/#{id1}/context") + |> json_response_and_validate_schema(:ok) + + for conn <- [build_conn(), remote_conn] do + assert %{"descendants" => []} = + conn + |> get("/api/v1/statuses/#{id1}/context") + |> json_response_and_validate_schema(:ok) + end + end + test "context doesn't leak priv posts" do %{user: user, conn: conn} = oauth_access(["read:statuses"]) stranger = insert(:user) From c3fb297b895e8ea02a6f54f041e2a6b6858485de Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sat, 31 Oct 2026 13:43:59 +0400 Subject: [PATCH 37/51] fix(activitypub): apply the Note fixups to incoming Questions Ported from Akkoma 678f6fdb7. Co-authored-by: Oneric --- changelog.d/question-note-fixups.fix | 1 + .../article_note_page_validator.ex | 2 +- .../object_validators/question_validator.ex | 9 ++------- .../transmogrifier/question_handling_test.exs | 19 +++++++++++++++++++ 4 files changed, 23 insertions(+), 8 deletions(-) create mode 100644 changelog.d/question-note-fixups.fix diff --git a/changelog.d/question-note-fixups.fix b/changelog.d/question-note-fixups.fix new file mode 100644 index 000000000..31f54b944 --- /dev/null +++ b/changelog.d/question-note-fixups.fix @@ -0,0 +1 @@ +Apply the Note fixups to incoming Questions, so polls with a single non-array tag or attachment are accepted diff --git a/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex b/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex index 1d6e0abfb..2f4490e91 100644 --- a/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex @@ -213,7 +213,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidator do defp fix_misskey_content(object), do: object - defp fix(data) do + def fix(data) do data |> CommonFixes.fix_actor() |> CommonFixes.fix_object_defaults() diff --git a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex index 065c75910..be9dca4fa 100644 --- a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex @@ -6,10 +6,9 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do use Ecto.Schema alias Pleroma.EctoType.ActivityPub.ObjectValidators - alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes + alias Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidator alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations alias Pleroma.Web.ActivityPub.ObjectValidators.QuestionOptionsValidator - alias Pleroma.Web.ActivityPub.Transmogrifier import Ecto.Changeset @@ -62,11 +61,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do defp fix(data) do data - |> CommonFixes.fix_actor() - |> CommonFixes.fix_object_defaults() - |> CommonFixes.fix_quote_url() - |> CommonFixes.fix_likes() - |> Transmogrifier.fix_emoji() + |> ArticleNotePageValidator.fix() |> fix_closed() end diff --git a/test/pleroma/web/activity_pub/transmogrifier/question_handling_test.exs b/test/pleroma/web/activity_pub/transmogrifier/question_handling_test.exs index 2a4e78cf0..7da4e9541 100644 --- a/test/pleroma/web/activity_pub/transmogrifier/question_handling_test.exs +++ b/test/pleroma/web/activity_pub/transmogrifier/question_handling_test.exs @@ -103,6 +103,25 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.QuestionHandlingTest do assert Enum.sort(object.data["oneOf"]) == Enum.sort(options) end + test "Question activity with a single, non-array tag" do + tag = %{ + "href" => "https://gts.example/tags/caturday", + "name" => "#caturday", + "type" => "Hashtag" + } + + data = + File.read!("test/fixtures/mastodon-question-activity.json") + |> Jason.decode!() + |> Kernel.put_in(["object", "tag"], tag) + + {:ok, %Activity{local: false} = activity} = Transmogrifier.handle_incoming(data) + object = Object.normalize(activity, fetch: false) + + assert tag in object.data["tag"] + assert Object.hashtags(object) == ["caturday"] + end + test "Mastodon Question activity with custom emojis" do options = [ %{ From 58a6534a44f2cfc7214b915f2919374f70158fbf Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sat, 31 Oct 2026 13:43:59 +0400 Subject: [PATCH 38/51] fix(media_proxy): serve static previews instead of redirecting to the main host Ported from Akkoma 4a810bb2e. Co-authored-by: Oneric --- changelog.d/media-proxy-static-redirect.fix | 1 + .../web/media_proxy/media_proxy_controller.ex | 13 ------------- .../web/media_proxy/media_proxy_controller_test.exs | 13 +++++++++---- 3 files changed, 10 insertions(+), 17 deletions(-) create mode 100644 changelog.d/media-proxy-static-redirect.fix diff --git a/changelog.d/media-proxy-static-redirect.fix b/changelog.d/media-proxy-static-redirect.fix new file mode 100644 index 000000000..4408278ba --- /dev/null +++ b/changelog.d/media-proxy-static-redirect.fix @@ -0,0 +1 @@ +Media proxy: serve a preview for static=true on non-GIF files instead of redirecting to the main endpoint host diff --git a/lib/pleroma/web/media_proxy/media_proxy_controller.ex b/lib/pleroma/web/media_proxy/media_proxy_controller.ex index a0aafc32e..04e8ffd52 100644 --- a/lib/pleroma/web/media_proxy/media_proxy_controller.ex +++ b/lib/pleroma/web/media_proxy/media_proxy_controller.ex @@ -7,7 +7,6 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do alias Pleroma.Config alias Pleroma.Helpers.MediaHelper - alias Pleroma.Helpers.UriHelper alias Pleroma.ReverseProxy alias Pleroma.Web.MediaProxy alias Plug.Conn @@ -67,9 +66,6 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do static and content_type == "image/gif" -> handle_jpeg_preview(conn, media_proxy_url) - static -> - drop_static_param_and_redirect(conn) - content_type == "image/gif" -> conn |> put_status(301) @@ -166,15 +162,6 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do end end - defp drop_static_param_and_redirect(conn) do - uri_without_static_param = - conn - |> current_url() - |> UriHelper.modify_uri_params(%{}, ["static"]) - - redirect(conn, external: uri_without_static_param) - end - defp fallback_on_preview_error(conn, media_proxy_url) do redirect(conn, external: media_proxy_url) end diff --git a/test/pleroma/web/media_proxy/media_proxy_controller_test.exs b/test/pleroma/web/media_proxy/media_proxy_controller_test.exs index f7e52483c..71ad19c09 100644 --- a/test/pleroma/web/media_proxy/media_proxy_controller_test.exs +++ b/test/pleroma/web/media_proxy/media_proxy_controller_test.exs @@ -252,22 +252,27 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyControllerTest do assert redirected_to(response, 301) == media_proxy_url end - test "with `static` param and non-GIF image preview requested, " <> - "redirects to media preview proxy URI without `static` param", + test "with `static` param and non-GIF image preview requested, responds with a preview", %{ conn: conn, url: url, media_proxy_url: media_proxy_url } do + assert_dependencies_installed() + Tesla.Mock.mock(fn %{method: :head, url: ^media_proxy_url} -> %Tesla.Env{status: 200, body: "", headers: [{"content-type", "image/jpeg"}]} + + %{method: :get, url: ^media_proxy_url} -> + %Tesla.Env{status: 200, body: File.read!("test/fixtures/image.jpg")} end) response = get(conn, url <> "?static=true") - assert response.status == 302 - assert redirected_to(response) == url + assert response.status == 200 + assert Conn.get_resp_header(response, "content-type") == ["image/jpeg"] + assert response.resp_body != "" end test "with :min_content_length setting not matched by Content-Length header, " <> From 8ce283f966376409ec92160962f14ccf85018b8b Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sat, 31 Oct 2026 14:18:40 +0400 Subject: [PATCH 39/51] feat(http): back off from hosts that answer 429 or 503 Object fetches and WebFinger stop sending requests to a host until Retry-After or X-RateLimit-Reset (default 5 minutes, max 1 hour). Ported from Akkoma's Pleroma.HTTP.Backoff. Co-authored-by: FloatingGhost Co-authored-by: Oneric --- changelog.d/http-backoff.add | 1 + lib/mix/pleroma.ex | 2 +- lib/pleroma/application.ex | 1 + lib/pleroma/http/backoff.ex | 101 ++++++++++++ lib/pleroma/object/fetcher.ex | 2 +- lib/pleroma/web/web_finger.ex | 14 +- test/pleroma/http/backoff_test.exs | 154 ++++++++++++++++++ test/pleroma/object/fetcher_test.exs | 21 +++ test/pleroma/web/web_finger_test.exs | 25 +++ .../workers/remote_fetcher_worker_test.exs | 14 ++ 10 files changed, 330 insertions(+), 5 deletions(-) create mode 100644 changelog.d/http-backoff.add create mode 100644 lib/pleroma/http/backoff.ex create mode 100644 test/pleroma/http/backoff_test.exs diff --git a/changelog.d/http-backoff.add b/changelog.d/http-backoff.add new file mode 100644 index 000000000..5c9eb988b --- /dev/null +++ b/changelog.d/http-backoff.add @@ -0,0 +1 @@ +Back off from hosts that answer 429 or 503 to object fetches and WebFinger, until Retry-After or X-RateLimit-Reset (default 5 minutes, max 1 hour) diff --git a/lib/mix/pleroma.ex b/lib/mix/pleroma.ex index aebad6984..080dd1065 100644 --- a/lib/mix/pleroma.ex +++ b/lib/mix/pleroma.ex @@ -17,7 +17,7 @@ defmodule Mix.Pleroma do :oban, :logger_backends ] - @cachex_children ["object", "user", "scrubber", "web_resp"] + @cachex_children ["object", "user", "scrubber", "web_resp", "http_backoff"] @doc "Common functions to be reused in mix tasks" def start_pleroma do Pleroma.Config.Holder.save_default() diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index adc054f3b..2c8eede1c 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -198,6 +198,7 @@ defmodule Pleroma.Application do ), build_cachex("rel_me", limit: 2500), build_cachex("host_meta", default_ttl: :timer.minutes(120), limit: 5_000), + build_cachex("http_backoff", limit: 10_000), build_cachex("translations", default_ttl: :timer.hours(24), limit: 5_000) ] end diff --git a/lib/pleroma/http/backoff.ex b/lib/pleroma/http/backoff.ex new file mode 100644 index 000000000..0d2b99742 --- /dev/null +++ b/lib/pleroma/http/backoff.ex @@ -0,0 +1,101 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.HTTP.Backoff do + @moduledoc """ + GET wrapper that stops sending requests to a host after it answers + 429 or 503, until the time given by `Retry-After` or `X-RateLimit-Reset`. + + Based on Akkoma's `Pleroma.HTTP.Backoff`. + """ + + alias Pleroma.HTTP + + require Logger + + @cachex Pleroma.Config.get([:cachex, :provider], Cachex) + @cache :http_backoff_cache + + @default_seconds 5 * 60 + @max_seconds 60 * 60 + + @spec get(String.t(), list(), keyword()) :: {:ok, Tesla.Env.t()} | {:error, any()} + def get(url, headers \\ [], options \\ []) when is_binary(url) do + host = URI.parse(url).host + + case @cachex.get(@cache, host) do + {:ok, true} -> + {:error, :ratelimit} + + _ -> + url + |> HTTP.get(headers, options) + |> check_backoff(host) + end + end + + defp check_backoff({:ok, %{status: status, headers: headers}}, host) + when status in [429, 503] do + seconds = backoff_seconds(headers, DateTime.utc_now()) + Logger.warning("#{host} answered #{status}, backing off for #{seconds} seconds") + @cachex.put(@cache, host, true, ttl: :timer.seconds(seconds)) + {:error, :ratelimit} + end + + defp check_backoff(response, _host), do: response + + @doc """ + Seconds to back off for, from the first valid `Retry-After` or + `X-RateLimit-Reset` header. Missing, invalid or past values give the + default of #{@default_seconds} seconds; values are capped at #{@max_seconds}. + """ + @spec backoff_seconds(list(), DateTime.t()) :: pos_integer() + def backoff_seconds(headers, now) do + [ + parse_retry_after(header(headers, "retry-after"), now), + parse_timestamp(header(headers, "x-ratelimit-reset"), now) + ] + |> Enum.find(@default_seconds, &is_integer/1) + |> min(@max_seconds) + end + + defp header(headers, name) do + Enum.find_value(headers, fn + {key, value} when is_binary(key) and is_binary(value) -> + if String.downcase(key) == name, do: String.trim(value) + + _ -> + nil + end) + end + + # Retry-After is either delta-seconds or an HTTP-date (RFC 9110, 10.2.3) + defp parse_retry_after(nil, _now), do: nil + + defp parse_retry_after(value, now) do + case Integer.parse(value) do + {seconds, ""} when seconds > 0 -> seconds + {_, ""} -> nil + _ -> parse_timestamp(value, now) + end + end + + defp parse_timestamp(nil, _now), do: nil + + defp parse_timestamp(value, now) do + with {:ok, %DateTime{} = datetime} <- parse_datetime(value), + seconds when seconds > 0 <- DateTime.diff(datetime, now) do + seconds + else + _ -> nil + end + end + + defp parse_datetime(value) do + with {:error, _} <- Timex.parse(value, "{RFC1123}"), + {:ok, datetime, _offset} <- DateTime.from_iso8601(value) do + {:ok, datetime} + end + end +end diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index 706ee317c..3313583cc 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -216,7 +216,7 @@ defmodule Pleroma.Object.Fetcher do |> maybe_date_fetch(date) |> sign_fetch(id, date) - case HTTP.get(id, headers) do + case HTTP.Backoff.get(id, headers) do {:ok, %{body: body, status: code, headers: headers, url: final_url}} when code in 200..299 -> remote_host = if final_url, do: URI.parse(final_url).host, else: nil diff --git a/lib/pleroma/web/web_finger.ex b/lib/pleroma/web/web_finger.ex index 62a2715f5..cb312405c 100644 --- a/lib/pleroma/web/web_finger.ex +++ b/lib/pleroma/web/web_finger.ex @@ -158,7 +158,11 @@ defmodule Pleroma.Web.WebFinger do @cachex Pleroma.Config.get([:cachex, :provider], Cachex) def find_lrdd_template(domain) do @cachex.fetch!(:host_meta_cache, domain, fn _ -> - {:commit, fetch_lrdd_template(domain)} + case fetch_lrdd_template(domain) do + # retry once the backoff expires instead of caching the miss + {:error, :ratelimit} = error -> {:ignore, error} + result -> {:commit, result} + end end) rescue e -> {:error, "Cachex error: #{inspect(e)}"} @@ -168,9 +172,13 @@ defmodule Pleroma.Web.WebFinger do # WebFinger is restricted to HTTPS - https://tools.ietf.org/html/rfc7033#section-9.1 meta_url = "https://#{domain}/.well-known/host-meta" - with {:ok, %{status: status, body: body}} when status in 200..299 <- HTTP.get(meta_url) do + with {:ok, %{status: status, body: body}} when status in 200..299 <- + HTTP.Backoff.get(meta_url) do get_template_from_xml(body) else + {:error, :ratelimit} = error -> + error + error -> Logger.warning("Can't find LRDD template in #{inspect(meta_url)}: #{inspect(error)}") {:error, :lrdd_not_found} @@ -210,7 +218,7 @@ defmodule Pleroma.Web.WebFinger do with address when is_binary(address) <- get_address_from_domain(domain, account), {:ok, %{status: status, body: body, headers: headers}} when status in 200..299 <- - HTTP.get( + HTTP.Backoff.get( address, [{"accept", "application/xrd+xml,application/jrd+json"}] ) do diff --git a/test/pleroma/http/backoff_test.exs b/test/pleroma/http/backoff_test.exs new file mode 100644 index 000000000..48c8514bd --- /dev/null +++ b/test/pleroma/http/backoff_test.exs @@ -0,0 +1,154 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +# Based on Akkoma's test/pleroma/http/backoff_test.exs +defmodule Pleroma.HTTP.BackoffTest do + use Pleroma.DataCase, async: false + + alias Pleroma.HTTP.Backoff + + @cache :http_backoff_cache + @url "https://ratelimited.example.com/users/alice" + @host "ratelimited.example.com" + + defp mock_response(status, headers \\ []) do + test_pid = self() + + Tesla.Mock.mock(fn %{method: :get} -> + send(test_pid, :request) + %Tesla.Env{status: status, headers: headers, body: ""} + end) + end + + defp ttl_seconds do + {:ok, ttl} = Cachex.ttl(@cache, @host) + div(ttl, 1000) + end + + defp assert_ttl_about(expected) do + ttl = ttl_seconds() + assert ttl in (expected - 15)..expected//1, "expected a TTL near #{expected}, got #{ttl}" + end + + describe "get/3" do + test "returns the response when the host is not backed off" do + mock_response(200) + + assert {:ok, %Tesla.Env{status: 200}} = Backoff.get(@url) + assert {:ok, nil} = Cachex.get(@cache, @host) + end + + test "does not send a request while the host is backed off" do + Cachex.put(@cache, @host, true) + mock_response(200) + + assert {:error, :ratelimit} = Backoff.get(@url) + refute_received :request + end + + for status <- [429, 503] do + test "backs off from the host after a #{status} response" do + mock_response(unquote(status)) + + assert {:error, :ratelimit} = Backoff.get(@url) + assert_received :request + assert {:ok, true} = Cachex.get(@cache, @host) + + assert {:error, :ratelimit} = Backoff.get(@url <> "/other") + refute_received :request + end + end + + test "uses a default backoff without headers" do + mock_response(429) + + assert {:error, :ratelimit} = Backoff.get(@url) + assert_ttl_about(300) + end + + test "only backs off from the host that answered" do + mock_response(429) + assert {:error, :ratelimit} = Backoff.get(@url) + + mock_response(200) + assert {:ok, _} = Backoff.get("https://other.example.com/users/bob") + end + end + + describe "backoff headers" do + test "parses Retry-After in seconds" do + mock_response(429, [{"retry-after", "600"}]) + + assert {:error, :ratelimit} = Backoff.get(@url) + assert_ttl_about(600) + end + + test "parses Retry-After as an HTTP-date" do + date = + DateTime.utc_now() + |> DateTime.add(600) + |> Timex.format!("{WDshort}, {0D} {Mshort} {YYYY} {h24}:{m}:{s} GMT") + + mock_response(503, [{"retry-after", date}]) + + assert {:error, :ratelimit} = Backoff.get(@url) + assert_ttl_about(600) + end + + test "parses Retry-After as an ISO 8601 timestamp" do + date = DateTime.utc_now() |> DateTime.add(600) |> DateTime.to_iso8601() + mock_response(429, [{"retry-after", date}]) + + assert {:error, :ratelimit} = Backoff.get(@url) + assert_ttl_about(600) + end + + test "parses X-RateLimit-Reset as an ISO 8601 timestamp" do + date = DateTime.utc_now() |> DateTime.add(600) |> DateTime.to_iso8601() + mock_response(429, [{"x-ratelimit-reset", date}]) + + assert {:error, :ratelimit} = Backoff.get(@url) + assert_ttl_about(600) + end + + test "matches header names case-insensitively" do + mock_response(429, [{"Retry-After", "600"}]) + + assert {:error, :ratelimit} = Backoff.get(@url) + assert_ttl_about(600) + end + + test "prefers Retry-After over X-RateLimit-Reset" do + date = DateTime.utc_now() |> DateTime.add(1200) |> DateTime.to_iso8601() + mock_response(429, [{"x-ratelimit-reset", date}, {"retry-after", "600"}]) + + assert {:error, :ratelimit} = Backoff.get(@url) + assert_ttl_about(600) + end + + test "falls back to the next header when one is invalid" do + date = DateTime.utc_now() |> DateTime.add(600) |> DateTime.to_iso8601() + mock_response(429, [{"retry-after", "soon"}, {"x-ratelimit-reset", date}]) + + assert {:error, :ratelimit} = Backoff.get(@url) + assert_ttl_about(600) + end + + for value <- ["soon", "-5", "0", "12.5", "", "Wed, 21 Oct 2015 07:28:00 GMT"] do + test "ignores the invalid or past Retry-After value #{inspect(value)}" do + mock_response(429, [{"retry-after", unquote(value)}]) + + assert {:error, :ratelimit} = Backoff.get(@url) + assert_ttl_about(300) + end + end + + test "caps absurd values at one hour" do + mock_response(429, [{"retry-after", "99999999999"}]) + + assert {:error, :ratelimit} = Backoff.get(@url) + assert_ttl_about(3600) + end + end +end diff --git a/test/pleroma/object/fetcher_test.exs b/test/pleroma/object/fetcher_test.exs index 61a6fb517..80b172a36 100644 --- a/test/pleroma/object/fetcher_test.exs +++ b/test/pleroma/object/fetcher_test.exs @@ -94,6 +94,27 @@ defmodule Pleroma.Object.FetcherTest do end end + describe "rate limiting" do + test "a backed-off host gets no further requests" do + test_pid = self() + + mock(fn %{method: :get, url: url} -> + send(test_pid, {:request, url}) + %Tesla.Env{status: 429, headers: [{"retry-after", "600"}]} + end) + + first = "https://ratelimited.example.com/objects/1" + second = "https://ratelimited.example.com/objects/2" + + assert {:fetch, {:error, :ratelimit}} = Fetcher.fetch_object_from_id(first) + assert_received {:request, ^first} + + assert {:error, :ratelimit} = Fetcher.fetch_and_contain_remote_object_from_id(second) + assert {:fetch, {:error, :ratelimit}} = Fetcher.fetch_object_from_id(second) + refute_received {:request, _} + end + end + describe "max thread distance restriction" do @ap_id "http://mastodon.example.org/@admin/99541947525187367" setup do: clear_config([:instance, :federation_incoming_replies_max_depth]) diff --git a/test/pleroma/web/web_finger_test.exs b/test/pleroma/web/web_finger_test.exs index da10abdd2..fc99bee41 100644 --- a/test/pleroma/web/web_finger_test.exs +++ b/test/pleroma/web/web_finger_test.exs @@ -305,4 +305,29 @@ defmodule Pleroma.Web.WebFingerTest do {:ok, _data} = WebFinger.finger("a@sub.mastodon.example") end end + + describe "rate limiting" do + setup do + Mox.stub_with(Pleroma.CachexMock, Pleroma.CachexProxy) + test_pid = self() + + Tesla.Mock.mock_global(fn %{url: url} -> + send(test_pid, {:request, url}) + %Tesla.Env{status: 429, headers: [{"retry-after", "600"}]} + end) + end + + test "backs off from a host that rate limits host-meta" do + assert {:error, :ratelimit} = WebFinger.finger("alice@ratelimited.example") + assert_received {:request, "https://ratelimited.example/.well-known/host-meta"} + refute_received {:request, _} + + # a rate-limited host-meta lookup must not be cached as "no LRDD template" + assert {:ok, nil} = Cachex.get(:host_meta_cache, "ratelimited.example") + + assert {:error, :ratelimit} = WebFinger.finger("bob@ratelimited.example") + assert {:error, :ratelimit} = WebFinger.find_lrdd_template("ratelimited.example") + refute_received {:request, _} + end + end end diff --git a/test/pleroma/workers/remote_fetcher_worker_test.exs b/test/pleroma/workers/remote_fetcher_worker_test.exs index a3b900a9b..9945ca143 100644 --- a/test/pleroma/workers/remote_fetcher_worker_test.exs +++ b/test/pleroma/workers/remote_fetcher_worker_test.exs @@ -13,6 +13,7 @@ defmodule Pleroma.Workers.RemoteFetcherWorkerTest do @unauthorized_object "https://unauthorized.example.com/" @depth_object "https://depth.example.com/" @content_type_object "https://bad_content_type.example.com/" + @ratelimited_object "https://ratelimited.example.com/" describe "RemoteFetcherWorker" do setup do @@ -37,6 +38,11 @@ defmodule Pleroma.Workers.RemoteFetcherWorkerTest do status: 200 } + %{method: :get, url: @ratelimited_object} -> + %Tesla.Env{ + status: 429 + } + %{method: :get, url: @content_type_object} -> %Tesla.Env{ status: 200, @@ -73,6 +79,14 @@ defmodule Pleroma.Workers.RemoteFetcherWorkerTest do }) end + test "retries jobs for a rate-limited host" do + assert {:error, :ratelimit} = + perform_job(RemoteFetcherWorker, %{ + "op" => "fetch_remote", + "id" => @ratelimited_object + }) + end + test "does not retry jobs for when object returns wrong content type" do assert {:cancel, _} = perform_job(RemoteFetcherWorker, %{ From b29d9d09810c58a1e48c85796d75283b74eb0ffb Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sat, 31 Oct 2026 14:35:58 +0400 Subject: [PATCH 40/51] fix(workers): snooze incoming jobs while a host is backed off With 5 attempts and Oban's short retry delays, an incoming activity from an unknown actor on a backed-off host was discarded without a request. ReceiverWorker and SignatureRetryWorker now snooze for up to 2 hours. Also lowercase backoff host keys and log backed-off user fetches at debug. --- lib/pleroma/http/backoff.ex | 23 ++++++++++++++- lib/pleroma/web/activity_pub/activity_pub.ex | 4 +++ lib/pleroma/workers/receiver_worker.ex | 5 +++- lib/pleroma/workers/signature_retry_worker.ex | 28 +++++++++++-------- test/pleroma/http/backoff_test.exs | 9 ++++++ test/pleroma/web/web_finger_test.exs | 1 + test/pleroma/workers/receiver_worker_test.exs | 22 +++++++++++++++ .../workers/signature_retry_worker_test.exs | 18 ++++++++++++ 8 files changed, 97 insertions(+), 13 deletions(-) diff --git a/lib/pleroma/http/backoff.ex b/lib/pleroma/http/backoff.ex index 0d2b99742..44b66fbdd 100644 --- a/lib/pleroma/http/backoff.ex +++ b/lib/pleroma/http/backoff.ex @@ -19,10 +19,11 @@ defmodule Pleroma.HTTP.Backoff do @default_seconds 5 * 60 @max_seconds 60 * 60 + @max_job_age 2 * 60 * 60 @spec get(String.t(), list(), keyword()) :: {:ok, Tesla.Env.t()} | {:error, any()} def get(url, headers \\ [], options \\ []) when is_binary(url) do - host = URI.parse(url).host + host = url |> URI.parse() |> Map.get(:host) |> downcase() case @cachex.get(@cache, host) do {:ok, true} -> @@ -35,6 +36,9 @@ defmodule Pleroma.HTTP.Backoff do end end + defp downcase(nil), do: nil + defp downcase(host), do: String.downcase(host) + defp check_backoff({:ok, %{status: status, headers: headers}}, host) when status in [429, 503] do seconds = backoff_seconds(headers, DateTime.utc_now()) @@ -45,6 +49,23 @@ defmodule Pleroma.HTTP.Backoff do defp check_backoff(response, _host), do: response + @doc """ + Turns an Oban job's `{:error, :ratelimit}` result into a snooze for the + default backoff, so the job's attempts aren't all spent while the host is + backed off. Jobs inserted more than #{@max_job_age} seconds ago get the + error back and fail as usual. + """ + @spec snooze_ratelimited(any(), Oban.Job.t()) :: any() + def snooze_ratelimited({:error, :ratelimit} = error, %Oban.Job{inserted_at: inserted_at}) do + if is_nil(inserted_at) or DateTime.diff(DateTime.utc_now(), inserted_at) < @max_job_age do + {:snooze, @default_seconds} + else + error + end + end + + def snooze_ratelimited(result, _job), do: result + @doc """ Seconds to back off for, from the first valid `Retry-After` or `X-RateLimit-Reset` header. Missing, invalid or past values give the diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index f6c74bdac..a6e6d4618 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -1949,6 +1949,10 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do Logger.info("Rejected user #{ap_id}: #{inspect(reason)}") {:error, e} + {:error, :ratelimit = e} -> + Logger.debug("Backing off from fetching user #{ap_id}") + {:error, e} + {:error, e} -> Logger.error("Could not decode user at fetch #{ap_id}, #{inspect(e)}") {:error, e} diff --git a/lib/pleroma/workers/receiver_worker.ex b/lib/pleroma/workers/receiver_worker.ex index 3afbe138d..4d1b5ba92 100644 --- a/lib/pleroma/workers/receiver_worker.ex +++ b/lib/pleroma/workers/receiver_worker.ex @@ -3,6 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Workers.ReceiverWorker do + alias Pleroma.HTTP.Backoff alias Pleroma.Instances alias Pleroma.Web.Federator alias Pleroma.Workers.SignatureRetryWorker @@ -14,7 +15,9 @@ defmodule Pleroma.Workers.ReceiverWorker do if signature_retry_job?(args) do perform_signature_retry(job) else - perform_incoming(params) + params + |> perform_incoming() + |> Backoff.snooze_ratelimited(job) end end diff --git a/lib/pleroma/workers/signature_retry_worker.ex b/lib/pleroma/workers/signature_retry_worker.ex index 28958faff..159d802ad 100644 --- a/lib/pleroma/workers/signature_retry_worker.ex +++ b/lib/pleroma/workers/signature_retry_worker.ex @@ -3,6 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Workers.SignatureRetryWorker do + alias Pleroma.HTTP.Backoff alias Pleroma.Instances alias Pleroma.Signature alias Pleroma.User @@ -16,16 +17,18 @@ defmodule Pleroma.Workers.SignatureRetryWorker do use Oban.Worker, queue: :federator_incoming, max_attempts: 5, unique: [period: :infinity] @impl true - def perform(%Job{ - args: %{ - "op" => "incoming_failed_signature_ap_doc", - "method" => method, - "params" => params, - "req_headers" => req_headers, - "request_path" => request_path, - "query_string" => query_string - } - }) + def perform( + %Job{ + args: %{ + "op" => "incoming_failed_signature_ap_doc", + "method" => method, + "params" => params, + "req_headers" => req_headers, + "request_path" => request_path, + "query_string" => query_string + } + } = job + ) when is_binary(method) and is_map(params) and is_list(req_headers) and is_binary(request_path) and is_binary(query_string) do case normalize_req_headers(req_headers) do @@ -58,7 +61,10 @@ defmodule Pleroma.Workers.SignatureRetryWorker do {:ok, res} else - e -> process_errors(e, retry_log_context(params, request_path, signature_actor_result)) + e -> + e + |> process_errors(retry_log_context(params, request_path, signature_actor_result)) + |> Backoff.snooze_ratelimited(job) end e -> diff --git a/test/pleroma/http/backoff_test.exs b/test/pleroma/http/backoff_test.exs index 48c8514bd..f893eef2e 100644 --- a/test/pleroma/http/backoff_test.exs +++ b/test/pleroma/http/backoff_test.exs @@ -67,6 +67,15 @@ defmodule Pleroma.HTTP.BackoffTest do assert_ttl_about(300) end + test "treats host names case-insensitively" do + mock_response(429) + assert {:error, :ratelimit} = Backoff.get("https://RateLimited.Example.com/users/alice") + assert_received :request + + assert {:error, :ratelimit} = Backoff.get(@url) + refute_received :request + end + test "only backs off from the host that answered" do mock_response(429) assert {:error, :ratelimit} = Backoff.get(@url) diff --git a/test/pleroma/web/web_finger_test.exs b/test/pleroma/web/web_finger_test.exs index fc99bee41..cf7cfa8b6 100644 --- a/test/pleroma/web/web_finger_test.exs +++ b/test/pleroma/web/web_finger_test.exs @@ -308,6 +308,7 @@ defmodule Pleroma.Web.WebFingerTest do describe "rate limiting" do setup do + # the module setup stubs NullCache, the backoff needs a real cache Mox.stub_with(Pleroma.CachexMock, Pleroma.CachexProxy) test_pid = self() diff --git a/test/pleroma/workers/receiver_worker_test.exs b/test/pleroma/workers/receiver_worker_test.exs index ea05f38f1..855ae3e2a 100644 --- a/test/pleroma/workers/receiver_worker_test.exs +++ b/test/pleroma/workers/receiver_worker_test.exs @@ -133,6 +133,28 @@ defmodule Pleroma.Workers.ReceiverWorkerTest do end end + describe "rate-limited fetches" do + setup do + Cachex.put(:http_backoff_cache, "ratelimited.example", true) + + params = + insert(:note_activity).data + |> Map.put("actor", "https://ratelimited.example/users/alice") + + %{args: %{"op" => "incoming_ap_doc", "params" => params}} + end + + test "snooze while the actor's host is backed off", %{args: args} do + assert {:snooze, 300} = perform_job(ReceiverWorker, args) + end + + test "stop snoozing two hours after the job was inserted", %{args: args} do + inserted_at = DateTime.add(DateTime.utc_now(), -121, :minute) + + assert {:error, :ratelimit} = perform_job(ReceiverWorker, args, inserted_at: inserted_at) + end + end + test "cancels due to origin containment" do params = insert(:note_activity).data diff --git a/test/pleroma/workers/signature_retry_worker_test.exs b/test/pleroma/workers/signature_retry_worker_test.exs index 3806ecac9..56835bbbc 100644 --- a/test/pleroma/workers/signature_retry_worker_test.exs +++ b/test/pleroma/workers/signature_retry_worker_test.exs @@ -306,6 +306,24 @@ defmodule Pleroma.Workers.SignatureRetryWorkerTest do refute Activity.get_by_ap_id(create["id"]) end + test "snoozes while the payload actor's host is backed off" do + Cachex.put(:http_backoff_cache, "ratelimited.example", true) + alice = build(:user, local: false, ap_id: "https://ratelimited.example/users/alice") + + params = %{ + "type" => "Create", + "actor" => alice.ap_id, + "id" => "https://ratelimited.example/activities/1", + "object" => "https://ratelimited.example/objects/1" + } + + job = failed_signature_job(params, signature_headers_for(alice)) + assert {:snooze, 300} = SignatureRetryWorker.perform(job) + + old_job = %{job | inserted_at: DateTime.add(DateTime.utc_now(), -121, :minute)} + assert {:error, :ratelimit} = SignatureRetryWorker.perform(old_job) + end + test "processes the activity after refetching a valid matching signature" do alice = insert(:user, local: false, ap_id: "https://one.com/users/alice") From 654a41b2c92acf7fe462c5452fc36d408e32b494 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sat, 31 Oct 2026 15:37:41 +0400 Subject: [PATCH 41/51] fix(activitypub): accept emoji reactions with a single tag object Ported from Akkoma 462225880. Co-authored-by: Oneric --- changelog.d/emoji-react-single-tag.fix | 1 + .../emoji_react_validator.ex | 6 ++++- .../emoji_react_handling_test.exs | 25 +++++++++++++++++++ 3 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 changelog.d/emoji-react-single-tag.fix diff --git a/changelog.d/emoji-react-single-tag.fix b/changelog.d/emoji-react-single-tag.fix new file mode 100644 index 000000000..565775830 --- /dev/null +++ b/changelog.d/emoji-react-single-tag.fix @@ -0,0 +1 @@ +Accept incoming emoji reactions whose tag is a single object instead of an array diff --git a/lib/pleroma/web/activity_pub/object_validators/emoji_react_validator.ex b/lib/pleroma/web/activity_pub/object_validators/emoji_react_validator.ex index 65ba047e6..b7197c3f3 100644 --- a/lib/pleroma/web/activity_pub/object_validators/emoji_react_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/emoji_react_validator.ex @@ -57,7 +57,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.EmojiReactValidator do |> CommonFixes.fix_actor() |> CommonFixes.fix_activity_addressing() - data = Map.put_new(data, "tag", []) + data = fix_tag(data) case Object.normalize(data["object"]) do %Object{} = object -> @@ -70,6 +70,10 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.EmojiReactValidator do end end + # JSON-LD compaction turns a single-element array into a bare object + defp fix_tag(%{"tag" => %{} = tag} = data), do: Map.put(data, "tag", [tag]) + defp fix_tag(data), do: Map.put_new(data, "tag", []) + defp fix_emoji_qualification(%{"content" => emoji} = data) do new_emoji = Pleroma.Emoji.fully_qualify_emoji(emoji) diff --git a/test/pleroma/web/activity_pub/transmogrifier/emoji_react_handling_test.exs b/test/pleroma/web/activity_pub/transmogrifier/emoji_react_handling_test.exs index f2e1cefa3..2e1841559 100644 --- a/test/pleroma/web/activity_pub/transmogrifier/emoji_react_handling_test.exs +++ b/test/pleroma/web/activity_pub/transmogrifier/emoji_react_handling_test.exs @@ -86,6 +86,31 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.EmojiReactHandlingTest do ) end + test "it works for incoming custom emoji reactions with a single tag object" do + user = insert(:user) + other_user = insert(:user, local: false) + {:ok, activity} = CommonAPI.post(user, %{status: "hello"}) + + data = + File.read!("test/fixtures/custom-emoji-reaction.json") + |> Jason.decode!() + |> Map.put("object", activity.data["object"]) + |> Map.put("actor", other_user.ap_id) + + [tag] = data["tag"] + data = Map.put(data, "tag", tag) + + {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data) + + assert data["content"] == ":hanapog:" + assert [%{"name" => "hanapog"}] = data["tag"] + + object = Object.get_by_ap_id(data["object"]) + url = tag["icon"]["url"] + + assert match?([["hanapog", _, ^url]], object.data["reactions"]) + end + test "it works for incoming unqualified emoji reactions" do user = insert(:user) other_user = insert(:user, local: false) From e0546f188da23c9babb4589c2fc7286d4699cc0f Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sat, 31 Oct 2026 15:37:41 +0400 Subject: [PATCH 42/51] fix(activitypub): do not require the personal inbox owner to be addressed Bridgy Fed delivers Follow and Accept to the personal inbox addressed only to Public. Any server can already address a local user directly, so the check gave no protection. Ported from Akkoma 2b885288f. Co-authored-by: Oneric --- changelog.d/personal-inbox-addressing.fix | 1 + .../activity_pub/activity_pub_controller.ex | 1 - lib/pleroma/web/activity_pub/utils.ex | 16 ----- .../activity_pub_controller_test.exs | 24 +++++++ test/pleroma/web/activity_pub/utils_test.exs | 68 ------------------- 5 files changed, 25 insertions(+), 85 deletions(-) create mode 100644 changelog.d/personal-inbox-addressing.fix diff --git a/changelog.d/personal-inbox-addressing.fix b/changelog.d/personal-inbox-addressing.fix new file mode 100644 index 000000000..9a2ba48a2 --- /dev/null +++ b/changelog.d/personal-inbox-addressing.fix @@ -0,0 +1 @@ +Accept activities delivered to a personal inbox that do not address the inbox owner (fixes Bridgy Fed follows) diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index dd5f6028f..a45f7bf27 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -341,7 +341,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do {:sender_exists, User.get_or_fetch_by_ap_id(params["actor"])}, {:recipient_active, true} <- {:recipient_active, recipient.is_active}, {:sender_active, true} <- {:sender_active, actor.is_active}, - true <- Utils.recipient_in_message(recipient, actor, params), params <- Utils.maybe_splice_recipient(recipient.ap_id, params) do Federator.incoming_ap_doc(params) json(conn, "ok") diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index 0af4ceaf5..5c25f4c69 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -78,22 +78,6 @@ defmodule Pleroma.Web.ActivityPub.Utils do [params["to"], params["cc"], params["bto"], params["bcc"]] |> Enum.any?(&label_in_collection?(label, &1)) - @spec unaddressed_message?(map()) :: boolean() - def unaddressed_message?(params), - do: - [params["to"], params["cc"], params["bto"], params["bcc"]] - |> Enum.all?(fn - nil -> true - [] -> true - _ -> false - end) - - @spec recipient_in_message(User.t(), User.t(), map()) :: boolean() - def recipient_in_message(%User{ap_id: ap_id} = recipient, %User{} = actor, params), - do: - label_in_message?(ap_id, params) || unaddressed_message?(params) || - User.following?(recipient, actor) - defp extract_list(target) when is_binary(target), do: [target] defp extract_list(lst) when is_list(lst), do: lst defp extract_list(_), do: [] diff --git a/test/pleroma/web/activity_pub/activity_pub_controller_test.exs b/test/pleroma/web/activity_pub/activity_pub_controller_test.exs index 606d47c8d..e1376b460 100644 --- a/test/pleroma/web/activity_pub/activity_pub_controller_test.exs +++ b/test/pleroma/web/activity_pub/activity_pub_controller_test.exs @@ -1872,6 +1872,30 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do assert Activity.get_by_ap_id(data["id"]) end + test "it accepts activities that do not address the inbox owner", %{conn: conn} do + user = insert(:user) + actor = insert(:user, local: false) + + data = %{ + "@context" => "https://www.w3.org/ns/activitystreams", + "type" => "Follow", + "id" => "#{actor.ap_id}/follows/1", + "actor" => actor.ap_id, + "object" => user.ap_id, + "to" => ["https://www.w3.org/ns/activitystreams#Public"] + } + + conn = + conn + |> assign_valid_signature_for_actor(data["actor"]) + |> put_req_header("content-type", "application/activity+json") + |> post("/users/#{user.nickname}/inbox", data) + + assert "ok" == json_response(conn, 200) + ObanHelpers.perform(all_enqueued(worker: ReceiverWorker)) + assert Activity.get_by_ap_id(data["id"]) + end + test "it returns an error when receiving an activity sent to a deactivated user", %{ conn: conn, data: data diff --git a/test/pleroma/web/activity_pub/utils_test.exs b/test/pleroma/web/activity_pub/utils_test.exs index 93234a015..88c8a283b 100644 --- a/test/pleroma/web/activity_pub/utils_test.exs +++ b/test/pleroma/web/activity_pub/utils_test.exs @@ -406,74 +406,6 @@ defmodule Pleroma.Web.ActivityPub.UtilsTest do end end - describe "recipient_in_message/3" do - test "returns true when recipient in `to`" do - recipient = insert(:user) - actor = insert(:user) - assert Utils.recipient_in_message(recipient, actor, %{"to" => recipient.ap_id}) - - assert Utils.recipient_in_message( - recipient, - actor, - %{"to" => [recipient.ap_id], "cc" => ""} - ) - end - - test "returns true when recipient in `cc`" do - recipient = insert(:user) - actor = insert(:user) - assert Utils.recipient_in_message(recipient, actor, %{"cc" => recipient.ap_id}) - - assert Utils.recipient_in_message( - recipient, - actor, - %{"cc" => [recipient.ap_id], "to" => ""} - ) - end - - test "returns true when recipient in `bto`" do - recipient = insert(:user) - actor = insert(:user) - assert Utils.recipient_in_message(recipient, actor, %{"bto" => recipient.ap_id}) - - assert Utils.recipient_in_message( - recipient, - actor, - %{"bcc" => "", "bto" => [recipient.ap_id]} - ) - end - - test "returns true when recipient in `bcc`" do - recipient = insert(:user) - actor = insert(:user) - assert Utils.recipient_in_message(recipient, actor, %{"bcc" => recipient.ap_id}) - - assert Utils.recipient_in_message( - recipient, - actor, - %{"bto" => "", "bcc" => [recipient.ap_id]} - ) - end - - test "returns true when message without addresses fields" do - recipient = insert(:user) - actor = insert(:user) - assert Utils.recipient_in_message(recipient, actor, %{"bccc" => recipient.ap_id}) - - assert Utils.recipient_in_message( - recipient, - actor, - %{"btod" => "", "bccc" => [recipient.ap_id]} - ) - end - - test "returns false" do - recipient = insert(:user) - actor = insert(:user) - refute Utils.recipient_in_message(recipient, actor, %{"to" => "ap_id"}) - end - end - describe "lazy_put_activity_defaults/2" do test "returns map with id and published data" do note_activity = insert(:note_activity) From 3cd201c365e10cb54466a3b8827c5c38f8ba651e Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sat, 31 Oct 2026 15:53:41 +0400 Subject: [PATCH 43/51] fix(activitypub): refetch a stale Move target once A Move usually follows right after the alias was added, and the cached remote target may not have it yet. Refetch the target when the alias is missing instead of rejecting the Move. Same issue as Akkoma f18e2ba42. --- changelog.d/move-refetch-target.fix | 1 + .../web/activity_pub/transmogrifier.ex | 18 ++++++- .../web/activity_pub/transmogrifier_test.exs | 47 +++++++++++++++++++ 3 files changed, 65 insertions(+), 1 deletion(-) create mode 100644 changelog.d/move-refetch-target.fix diff --git a/changelog.d/move-refetch-target.fix b/changelog.d/move-refetch-target.fix new file mode 100644 index 000000000..1c200568e --- /dev/null +++ b/changelog.d/move-refetch-target.fix @@ -0,0 +1 @@ +Accept a Move when the cached remote target does not list the alias yet, by refetching the target once diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 33ab62ca3..0d4b15288 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -799,7 +799,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do ) do with %User{} = origin_user <- User.get_cached_by_ap_id(origin_actor), {:ok, %User{} = target_user} <- User.get_or_fetch_by_ap_id(target_actor), - true <- origin_actor in target_user.also_known_as do + {:ok, %User{} = target_user} <- refetch_unless_aliased(target_user, origin_actor) do ActivityPub.move(origin_user, target_user, false) else _e -> :error @@ -1204,4 +1204,20 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def maybe_fix_user_url(data), do: data def maybe_fix_user_object(data), do: maybe_fix_user_url(data) + + # A Move usually follows right after the alias was added, which the cached + # remote target may not have yet. + defp refetch_unless_aliased(%User{} = target, origin_ap_id, refetch? \\ true) do + cond do + origin_ap_id in target.also_known_as -> + {:ok, target} + + refetch? and not target.local -> + with {:ok, %User{} = target} <- User.fetch_by_ap_id(target.ap_id), + do: refetch_unless_aliased(target, origin_ap_id, false) + + true -> + :error + end + end end diff --git a/test/pleroma/web/activity_pub/transmogrifier_test.exs b/test/pleroma/web/activity_pub/transmogrifier_test.exs index bedb466a4..0e4a4c2db 100644 --- a/test/pleroma/web/activity_pub/transmogrifier_test.exs +++ b/test/pleroma/web/activity_pub/transmogrifier_test.exs @@ -147,6 +147,53 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do assert activity.data["type"] == "Move" end + test "it refetches a cached remote Move target that lacks the alias" do + old_user = insert(:user, local: false, ap_id: "https://old.example/users/old") + + new_user = + insert(:user, + local: false, + ap_id: "https://new.example/users/new", + nickname: "new@new.example", + last_refreshed_at: NaiveDateTime.utc_now() + ) + + actor = %{ + "@context" => "https://www.w3.org/ns/activitystreams", + "id" => new_user.ap_id, + "type" => "Person", + "preferredUsername" => "new", + "inbox" => new_user.ap_id <> "/inbox", + "outbox" => new_user.ap_id <> "/outbox", + "followers" => new_user.ap_id <> "/followers", + "following" => new_user.ap_id <> "/following", + "alsoKnownAs" => [old_user.ap_id] + } + + Tesla.Mock.mock(fn + %{method: :get, url: "https://new.example/users/new"} -> + %Tesla.Env{ + status: 200, + headers: [{"content-type", "application/activity+json"}], + body: Jason.encode!(actor) + } + + _ -> + %Tesla.Env{status: 404} + end) + + message = %{ + "@context" => "https://www.w3.org/ns/activitystreams", + "type" => "Move", + "actor" => old_user.ap_id, + "object" => old_user.ap_id, + "target" => new_user.ap_id + } + + assert {:ok, %Activity{data: %{"type" => "Move"}}} = Transmogrifier.handle_incoming(message) + assert old_user.ap_id in User.get_cached_by_ap_id(new_user.ap_id).also_known_as + end + test "it fixes both the Create and object contexts in a reply" do insert(:user, ap_id: "https://mk.absturztau.be/users/8ozbzjs3o8") insert(:user, ap_id: "https://p.helene.moe/users/helene") From 6f31149d21b2e7e39ff48df3dd8e28602fc3ac53 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sat, 31 Oct 2026 15:53:51 +0400 Subject: [PATCH 44/51] fix(gun): connect to hosts that only have IPv6 addresses Gun resolves host names over IPv4 only unless :inet6 is in tcp_opts. When the A lookup finds nothing, retry once with :inet6. Gun now runs under its supervisor, so a failed connect no longer kills the pool worker through the link; the worker still links after a successful open. --- changelog.d/gun-ipv6-only-hosts.fix | 1 + lib/pleroma/gun/conn.ex | 41 +++++++++++++++++-- test/pleroma/gun/conn_test.exs | 62 +++++++++++++++++++++++++++++ 3 files changed, 101 insertions(+), 3 deletions(-) create mode 100644 changelog.d/gun-ipv6-only-hosts.fix diff --git a/changelog.d/gun-ipv6-only-hosts.fix b/changelog.d/gun-ipv6-only-hosts.fix new file mode 100644 index 000000000..69cdc56e3 --- /dev/null +++ b/changelog.d/gun-ipv6-only-hosts.fix @@ -0,0 +1 @@ +Gun adapter: connect to hosts that only have IPv6 (AAAA) addresses diff --git a/lib/pleroma/gun/conn.ex b/lib/pleroma/gun/conn.ex index 85a2fb0e8..3291b9048 100644 --- a/lib/pleroma/gun/conn.ex +++ b/lib/pleroma/gun/conn.ex @@ -14,7 +14,9 @@ defmodule Pleroma.Gun.Conn do opts |> Enum.into(%{}) |> Map.put_new(:connect_timeout, pool_opts[:connect_timeout] || 5_000) - |> Map.put_new(:supervise, false) + # Under gun's supervisor, a failed connect does not kill the caller + # through the link. Gun still stops when its owner goes down. + |> Map.put_new(:supervise, true) |> maybe_add_tls_opts(uri) do_open(uri, opts) @@ -117,8 +119,7 @@ defmodule Pleroma.Gun.Conn do host = Pleroma.HTTP.AdapterHelper.parse_host(host) opts = Map.put(opts, :transport, transport(uri.scheme)) - with {:ok, conn} <- Gun.open(host, port, opts), - {:ok, protocol} <- await_up(conn, opts[:connect_timeout]) do + with {:ok, conn, protocol} <- open_direct(host, port, opts) do {:ok, conn, protocol, nil} else error -> @@ -130,6 +131,40 @@ defmodule Pleroma.Gun.Conn do end end + defp open_direct(host, port, opts) do + result = + with {:ok, conn} <- Gun.open(host, port, opts), + {:ok, protocol} <- await_up(conn, opts[:connect_timeout]) do + {:ok, conn, protocol} + end + + case inet6_fallback(result, host, opts) do + {:ok, inet6_opts} -> open_direct(host, port, inet6_opts) + :error -> result + end + end + + # Gun's default when :tcp_opts is not set. + @default_tcp_opts [send_timeout: 15_000, send_timeout_close: true] + + @doc """ + Gun resolves hostnames with `inet_tcp` (A records only) unless `:inet6` is + in `:tcp_opts`, for both the :tcp and :tls transports. When the A lookup + finds nothing, returns the options for one more attempt over IPv6. + """ + @spec inet6_fallback(term(), Pleroma.HTTP.AdapterHelper.host(), map()) :: {:ok, map()} | :error + def inet6_fallback({:error, {:down, {:shutdown, :nxdomain}}}, host, opts) when is_list(host) do + tcp_opts = Map.get(opts, :tcp_opts, @default_tcp_opts) + + if :inet6 in tcp_opts do + :error + else + {:ok, Map.put(opts, :tcp_opts, [:inet6 | tcp_opts])} + end + end + + def inet6_fallback(_result, _host, _opts), do: :error + defp destination_opts(%URI{host: host, port: port}) do host = Pleroma.HTTP.AdapterHelper.parse_host(host) %{host: host, port: port} diff --git a/test/pleroma/gun/conn_test.exs b/test/pleroma/gun/conn_test.exs index a38740a42..db3ce6275 100644 --- a/test/pleroma/gun/conn_test.exs +++ b/test/pleroma/gun/conn_test.exs @@ -42,6 +42,7 @@ defmodule Pleroma.Gun.ConnTest do expect(Pleroma.GunMock, :open, fn ~c"proxy.example", 8080, opts -> assert opts.transport == :tcp + assert opts.supervise refute Map.has_key?(opts, :tls_opts) {:ok, conn} end) @@ -172,6 +173,67 @@ defmodule Pleroma.Gun.ConnTest do ) end + describe "IPv6-only hosts" do + @nxdomain {:error, {:down, {:shutdown, :nxdomain}}} + + test "retries once over IPv6 when the IPv4 lookup finds no address" do + ipv4_conn = spawn_link(fn -> Process.sleep(:infinity) end) + ipv6_conn = spawn_link(fn -> Process.sleep(:infinity) end) + + expect(Pleroma.GunMock, :open, 2, fn + ~c"v6only.example", 443, %{tcp_opts: tcp_opts} = opts -> + assert :inet6 in tcp_opts + assert opts.transport == :tls + assert opts.tls_opts[:verify] == :verify_peer + {:ok, ipv6_conn} + + ~c"v6only.example", 443, opts -> + refute Map.has_key?(opts, :tcp_opts) + # A linked Gun process would take the caller down with the lookup failure. + assert opts.supervise + {:ok, ipv4_conn} + end) + + stub(Pleroma.GunMock, :await_up, fn + ^ipv4_conn, _ -> @nxdomain + ^ipv6_conn, _ -> {:ok, :http2} + end) + + assert {:ok, ^ipv6_conn, :http2, nil} = + Conn.open(URI.parse("https://v6only.example/inbox"), []) + + stop_conn(ipv4_conn) + stop_conn(ipv6_conn) + end + + test "does not retry IP literals or other errors" do + expect(Pleroma.GunMock, :open, 2, fn _, _, _ -> + {:ok, spawn(fn -> Process.sleep(:infinity) end)} + end) + + stub(Pleroma.GunMock, :await_up, fn _, _ -> @nxdomain end) + assert @nxdomain = Conn.open(URI.parse("https://192.0.2.1/inbox"), []) + + stub(Pleroma.GunMock, :await_up, fn _, _ -> + {:error, {:down, {:shutdown, :econnrefused}}} + end) + + assert {:error, _} = Conn.open(URI.parse("https://v6only.example/inbox"), []) + end + + test "inet6_fallback/3 keeps configured TCP options and skips hosts already on IPv6" do + assert {:ok, %{tcp_opts: [:inet6, nodelay: true]}} = + Conn.inet6_fallback(@nxdomain, ~c"v6only.example", %{tcp_opts: [nodelay: true]}) + + assert {:ok, %{tcp_opts: [:inet6, send_timeout: 15_000, send_timeout_close: true]}} = + Conn.inet6_fallback(@nxdomain, ~c"v6only.example", %{}) + + assert :error = Conn.inet6_fallback(@nxdomain, ~c"v6only.example", %{tcp_opts: [:inet6]}) + assert :error = Conn.inet6_fallback(@nxdomain, {192, 0, 2, 1}, %{}) + assert :error = Conn.inet6_fallback({:error, :timeout}, ~c"v6only.example", %{}) + end + end + defp stop_conn(conn) do Process.unlink(conn) Process.exit(conn, :kill) From 88a9562a9b09bf7612b9e0c29eacf025e80a5a3e Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sat, 31 Oct 2026 13:43:59 +0400 Subject: [PATCH 45/51] fix(config): skip ConfigDB settings for modules that no longer exist Ported from Akkoma af7c3fab9; the rescue is in the Ecto type load so admin input is still validated. Co-authored-by: FloatingGhost --- changelog.d/config-db-unknown-module.fix | 1 + lib/pleroma/config/transfer_task.ex | 1 + lib/pleroma/ecto_type/config/atom.ex | 11 +++++++++ test/pleroma/config/transfer_task_test.exs | 23 +++++++++++++++++++ test/pleroma/config_db_test.exs | 16 +++++++++++++ .../controllers/config_controller_test.exs | 22 ++++++++++++++++++ 6 files changed, 74 insertions(+) create mode 100644 changelog.d/config-db-unknown-module.fix diff --git a/changelog.d/config-db-unknown-module.fix b/changelog.d/config-db-unknown-module.fix new file mode 100644 index 000000000..05a6a2b2f --- /dev/null +++ b/changelog.d/config-db-unknown-module.fix @@ -0,0 +1 @@ +Do not crash at boot or in the AdminAPI config listing when a ConfigDB setting refers to a module that no longer exists diff --git a/lib/pleroma/config/transfer_task.ex b/lib/pleroma/config/transfer_task.ex index 48cb158ca..731d05e8b 100644 --- a/lib/pleroma/config/transfer_task.ex +++ b/lib/pleroma/config/transfer_task.ex @@ -48,6 +48,7 @@ defmodule Pleroma.Config.TransferTask do settings = (Repo.all(ConfigDB) ++ deleted_settings) |> Enum.reject(&ConfigDB.static?/1) + |> Enum.reject(&(&1.group == :invalid_atom or &1.key == :invalid_atom)) |> Enum.map(&merge_with_default/1) started_applications = Application.started_applications() diff --git a/lib/pleroma/ecto_type/config/atom.ex b/lib/pleroma/ecto_type/config/atom.ex index c44d655e0..dad640f3d 100644 --- a/lib/pleroma/ecto_type/config/atom.ex +++ b/lib/pleroma/ecto_type/config/atom.ex @@ -5,6 +5,8 @@ defmodule Pleroma.EctoType.Config.Atom do use Ecto.Type + require Logger + def type, do: :atom def cast(key) when is_atom(key) do @@ -17,8 +19,17 @@ defmodule Pleroma.EctoType.Config.Atom do def cast(_), do: :error + # A key or group naming a module that no longer exists (e.g. a removed uploader) + # must not crash loading def load(key) do {:ok, Pleroma.ConfigDB.string_to_elixir_types(key)} + rescue + ArgumentError -> + Logger.warning( + "ConfigDB: unknown module #{key} in a setting key or group, the setting is not applied" + ) + + {:ok, :invalid_atom} end def dump(key) when is_atom(key), do: {:ok, inspect(key)} diff --git a/test/pleroma/config/transfer_task_test.exs b/test/pleroma/config/transfer_task_test.exs index ea5378ffd..2f39712e3 100644 --- a/test/pleroma/config/transfer_task_test.exs +++ b/test/pleroma/config/transfer_task_test.exs @@ -96,6 +96,29 @@ defmodule Pleroma.Config.TransferTaskTest do assert Application.get_env(:pleroma, Pleroma.Search.ParadeDB.Repo)[:pool_size] == 2 end + test "skips settings with an unknown module key" do + now = NaiveDateTime.utc_now() |> NaiveDateTime.truncate(:second) + + Repo.insert_all("config", [ + %{ + group: ":pleroma", + key: "Pleroma.Nowhere.Module", + value: :erlang.term_to_binary(enabled: true), + inserted_at: now, + updated_at: now + } + ]) + + insert(:config, key: :test_key, value: [live: 2]) + on_exit(fn -> Application.delete_env(:pleroma, :test_key) end) + + log = capture_log(fn -> assert :ok = TransferTask.load_and_update_env([], false) end) + + assert log =~ "Pleroma.Nowhere.Module" + assert Application.get_env(:pleroma, :test_key) == [live: 2] + refute Application.get_env(:pleroma, :invalid_atom) + end + describe "pleroma restart" do setup do on_exit(fn -> diff --git a/test/pleroma/config_db_test.exs b/test/pleroma/config_db_test.exs index ae17a9d34..43aca06c2 100644 --- a/test/pleroma/config_db_test.exs +++ b/test/pleroma/config_db_test.exs @@ -50,6 +50,22 @@ defmodule Pleroma.ConfigDBTest do assert config[:pleroma][:instance] == [name: "Pleroma"] end + test "loads a row with an unknown module key as :invalid_atom" do + now = NaiveDateTime.utc_now() |> NaiveDateTime.truncate(:second) + + Repo.insert_all("config", [ + %{ + group: ":pleroma", + key: "Pleroma.Nowhere.Module", + value: :erlang.term_to_binary(enabled: true), + inserted_at: now, + updated_at: now + } + ]) + + assert [%ConfigDB{group: :pleroma, key: :invalid_atom}] = Repo.all(ConfigDB) + end + describe "update_or_create/1" do test "common" do config = insert(:config) diff --git a/test/pleroma/web/admin_api/controllers/config_controller_test.exs b/test/pleroma/web/admin_api/controllers/config_controller_test.exs index e4c8d866f..a035d558f 100644 --- a/test/pleroma/web/admin_api/controllers/config_controller_test.exs +++ b/test/pleroma/web/admin_api/controllers/config_controller_test.exs @@ -60,6 +60,28 @@ defmodule Pleroma.Web.AdminAPI.ConfigControllerTest do assert key2 == inspect(config2.key) end + test "does not crash on a setting with an unknown module key", %{conn: conn} do + now = NaiveDateTime.utc_now() |> NaiveDateTime.truncate(:second) + + Pleroma.Repo.insert_all("config", [ + %{ + group: ":pleroma", + key: "Pleroma.Nowhere.Module", + value: :erlang.term_to_binary(enabled: true), + inserted_at: now, + updated_at: now + } + ]) + + capture_log(fn -> + for path <- ["/api/pleroma/admin/config", "/api/pleroma/admin/config?only_db=true"] do + conn + |> get(path) + |> json_response_and_validate_schema(200) + end + end) + end + test "db is added to settings that are in db", %{conn: conn} do _config = insert(:config, key: ":instance", value: [name: "Some name"]) From d601a94c693161bb8c371f4187a8f4a96f093bb5 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sat, 31 Oct 2026 13:44:00 +0400 Subject: [PATCH 46/51] fix(api_spec): drop every unexpected query param in permissive mode Ported from Akkoma 7e6efb235. Co-authored-by: Oneric --- changelog.d/cast-and-validate-many-params.fix | 1 + lib/pleroma/web/api_spec/cast_and_validate.ex | 64 ++++++++++++------- .../web/api_spec/cast_and_validate_test.exs | 24 +++++++ 3 files changed, 66 insertions(+), 23 deletions(-) create mode 100644 changelog.d/cast-and-validate-many-params.fix create mode 100644 test/pleroma/web/api_spec/cast_and_validate_test.exs diff --git a/changelog.d/cast-and-validate-many-params.fix b/changelog.d/cast-and-validate-many-params.fix new file mode 100644 index 000000000..c44396618 --- /dev/null +++ b/changelog.d/cast-and-validate-many-params.fix @@ -0,0 +1 @@ +Drop all unexpected query params, not just the first one, when `Pleroma.Web.ApiSpec.CastAndValidate` `strict` is off diff --git a/lib/pleroma/web/api_spec/cast_and_validate.ex b/lib/pleroma/web/api_spec/cast_and_validate.ex index 92f1c5376..41e69727e 100644 --- a/lib/pleroma/web/api_spec/cast_and_validate.ex +++ b/lib/pleroma/web/api_spec/cast_and_validate.ex @@ -20,6 +20,9 @@ defmodule Pleroma.Web.ApiSpec.CastAndValidate do require Logger + # How often to drop offending query params and retry in permissive mode + @max_retries 20 + @impl Plug def init(opts) do opts @@ -114,34 +117,49 @@ defmodule Pleroma.Web.ApiSpec.CastAndValidate do false = _strict, cast_opts ) do - case OpenApiSpex.cast_and_validate(spec, operation, conn, content_type, cast_opts) do - {:ok, conn} -> - {:ok, conn} + permissive_cast_and_validate(spec, operation, conn, content_type, cast_opts, @max_retries) + end - # Remove unexpected query params and cast/validate again - {:error, errors} -> - query_params = - Enum.reduce(errors, conn.query_params, fn - %{reason: :unexpected_field, name: name, path: [name]}, params -> - Map.delete(params, name) + # OpenApiSpex reports one offending param at a time, so drop it and cast/validate again + defp permissive_cast_and_validate( + spec, + operation, + %Conn{} = conn, + content_type, + cast_opts, + retries + ) do + with {:error, errors} = error <- + OpenApiSpex.cast_and_validate(spec, operation, conn, content_type, cast_opts) do + query_params = drop_invalid_query_params(errors, conn.query_params) - # Filter out empty params - %{reason: :invalid_type, path: [name_atom], value: ""}, params -> - Map.delete(params, to_string(name_atom)) - - %{reason: :invalid_enum, name: nil, path: path, value: value}, params -> - path = path |> Enum.reverse() |> tl() |> Enum.reverse() |> list_items_to_string() - update_in(params, path, &List.delete(&1, value)) - - _, params -> - params - end) - - conn = %Conn{conn | query_params: query_params} - OpenApiSpex.cast_and_validate(spec, operation, conn, content_type, cast_opts) + if retries > 0 and query_params != conn.query_params do + conn = %{conn | query_params: query_params} + permissive_cast_and_validate(spec, operation, conn, content_type, cast_opts, retries - 1) + else + error + end end end + defp drop_invalid_query_params(errors, query_params) do + Enum.reduce(errors, query_params, fn + %{reason: :unexpected_field, name: name, path: [name]}, params -> + Map.delete(params, name) + + # Filter out empty params + %{reason: :invalid_type, path: [name_atom], value: ""}, params -> + Map.delete(params, to_string(name_atom)) + + %{reason: :invalid_enum, name: nil, path: path, value: value}, params -> + path = path |> Enum.reverse() |> tl() |> Enum.reverse() |> list_items_to_string() + update_in(params, path, &List.delete(&1, value)) + + _, params -> + params + end) + end + defp list_items_to_string(list) do Enum.map(list, fn i when is_atom(i) -> to_string(i) diff --git a/test/pleroma/web/api_spec/cast_and_validate_test.exs b/test/pleroma/web/api_spec/cast_and_validate_test.exs new file mode 100644 index 000000000..b7ee673e1 --- /dev/null +++ b/test/pleroma/web/api_spec/cast_and_validate_test.exs @@ -0,0 +1,24 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ApiSpec.CastAndValidateTest do + use Pleroma.Web.ConnCase, async: false + + describe "permissive mode" do + setup do: clear_config([Pleroma.Web.ApiSpec.CastAndValidate, :strict], false) + + test "drops several unexpected query params", %{conn: conn} do + assert conn + |> get("/api/v1/timelines/public?foo=1&bar=2&baz=3&local=true") + |> json_response_and_validate_schema(200) + end + + test "still rejects an invalid known param", %{conn: conn} do + assert %{"error" => _} = + conn + |> get("/api/v1/timelines/public?foo=1&bar=2&limit=lots") + |> json_response(400) + end + end +end From 13663cba71d4e643b6c080caabc8e590c19001e0 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sat, 31 Oct 2026 15:37:42 +0400 Subject: [PATCH 47/51] fix(mastodon_api): use the Mastodon date format for marker updated_at Ported from Akkoma 720785b4c. Co-authored-by: Oneric --- changelog.d/marker-updated-at-format.fix | 1 + lib/pleroma/web/mastodon_api/views/marker_view.ex | 4 +++- .../controllers/marker_controller_test.exs | 5 +++-- .../web/mastodon_api/views/marker_view_test.exs | 14 ++++++++++++-- 4 files changed, 19 insertions(+), 5 deletions(-) create mode 100644 changelog.d/marker-updated-at-format.fix diff --git a/changelog.d/marker-updated-at-format.fix b/changelog.d/marker-updated-at-format.fix new file mode 100644 index 000000000..2ff681525 --- /dev/null +++ b/changelog.d/marker-updated-at-format.fix @@ -0,0 +1 @@ +Mastodon API: Use the Mastodon date format for marker updated_at diff --git a/lib/pleroma/web/mastodon_api/views/marker_view.ex b/lib/pleroma/web/mastodon_api/views/marker_view.ex index 944769b1a..fd067f713 100644 --- a/lib/pleroma/web/mastodon_api/views/marker_view.ex +++ b/lib/pleroma/web/mastodon_api/views/marker_view.ex @@ -5,13 +5,15 @@ defmodule Pleroma.Web.MastodonAPI.MarkerView do use Pleroma.Web, :view + alias Pleroma.Web.CommonAPI.Utils + def render("markers.json", %{markers: markers}) do Map.new(markers, fn m -> {m.timeline, %{ last_read_id: m.last_read_id, version: m.lock_version, - updated_at: NaiveDateTime.to_iso8601(m.updated_at), + updated_at: Utils.to_masto_date(m.updated_at), pleroma: %{ unread_count: m.unread_count } diff --git a/test/pleroma/web/mastodon_api/controllers/marker_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/marker_controller_test.exs index 4050528ff..ff85031f0 100644 --- a/test/pleroma/web/mastodon_api/controllers/marker_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/marker_controller_test.exs @@ -8,6 +8,7 @@ defmodule Pleroma.Web.MastodonAPI.MarkerControllerTest do alias Pleroma.Notification alias Pleroma.Repo alias Pleroma.Web.CommonAPI + alias Pleroma.Web.CommonAPI.Utils import Pleroma.Factory @@ -33,7 +34,7 @@ defmodule Pleroma.Web.MastodonAPI.MarkerControllerTest do assert response == %{ "notifications" => %{ "last_read_id" => "69420", - "updated_at" => NaiveDateTime.to_iso8601(marker.updated_at), + "updated_at" => Utils.to_masto_date(marker.updated_at), "version" => 0, "pleroma" => %{"unread_count" => 7} } @@ -107,7 +108,7 @@ defmodule Pleroma.Web.MastodonAPI.MarkerControllerTest do assert response == %{ "notifications" => %{ "last_read_id" => "69888", - "updated_at" => NaiveDateTime.to_iso8601(marker.updated_at), + "updated_at" => Utils.to_masto_date(marker.updated_at), "version" => 0, "pleroma" => %{"unread_count" => 0} } diff --git a/test/pleroma/web/mastodon_api/views/marker_view_test.exs b/test/pleroma/web/mastodon_api/views/marker_view_test.exs index 43f8c4302..4f4eeca82 100644 --- a/test/pleroma/web/mastodon_api/views/marker_view_test.exs +++ b/test/pleroma/web/mastodon_api/views/marker_view_test.exs @@ -4,6 +4,7 @@ defmodule Pleroma.Web.MastodonAPI.MarkerViewTest do use Pleroma.DataCase, async: true + alias Pleroma.Web.CommonAPI.Utils alias Pleroma.Web.MastodonAPI.MarkerView import Pleroma.Factory @@ -14,16 +15,25 @@ defmodule Pleroma.Web.MastodonAPI.MarkerViewTest do assert MarkerView.render("markers.json", %{markers: [marker1, marker2]}) == %{ "home" => %{ last_read_id: "42", - updated_at: NaiveDateTime.to_iso8601(marker2.updated_at), + updated_at: Utils.to_masto_date(marker2.updated_at), version: 0, pleroma: %{unread_count: 0} }, "notifications" => %{ last_read_id: "17", - updated_at: NaiveDateTime.to_iso8601(marker1.updated_at), + updated_at: Utils.to_masto_date(marker1.updated_at), version: 0, pleroma: %{unread_count: 5} } } end + + test "renders updated_at in the Mastodon date format" do + marker = insert(:marker, timeline: "home") + + %{"home" => %{updated_at: updated_at}} = + MarkerView.render("markers.json", %{markers: [marker]}) + + assert updated_at =~ ~r/\.\d{3}Z$/ + end end From 542f8b2aef34a1a3cade2d7e3e52ef0c80e3889c Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sat, 31 Oct 2026 15:37:42 +0400 Subject: [PATCH 48/51] fix(media_proxy): do not proxy data: URLs Ported from Akkoma 7632765b4. Co-authored-by: nopjmp --- changelog.d/media-proxy-data-urls.fix | 1 + lib/pleroma/web/media_proxy.ex | 6 ++++-- test/pleroma/web/media_proxy_test.exs | 12 ++++++++++++ 3 files changed, 17 insertions(+), 2 deletions(-) create mode 100644 changelog.d/media-proxy-data-urls.fix diff --git a/changelog.d/media-proxy-data-urls.fix b/changelog.d/media-proxy-data-urls.fix new file mode 100644 index 000000000..a83b8ef29 --- /dev/null +++ b/changelog.d/media-proxy-data-urls.fix @@ -0,0 +1 @@ +MediaProxy: Do not proxy data: URLs diff --git a/lib/pleroma/web/media_proxy.ex b/lib/pleroma/web/media_proxy.ex index 82a070767..a318ab10f 100644 --- a/lib/pleroma/web/media_proxy.ex +++ b/lib/pleroma/web/media_proxy.ex @@ -45,7 +45,7 @@ defmodule Pleroma.Web.MediaProxy do def url("/" <> _ = url), do: url def url(url) do - if enabled?() and url_proxiable?(url) do + if enabled?() and not data_url?(url) and url_proxiable?(url) do encode_url(url) else url @@ -61,7 +61,7 @@ defmodule Pleroma.Web.MediaProxy do def preview_url("//" <> _ = url, pparams), do: preview_url("https:" <> url, pparams) def preview_url(url, preview_params) do - if preview_enabled?() do + if preview_enabled?() and not data_url?(url) do encode_preview_url(url, preview_params) else url(url) @@ -74,6 +74,8 @@ defmodule Pleroma.Web.MediaProxy do # non-local non-whitelisted URLs through it and be sure that body size constraint is preserved. def preview_enabled?, do: enabled?() and !!Config.get([:media_preview_proxy, :enabled]) + defp data_url?(url), do: url |> String.slice(0, 5) |> String.downcase() == "data:" + def local?(url), do: String.starts_with?(url, Endpoint.url()) def whitelisted?(url) do diff --git a/test/pleroma/web/media_proxy_test.exs b/test/pleroma/web/media_proxy_test.exs index ab707a061..433a0fe95 100644 --- a/test/pleroma/web/media_proxy_test.exs +++ b/test/pleroma/web/media_proxy_test.exs @@ -45,6 +45,18 @@ defmodule Pleroma.Web.MediaProxyTest do assert MediaProxy.url(local_root) == local_root end + test "ignores data url" do + assert MediaProxy.url("data:image/png;base64,AAAA") == "data:image/png;base64,AAAA" + assert MediaProxy.url("DATA:image/png;base64,AAAA") == "DATA:image/png;base64,AAAA" + end + + test "preview_url ignores data url" do + clear_config([:media_preview_proxy, :enabled], true) + + assert MediaProxy.preview_url("data:image/png;base64,AAAA") == + "data:image/png;base64,AAAA" + end + test "encodes and decodes URL" do url = "https://pleroma.soykaf.com/static/logo.png" encoded = MediaProxy.url(url) From 9b7c59bc216749a1546fcc12524331af20e07bab Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sun, 1 Nov 2026 11:10:49 +0400 Subject: [PATCH 49/51] build: update ssl_verify_fun and phoenix_html for advisories ssl_verify_fun 1.1.9 fixes partial-label wildcard matching (EEF-CVE-2026-117666); Hackney uses its verify_fun for every HTTPS request. phoenix_html 3.3.5 escapes attribute names (EEF-CVE-2026-117645). --- changelog.d/advisory-bumps.security | 1 + mix.lock | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 changelog.d/advisory-bumps.security diff --git a/changelog.d/advisory-bumps.security b/changelog.d/advisory-bumps.security new file mode 100644 index 000000000..baf9e1ccb --- /dev/null +++ b/changelog.d/advisory-bumps.security @@ -0,0 +1 @@ +Update ssl_verify_fun to 1.1.9 (EEF-CVE-2026-117666) and phoenix_html to 3.3.5 (EEF-CVE-2026-117645) diff --git a/mix.lock b/mix.lock index 9e60402e5..e6c9969cf 100644 --- a/mix.lock +++ b/mix.lock @@ -106,7 +106,7 @@ "pbkdf2_elixir": {:hex, :pbkdf2_elixir, "1.2.1", "9cbe354b58121075bd20eb83076900a3832324b7dd171a6895fab57b6bb2752c", [:mix], [{:comeonin, "~> 5.3", [hex: :comeonin, repo: "hexpm", optional: false]}], "hexpm", "d3b40a4a4630f0b442f19eca891fcfeeee4c40871936fed2f68e1c4faa30481f"}, "phoenix": {:hex, :phoenix, "1.8.11", "9bb8f0c4e9f0b9eaf7f34cf8cb0bd25bd8599763a5587d64918a9ebb04027823", [:mix], [{:bandit, "~> 1.0", [hex: :bandit, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.7", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:plug_crypto, "~> 2.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:websock_adapter, "~> 0.5", [hex: :websock_adapter, repo: "hexpm", optional: false]}], "hexpm", "44f028f4129e5a29487e868f84903373e3d032da151ad0c789c3849f464e7351"}, "phoenix_ecto": {:hex, :phoenix_ecto, "4.6.5", "c4ef322acd15a574a8b1a08eff0ee0a85e73096b53ce1403b6563709f15e1cea", [:mix], [{:ecto, "~> 3.5", [hex: :ecto, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.14.2 or ~> 3.0 or ~> 4.1", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:plug, "~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.16 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}], "hexpm", "26ec3208eef407f31b748cadd044045c6fd485fbff168e35963d2f9dfff28d4b"}, - "phoenix_html": {:hex, :phoenix_html, "3.3.4", "42a09fc443bbc1da37e372a5c8e6755d046f22b9b11343bf885067357da21cb3", [:mix], [{:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "0249d3abec3714aff3415e7ee3d9786cb325be3151e6c4b3021502c585bf53fb"}, + "phoenix_html": {:hex, :phoenix_html, "3.3.5", "a2ce9bfc938bd7e23f10de438d19550074d7fdba3ea8d54095142f52da288fec", [:mix], [{:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "9966451c70e3080124b5a6b9b30f8a7927983528d890fd457f9eee4074dfa167"}, "phoenix_live_dashboard": {:hex, :phoenix_live_dashboard, "0.8.7", "405880012cb4b706f26dd1c6349125bfc903fb9e44d1ea668adaf4e04d4884b7", [:mix], [{:ecto, "~> 3.6.2 or ~> 3.7", [hex: :ecto, repo: "hexpm", optional: true]}, {:ecto_mysql_extras, "~> 0.5", [hex: :ecto_mysql_extras, repo: "hexpm", optional: true]}, {:ecto_psql_extras, "~> 0.7", [hex: :ecto_psql_extras, repo: "hexpm", optional: true]}, {:ecto_sqlite3_extras, "~> 1.1.7 or ~> 1.2.0", [hex: :ecto_sqlite3_extras, repo: "hexpm", optional: true]}, {:mime, "~> 1.6 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:phoenix_live_view, "~> 0.19 or ~> 1.0", [hex: :phoenix_live_view, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.6 or ~> 1.0", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "3a8625cab39ec261d48a13b7468dc619c0ede099601b084e343968309bd4d7d7"}, "phoenix_live_reload": {:hex, :phoenix_live_reload, "1.3.3", "3a53772a6118d5679bf50fc1670505a290e32a1d195df9e069d8c53ab040c054", [:mix], [{:file_system, "~> 0.2.1 or ~> 0.3", [hex: :file_system, repo: "hexpm", optional: false]}, {:phoenix, "~> 1.4", [hex: :phoenix, repo: "hexpm", optional: false]}], "hexpm", "766796676e5f558dbae5d1bdb066849673e956005e3730dfd5affd7a6da4abac"}, "phoenix_live_view": {:hex, :phoenix_live_view, "1.2.9", "d35ddac2fab4480e6bdbf712ff104fd9e969a2bbe81b578ee80f066b371f56db", [:mix], [{:igniter, ">= 0.6.16 and < 1.0.0-0", [hex: :igniter, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:lazy_html, "~> 0.1.0", [hex: :lazy_html, repo: "hexpm", optional: true]}, {:phoenix, "~> 1.6.15 or ~> 1.7.0 or ~> 1.8.0", [hex: :phoenix, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 3.3 or ~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.15", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.2 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "2f9528c3d7046edabbb30a91710ca33988f8d8bc20a964a1fc48b32134572afa"}, @@ -136,7 +136,7 @@ "recon": {:hex, :recon, "2.5.6", "9052588e83bfedfd9b72e1034532aee2a5369d9d9343b61aeb7fbce761010741", [:mix, :rebar3], [], "hexpm", "96c6799792d735cc0f0fd0f86267e9d351e63339cbe03df9d162010cefc26bb0"}, "remote_ip": {:hex, :remote_ip, "1.2.0", "fb078e12a44414f4cef5a75963c33008fe169b806572ccd17257c208a7bc760f", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "2ff91de19c48149ce19ed230a81d377186e4412552a597d6a5137373e5877cb7"}, "sleeplocks": {:hex, :sleeplocks, "1.1.3", "96a86460cc33b435c7310dbd27ec82ca2c1f24ae38e34f8edde97f756503441a", [:rebar3], [], "hexpm", "d3b3958552e6eb16f463921e70ae7c767519ef8f5be46d7696cc1ed649421321"}, - "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.7", "354c321cf377240c7b8716899e182ce4890c5938111a1296add3ec74cf1715df", [:make, :mix, :rebar3], [], "hexpm", "fe4c190e8f37401d30167c8c405eda19469f34577987c76dde613e838bbc67f8"}, + "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.9", "4ff292dc16e728fde1235cf5fa6bcd3ca01864004526f71a4a70935509d1a7f1", [:make, :mix, :rebar3], [], "hexpm", "19f24ad715ef6255d530b2f0db8b2d2191ae435b6527234e943f523e84ed0895"}, "statistex": {:hex, :statistex, "1.1.0", "7fec1eb2f580a0d2c1a05ed27396a084ab064a40cfc84246dbfb0c72a5c761e5", [:mix], [], "hexpm", "f5950ea26ad43246ba2cce54324ac394a4e7408fdcf98b8e230f503a0cba9cf5"}, "sweet_xml": {:hex, :sweet_xml, "0.7.5", "803a563113981aaac202a1dbd39771562d0ad31004ddbfc9b5090bdcd5605277", [:mix], [], "hexpm", "193b28a9b12891cae351d81a0cead165ffe67df1b73fe5866d10629f4faefb12"}, "swoosh": {:hex, :swoosh, "1.28.0", "30d9f3519a150128e90d97f9878a8a5cd8325ed03d618d1c626201496fc20baf", [:mix], [{:bandit, ">= 1.0.0", [hex: :bandit, repo: "hexpm", optional: true]}, {:cowboy, "~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:ex_aws, "~> 2.1", [hex: :ex_aws, repo: "hexpm", optional: true]}, {:finch, "~> 0.6", [hex: :finch, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13 or ~> 1.0", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, ">= 1.9.0 and < 5.0.0", [hex: :hackney, repo: "hexpm", optional: true]}, {:idna, ">= 6.0.0 and < 8.0.0", [hex: :idna, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mua, "~> 0.2.3", [hex: :mua, repo: "hexpm", optional: true]}, {:multipart, "~> 0.4", [hex: :multipart, repo: "hexpm", optional: true]}, {:plug, "~> 1.9", [hex: :plug, repo: "hexpm", optional: true]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:req, "~> 0.5.10 or ~> 0.6 or ~> 1.0", [hex: :req, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.2 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "bb5c0b7c988beb53786254a61580597fe1017a652039061ee2b4c28b5097b10e"}, From 4070b7be21a28fa2f994483a5111922a9b363f73 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sun, 1 Nov 2026 17:03:11 +0400 Subject: [PATCH 50/51] fix(http): back off from the host that answered, and read Unix resets After a redirect the 429/503 came from the final host, but the backoff was recorded for the requested one. X-RateLimit-Reset is also accepted as a Unix timestamp or as seconds. --- lib/pleroma/http/backoff.ex | 35 ++++++++++++++++++++++++++++-- test/pleroma/http/backoff_test.exs | 32 +++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/http/backoff.ex b/lib/pleroma/http/backoff.ex index 44b66fbdd..5e432b285 100644 --- a/lib/pleroma/http/backoff.ex +++ b/lib/pleroma/http/backoff.ex @@ -39,8 +39,9 @@ defmodule Pleroma.HTTP.Backoff do defp downcase(nil), do: nil defp downcase(host), do: String.downcase(host) - defp check_backoff({:ok, %{status: status, headers: headers}}, host) + defp check_backoff({:ok, %{status: status, headers: headers} = env}, host) when status in [429, 503] do + host = response_host(env, host) seconds = backoff_seconds(headers, DateTime.utc_now()) Logger.warning("#{host} answered #{status}, backing off for #{seconds} seconds") @cachex.put(@cache, host, true, ttl: :timer.seconds(seconds)) @@ -49,6 +50,13 @@ defmodule Pleroma.HTTP.Backoff do defp check_backoff(response, _host), do: response + # After a redirect, the 429/503 came from the last host, not the requested one. + defp response_host(%{url: url}, host) when is_binary(url) do + URI.parse(url).host |> downcase() || host + end + + defp response_host(_env, host), do: host + @doc """ Turns an Oban job's `{:error, :ratelimit}` result into a snooze for the default backoff, so the job's attempts aren't all spent while the host is @@ -75,7 +83,7 @@ defmodule Pleroma.HTTP.Backoff do def backoff_seconds(headers, now) do [ parse_retry_after(header(headers, "retry-after"), now), - parse_timestamp(header(headers, "x-ratelimit-reset"), now) + parse_ratelimit_reset(header(headers, "x-ratelimit-reset"), now) ] |> Enum.find(@default_seconds, &is_integer/1) |> min(@max_seconds) @@ -102,6 +110,29 @@ defmodule Pleroma.HTTP.Backoff do end end + # Servers send X-RateLimit-Reset as a timestamp (ISO 8601 like Mastodon, or + # Unix seconds) or as seconds until the reset. + @unix_timestamp_min 1_000_000_000 + + defp parse_ratelimit_reset(nil, _now), do: nil + + defp parse_ratelimit_reset(value, now) do + case Integer.parse(value) do + {timestamp, ""} when timestamp >= @unix_timestamp_min -> + timestamp - DateTime.to_unix(now) + + {seconds, ""} -> + seconds + + _ -> + parse_timestamp(value, now) + end + |> positive() + end + + defp positive(seconds) when is_integer(seconds) and seconds > 0, do: seconds + defp positive(_), do: nil + defp parse_timestamp(nil, _now), do: nil defp parse_timestamp(value, now) do diff --git a/test/pleroma/http/backoff_test.exs b/test/pleroma/http/backoff_test.exs index f893eef2e..b650391dd 100644 --- a/test/pleroma/http/backoff_test.exs +++ b/test/pleroma/http/backoff_test.exs @@ -83,6 +83,23 @@ defmodule Pleroma.HTTP.BackoffTest do mock_response(200) assert {:ok, _} = Backoff.get("https://other.example.com/users/bob") end + + test "backs off from the host that answered after a redirect" do + test_pid = self() + + Tesla.Mock.mock(fn + %{url: "https://redirector.example.com/users/alice"} = env -> + %{env | status: 302, headers: [{"location", @url}], body: ""} + + %{url: @url} = env -> + send(test_pid, :request) + %{env | status: 429, headers: [], body: ""} + end) + + assert {:error, :ratelimit} = Backoff.get("https://redirector.example.com/users/alice") + assert {:ok, true} = Cachex.get(@cache, @host) + assert {:ok, nil} = Cachex.get(@cache, "redirector.example.com") + end end describe "backoff headers" do @@ -121,6 +138,21 @@ defmodule Pleroma.HTTP.BackoffTest do assert_ttl_about(600) end + test "parses X-RateLimit-Reset as a Unix timestamp" do + reset = DateTime.utc_now() |> DateTime.add(600) |> DateTime.to_unix() |> to_string() + mock_response(429, [{"x-ratelimit-reset", reset}]) + + assert {:error, :ratelimit} = Backoff.get(@url) + assert_ttl_about(600) + end + + test "parses X-RateLimit-Reset in seconds" do + mock_response(429, [{"x-ratelimit-reset", "600"}]) + + assert {:error, :ratelimit} = Backoff.get(@url) + assert_ttl_about(600) + end + test "matches header names case-insensitively" do mock_response(429, [{"Retry-After", "600"}]) From 0667705b773162f86ae4ddc6ceef200219e1d984 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 3 Nov 2026 22:29:49 +0400 Subject: [PATCH 51/51] test: use sigils for HTML strings in quote link tests --- test/pleroma/html_test.exs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/pleroma/html_test.exs b/test/pleroma/html_test.exs index 2bb918556..b8b694db2 100644 --- a/test/pleroma/html_test.exs +++ b/test/pleroma/html_test.exs @@ -343,8 +343,8 @@ defmodule Pleroma.HTMLTest do data: %{ "quoteUrl" => "https://raru.re/users/ocean/statuses/116235427788617987", "content" => - "

RE: raru.re/@ocean/116235427788617987

" <> - "

Berapa lapis?

youtube.com/watch?v=S0dJMhWDRDE

" + ~s(

RE: raru.re/@ocean/116235427788617987

) <> + ~s(

Berapa lapis?

youtube.com/watch?v=S0dJMhWDRDE

) } } @@ -356,7 +356,7 @@ defmodule Pleroma.HTMLTest do object = %Object{ data: %{ "content" => - "hello

RE: https://misskey.example/notes/abc" + ~s(hello

RE: https://misskey.example/notes/abc) } } @@ -368,7 +368,7 @@ defmodule Pleroma.HTMLTest do data: %{ "quoteUrl" => "https://remote.example/objects/1", "content" => - "RE: https://remote.example/objects/1 cofe" + ~s(RE: https://remote.example/objects/1 cofe) } }