From 8ce283f966376409ec92160962f14ccf85018b8b Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sat, 31 Oct 2026 14:18:40 +0400 Subject: [PATCH 1/3] feat(http): back off from hosts that answer 429 or 503 Object fetches and WebFinger stop sending requests to a host until Retry-After or X-RateLimit-Reset (default 5 minutes, max 1 hour). Ported from Akkoma's Pleroma.HTTP.Backoff. Co-authored-by: FloatingGhost Co-authored-by: Oneric --- changelog.d/http-backoff.add | 1 + lib/mix/pleroma.ex | 2 +- lib/pleroma/application.ex | 1 + lib/pleroma/http/backoff.ex | 101 ++++++++++++ lib/pleroma/object/fetcher.ex | 2 +- lib/pleroma/web/web_finger.ex | 14 +- test/pleroma/http/backoff_test.exs | 154 ++++++++++++++++++ test/pleroma/object/fetcher_test.exs | 21 +++ test/pleroma/web/web_finger_test.exs | 25 +++ .../workers/remote_fetcher_worker_test.exs | 14 ++ 10 files changed, 330 insertions(+), 5 deletions(-) create mode 100644 changelog.d/http-backoff.add create mode 100644 lib/pleroma/http/backoff.ex create mode 100644 test/pleroma/http/backoff_test.exs diff --git a/changelog.d/http-backoff.add b/changelog.d/http-backoff.add new file mode 100644 index 000000000..5c9eb988b --- /dev/null +++ b/changelog.d/http-backoff.add @@ -0,0 +1 @@ +Back off from hosts that answer 429 or 503 to object fetches and WebFinger, until Retry-After or X-RateLimit-Reset (default 5 minutes, max 1 hour) diff --git a/lib/mix/pleroma.ex b/lib/mix/pleroma.ex index aebad6984..080dd1065 100644 --- a/lib/mix/pleroma.ex +++ b/lib/mix/pleroma.ex @@ -17,7 +17,7 @@ defmodule Mix.Pleroma do :oban, :logger_backends ] - @cachex_children ["object", "user", "scrubber", "web_resp"] + @cachex_children ["object", "user", "scrubber", "web_resp", "http_backoff"] @doc "Common functions to be reused in mix tasks" def start_pleroma do Pleroma.Config.Holder.save_default() diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index adc054f3b..2c8eede1c 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -198,6 +198,7 @@ defmodule Pleroma.Application do ), build_cachex("rel_me", limit: 2500), build_cachex("host_meta", default_ttl: :timer.minutes(120), limit: 5_000), + build_cachex("http_backoff", limit: 10_000), build_cachex("translations", default_ttl: :timer.hours(24), limit: 5_000) ] end diff --git a/lib/pleroma/http/backoff.ex b/lib/pleroma/http/backoff.ex new file mode 100644 index 000000000..0d2b99742 --- /dev/null +++ b/lib/pleroma/http/backoff.ex @@ -0,0 +1,101 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.HTTP.Backoff do + @moduledoc """ + GET wrapper that stops sending requests to a host after it answers + 429 or 503, until the time given by `Retry-After` or `X-RateLimit-Reset`. + + Based on Akkoma's `Pleroma.HTTP.Backoff`. + """ + + alias Pleroma.HTTP + + require Logger + + @cachex Pleroma.Config.get([:cachex, :provider], Cachex) + @cache :http_backoff_cache + + @default_seconds 5 * 60 + @max_seconds 60 * 60 + + @spec get(String.t(), list(), keyword()) :: {:ok, Tesla.Env.t()} | {:error, any()} + def get(url, headers \\ [], options \\ []) when is_binary(url) do + host = URI.parse(url).host + + case @cachex.get(@cache, host) do + {:ok, true} -> + {:error, :ratelimit} + + _ -> + url + |> HTTP.get(headers, options) + |> check_backoff(host) + end + end + + defp check_backoff({:ok, %{status: status, headers: headers}}, host) + when status in [429, 503] do + seconds = backoff_seconds(headers, DateTime.utc_now()) + Logger.warning("#{host} answered #{status}, backing off for #{seconds} seconds") + @cachex.put(@cache, host, true, ttl: :timer.seconds(seconds)) + {:error, :ratelimit} + end + + defp check_backoff(response, _host), do: response + + @doc """ + Seconds to back off for, from the first valid `Retry-After` or + `X-RateLimit-Reset` header. Missing, invalid or past values give the + default of #{@default_seconds} seconds; values are capped at #{@max_seconds}. + """ + @spec backoff_seconds(list(), DateTime.t()) :: pos_integer() + def backoff_seconds(headers, now) do + [ + parse_retry_after(header(headers, "retry-after"), now), + parse_timestamp(header(headers, "x-ratelimit-reset"), now) + ] + |> Enum.find(@default_seconds, &is_integer/1) + |> min(@max_seconds) + end + + defp header(headers, name) do + Enum.find_value(headers, fn + {key, value} when is_binary(key) and is_binary(value) -> + if String.downcase(key) == name, do: String.trim(value) + + _ -> + nil + end) + end + + # Retry-After is either delta-seconds or an HTTP-date (RFC 9110, 10.2.3) + defp parse_retry_after(nil, _now), do: nil + + defp parse_retry_after(value, now) do + case Integer.parse(value) do + {seconds, ""} when seconds > 0 -> seconds + {_, ""} -> nil + _ -> parse_timestamp(value, now) + end + end + + defp parse_timestamp(nil, _now), do: nil + + defp parse_timestamp(value, now) do + with {:ok, %DateTime{} = datetime} <- parse_datetime(value), + seconds when seconds > 0 <- DateTime.diff(datetime, now) do + seconds + else + _ -> nil + end + end + + defp parse_datetime(value) do + with {:error, _} <- Timex.parse(value, "{RFC1123}"), + {:ok, datetime, _offset} <- DateTime.from_iso8601(value) do + {:ok, datetime} + end + end +end diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index 706ee317c..3313583cc 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -216,7 +216,7 @@ defmodule Pleroma.Object.Fetcher do |> maybe_date_fetch(date) |> sign_fetch(id, date) - case HTTP.get(id, headers) do + case HTTP.Backoff.get(id, headers) do {:ok, %{body: body, status: code, headers: headers, url: final_url}} when code in 200..299 -> remote_host = if final_url, do: URI.parse(final_url).host, else: nil diff --git a/lib/pleroma/web/web_finger.ex b/lib/pleroma/web/web_finger.ex index 62a2715f5..cb312405c 100644 --- a/lib/pleroma/web/web_finger.ex +++ b/lib/pleroma/web/web_finger.ex @@ -158,7 +158,11 @@ defmodule Pleroma.Web.WebFinger do @cachex Pleroma.Config.get([:cachex, :provider], Cachex) def find_lrdd_template(domain) do @cachex.fetch!(:host_meta_cache, domain, fn _ -> - {:commit, fetch_lrdd_template(domain)} + case fetch_lrdd_template(domain) do + # retry once the backoff expires instead of caching the miss + {:error, :ratelimit} = error -> {:ignore, error} + result -> {:commit, result} + end end) rescue e -> {:error, "Cachex error: #{inspect(e)}"} @@ -168,9 +172,13 @@ defmodule Pleroma.Web.WebFinger do # WebFinger is restricted to HTTPS - https://tools.ietf.org/html/rfc7033#section-9.1 meta_url = "https://#{domain}/.well-known/host-meta" - with {:ok, %{status: status, body: body}} when status in 200..299 <- HTTP.get(meta_url) do + with {:ok, %{status: status, body: body}} when status in 200..299 <- + HTTP.Backoff.get(meta_url) do get_template_from_xml(body) else + {:error, :ratelimit} = error -> + error + error -> Logger.warning("Can't find LRDD template in #{inspect(meta_url)}: #{inspect(error)}") {:error, :lrdd_not_found} @@ -210,7 +218,7 @@ defmodule Pleroma.Web.WebFinger do with address when is_binary(address) <- get_address_from_domain(domain, account), {:ok, %{status: status, body: body, headers: headers}} when status in 200..299 <- - HTTP.get( + HTTP.Backoff.get( address, [{"accept", "application/xrd+xml,application/jrd+json"}] ) do diff --git a/test/pleroma/http/backoff_test.exs b/test/pleroma/http/backoff_test.exs new file mode 100644 index 000000000..48c8514bd --- /dev/null +++ b/test/pleroma/http/backoff_test.exs @@ -0,0 +1,154 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +# Based on Akkoma's test/pleroma/http/backoff_test.exs +defmodule Pleroma.HTTP.BackoffTest do + use Pleroma.DataCase, async: false + + alias Pleroma.HTTP.Backoff + + @cache :http_backoff_cache + @url "https://ratelimited.example.com/users/alice" + @host "ratelimited.example.com" + + defp mock_response(status, headers \\ []) do + test_pid = self() + + Tesla.Mock.mock(fn %{method: :get} -> + send(test_pid, :request) + %Tesla.Env{status: status, headers: headers, body: ""} + end) + end + + defp ttl_seconds do + {:ok, ttl} = Cachex.ttl(@cache, @host) + div(ttl, 1000) + end + + defp assert_ttl_about(expected) do + ttl = ttl_seconds() + assert ttl in (expected - 15)..expected//1, "expected a TTL near #{expected}, got #{ttl}" + end + + describe "get/3" do + test "returns the response when the host is not backed off" do + mock_response(200) + + assert {:ok, %Tesla.Env{status: 200}} = Backoff.get(@url) + assert {:ok, nil} = Cachex.get(@cache, @host) + end + + test "does not send a request while the host is backed off" do + Cachex.put(@cache, @host, true) + mock_response(200) + + assert {:error, :ratelimit} = Backoff.get(@url) + refute_received :request + end + + for status <- [429, 503] do + test "backs off from the host after a #{status} response" do + mock_response(unquote(status)) + + assert {:error, :ratelimit} = Backoff.get(@url) + assert_received :request + assert {:ok, true} = Cachex.get(@cache, @host) + + assert {:error, :ratelimit} = Backoff.get(@url <> "/other") + refute_received :request + end + end + + test "uses a default backoff without headers" do + mock_response(429) + + assert {:error, :ratelimit} = Backoff.get(@url) + assert_ttl_about(300) + end + + test "only backs off from the host that answered" do + mock_response(429) + assert {:error, :ratelimit} = Backoff.get(@url) + + mock_response(200) + assert {:ok, _} = Backoff.get("https://other.example.com/users/bob") + end + end + + describe "backoff headers" do + test "parses Retry-After in seconds" do + mock_response(429, [{"retry-after", "600"}]) + + assert {:error, :ratelimit} = Backoff.get(@url) + assert_ttl_about(600) + end + + test "parses Retry-After as an HTTP-date" do + date = + DateTime.utc_now() + |> DateTime.add(600) + |> Timex.format!("{WDshort}, {0D} {Mshort} {YYYY} {h24}:{m}:{s} GMT") + + mock_response(503, [{"retry-after", date}]) + + assert {:error, :ratelimit} = Backoff.get(@url) + assert_ttl_about(600) + end + + test "parses Retry-After as an ISO 8601 timestamp" do + date = DateTime.utc_now() |> DateTime.add(600) |> DateTime.to_iso8601() + mock_response(429, [{"retry-after", date}]) + + assert {:error, :ratelimit} = Backoff.get(@url) + assert_ttl_about(600) + end + + test "parses X-RateLimit-Reset as an ISO 8601 timestamp" do + date = DateTime.utc_now() |> DateTime.add(600) |> DateTime.to_iso8601() + mock_response(429, [{"x-ratelimit-reset", date}]) + + assert {:error, :ratelimit} = Backoff.get(@url) + assert_ttl_about(600) + end + + test "matches header names case-insensitively" do + mock_response(429, [{"Retry-After", "600"}]) + + assert {:error, :ratelimit} = Backoff.get(@url) + assert_ttl_about(600) + end + + test "prefers Retry-After over X-RateLimit-Reset" do + date = DateTime.utc_now() |> DateTime.add(1200) |> DateTime.to_iso8601() + mock_response(429, [{"x-ratelimit-reset", date}, {"retry-after", "600"}]) + + assert {:error, :ratelimit} = Backoff.get(@url) + assert_ttl_about(600) + end + + test "falls back to the next header when one is invalid" do + date = DateTime.utc_now() |> DateTime.add(600) |> DateTime.to_iso8601() + mock_response(429, [{"retry-after", "soon"}, {"x-ratelimit-reset", date}]) + + assert {:error, :ratelimit} = Backoff.get(@url) + assert_ttl_about(600) + end + + for value <- ["soon", "-5", "0", "12.5", "", "Wed, 21 Oct 2015 07:28:00 GMT"] do + test "ignores the invalid or past Retry-After value #{inspect(value)}" do + mock_response(429, [{"retry-after", unquote(value)}]) + + assert {:error, :ratelimit} = Backoff.get(@url) + assert_ttl_about(300) + end + end + + test "caps absurd values at one hour" do + mock_response(429, [{"retry-after", "99999999999"}]) + + assert {:error, :ratelimit} = Backoff.get(@url) + assert_ttl_about(3600) + end + end +end diff --git a/test/pleroma/object/fetcher_test.exs b/test/pleroma/object/fetcher_test.exs index 61a6fb517..80b172a36 100644 --- a/test/pleroma/object/fetcher_test.exs +++ b/test/pleroma/object/fetcher_test.exs @@ -94,6 +94,27 @@ defmodule Pleroma.Object.FetcherTest do end end + describe "rate limiting" do + test "a backed-off host gets no further requests" do + test_pid = self() + + mock(fn %{method: :get, url: url} -> + send(test_pid, {:request, url}) + %Tesla.Env{status: 429, headers: [{"retry-after", "600"}]} + end) + + first = "https://ratelimited.example.com/objects/1" + second = "https://ratelimited.example.com/objects/2" + + assert {:fetch, {:error, :ratelimit}} = Fetcher.fetch_object_from_id(first) + assert_received {:request, ^first} + + assert {:error, :ratelimit} = Fetcher.fetch_and_contain_remote_object_from_id(second) + assert {:fetch, {:error, :ratelimit}} = Fetcher.fetch_object_from_id(second) + refute_received {:request, _} + end + end + describe "max thread distance restriction" do @ap_id "http://mastodon.example.org/@admin/99541947525187367" setup do: clear_config([:instance, :federation_incoming_replies_max_depth]) diff --git a/test/pleroma/web/web_finger_test.exs b/test/pleroma/web/web_finger_test.exs index da10abdd2..fc99bee41 100644 --- a/test/pleroma/web/web_finger_test.exs +++ b/test/pleroma/web/web_finger_test.exs @@ -305,4 +305,29 @@ defmodule Pleroma.Web.WebFingerTest do {:ok, _data} = WebFinger.finger("a@sub.mastodon.example") end end + + describe "rate limiting" do + setup do + Mox.stub_with(Pleroma.CachexMock, Pleroma.CachexProxy) + test_pid = self() + + Tesla.Mock.mock_global(fn %{url: url} -> + send(test_pid, {:request, url}) + %Tesla.Env{status: 429, headers: [{"retry-after", "600"}]} + end) + end + + test "backs off from a host that rate limits host-meta" do + assert {:error, :ratelimit} = WebFinger.finger("alice@ratelimited.example") + assert_received {:request, "https://ratelimited.example/.well-known/host-meta"} + refute_received {:request, _} + + # a rate-limited host-meta lookup must not be cached as "no LRDD template" + assert {:ok, nil} = Cachex.get(:host_meta_cache, "ratelimited.example") + + assert {:error, :ratelimit} = WebFinger.finger("bob@ratelimited.example") + assert {:error, :ratelimit} = WebFinger.find_lrdd_template("ratelimited.example") + refute_received {:request, _} + end + end end diff --git a/test/pleroma/workers/remote_fetcher_worker_test.exs b/test/pleroma/workers/remote_fetcher_worker_test.exs index a3b900a9b..9945ca143 100644 --- a/test/pleroma/workers/remote_fetcher_worker_test.exs +++ b/test/pleroma/workers/remote_fetcher_worker_test.exs @@ -13,6 +13,7 @@ defmodule Pleroma.Workers.RemoteFetcherWorkerTest do @unauthorized_object "https://unauthorized.example.com/" @depth_object "https://depth.example.com/" @content_type_object "https://bad_content_type.example.com/" + @ratelimited_object "https://ratelimited.example.com/" describe "RemoteFetcherWorker" do setup do @@ -37,6 +38,11 @@ defmodule Pleroma.Workers.RemoteFetcherWorkerTest do status: 200 } + %{method: :get, url: @ratelimited_object} -> + %Tesla.Env{ + status: 429 + } + %{method: :get, url: @content_type_object} -> %Tesla.Env{ status: 200, @@ -73,6 +79,14 @@ defmodule Pleroma.Workers.RemoteFetcherWorkerTest do }) end + test "retries jobs for a rate-limited host" do + assert {:error, :ratelimit} = + perform_job(RemoteFetcherWorker, %{ + "op" => "fetch_remote", + "id" => @ratelimited_object + }) + end + test "does not retry jobs for when object returns wrong content type" do assert {:cancel, _} = perform_job(RemoteFetcherWorker, %{ From b29d9d09810c58a1e48c85796d75283b74eb0ffb Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sat, 31 Oct 2026 14:35:58 +0400 Subject: [PATCH 2/3] fix(workers): snooze incoming jobs while a host is backed off With 5 attempts and Oban's short retry delays, an incoming activity from an unknown actor on a backed-off host was discarded without a request. ReceiverWorker and SignatureRetryWorker now snooze for up to 2 hours. Also lowercase backoff host keys and log backed-off user fetches at debug. --- lib/pleroma/http/backoff.ex | 23 ++++++++++++++- lib/pleroma/web/activity_pub/activity_pub.ex | 4 +++ lib/pleroma/workers/receiver_worker.ex | 5 +++- lib/pleroma/workers/signature_retry_worker.ex | 28 +++++++++++-------- test/pleroma/http/backoff_test.exs | 9 ++++++ test/pleroma/web/web_finger_test.exs | 1 + test/pleroma/workers/receiver_worker_test.exs | 22 +++++++++++++++ .../workers/signature_retry_worker_test.exs | 18 ++++++++++++ 8 files changed, 97 insertions(+), 13 deletions(-) diff --git a/lib/pleroma/http/backoff.ex b/lib/pleroma/http/backoff.ex index 0d2b99742..44b66fbdd 100644 --- a/lib/pleroma/http/backoff.ex +++ b/lib/pleroma/http/backoff.ex @@ -19,10 +19,11 @@ defmodule Pleroma.HTTP.Backoff do @default_seconds 5 * 60 @max_seconds 60 * 60 + @max_job_age 2 * 60 * 60 @spec get(String.t(), list(), keyword()) :: {:ok, Tesla.Env.t()} | {:error, any()} def get(url, headers \\ [], options \\ []) when is_binary(url) do - host = URI.parse(url).host + host = url |> URI.parse() |> Map.get(:host) |> downcase() case @cachex.get(@cache, host) do {:ok, true} -> @@ -35,6 +36,9 @@ defmodule Pleroma.HTTP.Backoff do end end + defp downcase(nil), do: nil + defp downcase(host), do: String.downcase(host) + defp check_backoff({:ok, %{status: status, headers: headers}}, host) when status in [429, 503] do seconds = backoff_seconds(headers, DateTime.utc_now()) @@ -45,6 +49,23 @@ defmodule Pleroma.HTTP.Backoff do defp check_backoff(response, _host), do: response + @doc """ + Turns an Oban job's `{:error, :ratelimit}` result into a snooze for the + default backoff, so the job's attempts aren't all spent while the host is + backed off. Jobs inserted more than #{@max_job_age} seconds ago get the + error back and fail as usual. + """ + @spec snooze_ratelimited(any(), Oban.Job.t()) :: any() + def snooze_ratelimited({:error, :ratelimit} = error, %Oban.Job{inserted_at: inserted_at}) do + if is_nil(inserted_at) or DateTime.diff(DateTime.utc_now(), inserted_at) < @max_job_age do + {:snooze, @default_seconds} + else + error + end + end + + def snooze_ratelimited(result, _job), do: result + @doc """ Seconds to back off for, from the first valid `Retry-After` or `X-RateLimit-Reset` header. Missing, invalid or past values give the diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index f6c74bdac..a6e6d4618 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -1949,6 +1949,10 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do Logger.info("Rejected user #{ap_id}: #{inspect(reason)}") {:error, e} + {:error, :ratelimit = e} -> + Logger.debug("Backing off from fetching user #{ap_id}") + {:error, e} + {:error, e} -> Logger.error("Could not decode user at fetch #{ap_id}, #{inspect(e)}") {:error, e} diff --git a/lib/pleroma/workers/receiver_worker.ex b/lib/pleroma/workers/receiver_worker.ex index 3afbe138d..4d1b5ba92 100644 --- a/lib/pleroma/workers/receiver_worker.ex +++ b/lib/pleroma/workers/receiver_worker.ex @@ -3,6 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Workers.ReceiverWorker do + alias Pleroma.HTTP.Backoff alias Pleroma.Instances alias Pleroma.Web.Federator alias Pleroma.Workers.SignatureRetryWorker @@ -14,7 +15,9 @@ defmodule Pleroma.Workers.ReceiverWorker do if signature_retry_job?(args) do perform_signature_retry(job) else - perform_incoming(params) + params + |> perform_incoming() + |> Backoff.snooze_ratelimited(job) end end diff --git a/lib/pleroma/workers/signature_retry_worker.ex b/lib/pleroma/workers/signature_retry_worker.ex index 28958faff..159d802ad 100644 --- a/lib/pleroma/workers/signature_retry_worker.ex +++ b/lib/pleroma/workers/signature_retry_worker.ex @@ -3,6 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Workers.SignatureRetryWorker do + alias Pleroma.HTTP.Backoff alias Pleroma.Instances alias Pleroma.Signature alias Pleroma.User @@ -16,16 +17,18 @@ defmodule Pleroma.Workers.SignatureRetryWorker do use Oban.Worker, queue: :federator_incoming, max_attempts: 5, unique: [period: :infinity] @impl true - def perform(%Job{ - args: %{ - "op" => "incoming_failed_signature_ap_doc", - "method" => method, - "params" => params, - "req_headers" => req_headers, - "request_path" => request_path, - "query_string" => query_string - } - }) + def perform( + %Job{ + args: %{ + "op" => "incoming_failed_signature_ap_doc", + "method" => method, + "params" => params, + "req_headers" => req_headers, + "request_path" => request_path, + "query_string" => query_string + } + } = job + ) when is_binary(method) and is_map(params) and is_list(req_headers) and is_binary(request_path) and is_binary(query_string) do case normalize_req_headers(req_headers) do @@ -58,7 +61,10 @@ defmodule Pleroma.Workers.SignatureRetryWorker do {:ok, res} else - e -> process_errors(e, retry_log_context(params, request_path, signature_actor_result)) + e -> + e + |> process_errors(retry_log_context(params, request_path, signature_actor_result)) + |> Backoff.snooze_ratelimited(job) end e -> diff --git a/test/pleroma/http/backoff_test.exs b/test/pleroma/http/backoff_test.exs index 48c8514bd..f893eef2e 100644 --- a/test/pleroma/http/backoff_test.exs +++ b/test/pleroma/http/backoff_test.exs @@ -67,6 +67,15 @@ defmodule Pleroma.HTTP.BackoffTest do assert_ttl_about(300) end + test "treats host names case-insensitively" do + mock_response(429) + assert {:error, :ratelimit} = Backoff.get("https://RateLimited.Example.com/users/alice") + assert_received :request + + assert {:error, :ratelimit} = Backoff.get(@url) + refute_received :request + end + test "only backs off from the host that answered" do mock_response(429) assert {:error, :ratelimit} = Backoff.get(@url) diff --git a/test/pleroma/web/web_finger_test.exs b/test/pleroma/web/web_finger_test.exs index fc99bee41..cf7cfa8b6 100644 --- a/test/pleroma/web/web_finger_test.exs +++ b/test/pleroma/web/web_finger_test.exs @@ -308,6 +308,7 @@ defmodule Pleroma.Web.WebFingerTest do describe "rate limiting" do setup do + # the module setup stubs NullCache, the backoff needs a real cache Mox.stub_with(Pleroma.CachexMock, Pleroma.CachexProxy) test_pid = self() diff --git a/test/pleroma/workers/receiver_worker_test.exs b/test/pleroma/workers/receiver_worker_test.exs index ea05f38f1..855ae3e2a 100644 --- a/test/pleroma/workers/receiver_worker_test.exs +++ b/test/pleroma/workers/receiver_worker_test.exs @@ -133,6 +133,28 @@ defmodule Pleroma.Workers.ReceiverWorkerTest do end end + describe "rate-limited fetches" do + setup do + Cachex.put(:http_backoff_cache, "ratelimited.example", true) + + params = + insert(:note_activity).data + |> Map.put("actor", "https://ratelimited.example/users/alice") + + %{args: %{"op" => "incoming_ap_doc", "params" => params}} + end + + test "snooze while the actor's host is backed off", %{args: args} do + assert {:snooze, 300} = perform_job(ReceiverWorker, args) + end + + test "stop snoozing two hours after the job was inserted", %{args: args} do + inserted_at = DateTime.add(DateTime.utc_now(), -121, :minute) + + assert {:error, :ratelimit} = perform_job(ReceiverWorker, args, inserted_at: inserted_at) + end + end + test "cancels due to origin containment" do params = insert(:note_activity).data diff --git a/test/pleroma/workers/signature_retry_worker_test.exs b/test/pleroma/workers/signature_retry_worker_test.exs index 3806ecac9..56835bbbc 100644 --- a/test/pleroma/workers/signature_retry_worker_test.exs +++ b/test/pleroma/workers/signature_retry_worker_test.exs @@ -306,6 +306,24 @@ defmodule Pleroma.Workers.SignatureRetryWorkerTest do refute Activity.get_by_ap_id(create["id"]) end + test "snoozes while the payload actor's host is backed off" do + Cachex.put(:http_backoff_cache, "ratelimited.example", true) + alice = build(:user, local: false, ap_id: "https://ratelimited.example/users/alice") + + params = %{ + "type" => "Create", + "actor" => alice.ap_id, + "id" => "https://ratelimited.example/activities/1", + "object" => "https://ratelimited.example/objects/1" + } + + job = failed_signature_job(params, signature_headers_for(alice)) + assert {:snooze, 300} = SignatureRetryWorker.perform(job) + + old_job = %{job | inserted_at: DateTime.add(DateTime.utc_now(), -121, :minute)} + assert {:error, :ratelimit} = SignatureRetryWorker.perform(old_job) + end + test "processes the activity after refetching a valid matching signature" do alice = insert(:user, local: false, ap_id: "https://one.com/users/alice") From 4070b7be21a28fa2f994483a5111922a9b363f73 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sun, 1 Nov 2026 17:03:11 +0400 Subject: [PATCH 3/3] fix(http): back off from the host that answered, and read Unix resets After a redirect the 429/503 came from the final host, but the backoff was recorded for the requested one. X-RateLimit-Reset is also accepted as a Unix timestamp or as seconds. --- lib/pleroma/http/backoff.ex | 35 ++++++++++++++++++++++++++++-- test/pleroma/http/backoff_test.exs | 32 +++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/http/backoff.ex b/lib/pleroma/http/backoff.ex index 44b66fbdd..5e432b285 100644 --- a/lib/pleroma/http/backoff.ex +++ b/lib/pleroma/http/backoff.ex @@ -39,8 +39,9 @@ defmodule Pleroma.HTTP.Backoff do defp downcase(nil), do: nil defp downcase(host), do: String.downcase(host) - defp check_backoff({:ok, %{status: status, headers: headers}}, host) + defp check_backoff({:ok, %{status: status, headers: headers} = env}, host) when status in [429, 503] do + host = response_host(env, host) seconds = backoff_seconds(headers, DateTime.utc_now()) Logger.warning("#{host} answered #{status}, backing off for #{seconds} seconds") @cachex.put(@cache, host, true, ttl: :timer.seconds(seconds)) @@ -49,6 +50,13 @@ defmodule Pleroma.HTTP.Backoff do defp check_backoff(response, _host), do: response + # After a redirect, the 429/503 came from the last host, not the requested one. + defp response_host(%{url: url}, host) when is_binary(url) do + URI.parse(url).host |> downcase() || host + end + + defp response_host(_env, host), do: host + @doc """ Turns an Oban job's `{:error, :ratelimit}` result into a snooze for the default backoff, so the job's attempts aren't all spent while the host is @@ -75,7 +83,7 @@ defmodule Pleroma.HTTP.Backoff do def backoff_seconds(headers, now) do [ parse_retry_after(header(headers, "retry-after"), now), - parse_timestamp(header(headers, "x-ratelimit-reset"), now) + parse_ratelimit_reset(header(headers, "x-ratelimit-reset"), now) ] |> Enum.find(@default_seconds, &is_integer/1) |> min(@max_seconds) @@ -102,6 +110,29 @@ defmodule Pleroma.HTTP.Backoff do end end + # Servers send X-RateLimit-Reset as a timestamp (ISO 8601 like Mastodon, or + # Unix seconds) or as seconds until the reset. + @unix_timestamp_min 1_000_000_000 + + defp parse_ratelimit_reset(nil, _now), do: nil + + defp parse_ratelimit_reset(value, now) do + case Integer.parse(value) do + {timestamp, ""} when timestamp >= @unix_timestamp_min -> + timestamp - DateTime.to_unix(now) + + {seconds, ""} -> + seconds + + _ -> + parse_timestamp(value, now) + end + |> positive() + end + + defp positive(seconds) when is_integer(seconds) and seconds > 0, do: seconds + defp positive(_), do: nil + defp parse_timestamp(nil, _now), do: nil defp parse_timestamp(value, now) do diff --git a/test/pleroma/http/backoff_test.exs b/test/pleroma/http/backoff_test.exs index f893eef2e..b650391dd 100644 --- a/test/pleroma/http/backoff_test.exs +++ b/test/pleroma/http/backoff_test.exs @@ -83,6 +83,23 @@ defmodule Pleroma.HTTP.BackoffTest do mock_response(200) assert {:ok, _} = Backoff.get("https://other.example.com/users/bob") end + + test "backs off from the host that answered after a redirect" do + test_pid = self() + + Tesla.Mock.mock(fn + %{url: "https://redirector.example.com/users/alice"} = env -> + %{env | status: 302, headers: [{"location", @url}], body: ""} + + %{url: @url} = env -> + send(test_pid, :request) + %{env | status: 429, headers: [], body: ""} + end) + + assert {:error, :ratelimit} = Backoff.get("https://redirector.example.com/users/alice") + assert {:ok, true} = Cachex.get(@cache, @host) + assert {:ok, nil} = Cachex.get(@cache, "redirector.example.com") + end end describe "backoff headers" do @@ -121,6 +138,21 @@ defmodule Pleroma.HTTP.BackoffTest do assert_ttl_about(600) end + test "parses X-RateLimit-Reset as a Unix timestamp" do + reset = DateTime.utc_now() |> DateTime.add(600) |> DateTime.to_unix() |> to_string() + mock_response(429, [{"x-ratelimit-reset", reset}]) + + assert {:error, :ratelimit} = Backoff.get(@url) + assert_ttl_about(600) + end + + test "parses X-RateLimit-Reset in seconds" do + mock_response(429, [{"x-ratelimit-reset", "600"}]) + + assert {:error, :ratelimit} = Backoff.get(@url) + assert_ttl_about(600) + end + test "matches header names case-insensitively" do mock_response(429, [{"Retry-After", "600"}])