diff --git a/changelog.d/http-backoff.add b/changelog.d/http-backoff.add
new file mode 100644
index 000000000..5c9eb988b
--- /dev/null
+++ b/changelog.d/http-backoff.add
@@ -0,0 +1 @@
+Back off from hosts that answer 429 or 503 to object fetches and WebFinger, until Retry-After or X-RateLimit-Reset (default 5 minutes, max 1 hour)
diff --git a/lib/mix/pleroma.ex b/lib/mix/pleroma.ex
index aebad6984..080dd1065 100644
--- a/lib/mix/pleroma.ex
+++ b/lib/mix/pleroma.ex
@@ -17,7 +17,7 @@ defmodule Mix.Pleroma do
:oban,
:logger_backends
]
- @cachex_children ["object", "user", "scrubber", "web_resp"]
+ @cachex_children ["object", "user", "scrubber", "web_resp", "http_backoff"]
@doc "Common functions to be reused in mix tasks"
def start_pleroma do
Pleroma.Config.Holder.save_default()
diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex
index adc054f3b..2c8eede1c 100644
--- a/lib/pleroma/application.ex
+++ b/lib/pleroma/application.ex
@@ -198,6 +198,7 @@ defmodule Pleroma.Application do
),
build_cachex("rel_me", limit: 2500),
build_cachex("host_meta", default_ttl: :timer.minutes(120), limit: 5_000),
+ build_cachex("http_backoff", limit: 10_000),
build_cachex("translations", default_ttl: :timer.hours(24), limit: 5_000)
]
end
diff --git a/lib/pleroma/http/backoff.ex b/lib/pleroma/http/backoff.ex
new file mode 100644
index 000000000..5e432b285
--- /dev/null
+++ b/lib/pleroma/http/backoff.ex
@@ -0,0 +1,153 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.HTTP.Backoff do
+ @moduledoc """
+ GET wrapper that stops sending requests to a host after it answers
+ 429 or 503, until the time given by `Retry-After` or `X-RateLimit-Reset`.
+
+ Based on Akkoma's `Pleroma.HTTP.Backoff`.
+ """
+
+ alias Pleroma.HTTP
+
+ require Logger
+
+ @cachex Pleroma.Config.get([:cachex, :provider], Cachex)
+ @cache :http_backoff_cache
+
+ @default_seconds 5 * 60
+ @max_seconds 60 * 60
+ @max_job_age 2 * 60 * 60
+
+ @spec get(String.t(), list(), keyword()) :: {:ok, Tesla.Env.t()} | {:error, any()}
+ def get(url, headers \\ [], options \\ []) when is_binary(url) do
+ host = url |> URI.parse() |> Map.get(:host) |> downcase()
+
+ case @cachex.get(@cache, host) do
+ {:ok, true} ->
+ {:error, :ratelimit}
+
+ _ ->
+ url
+ |> HTTP.get(headers, options)
+ |> check_backoff(host)
+ end
+ end
+
+ defp downcase(nil), do: nil
+ defp downcase(host), do: String.downcase(host)
+
+ defp check_backoff({:ok, %{status: status, headers: headers} = env}, host)
+ when status in [429, 503] do
+ host = response_host(env, host)
+ seconds = backoff_seconds(headers, DateTime.utc_now())
+ Logger.warning("#{host} answered #{status}, backing off for #{seconds} seconds")
+ @cachex.put(@cache, host, true, ttl: :timer.seconds(seconds))
+ {:error, :ratelimit}
+ end
+
+ defp check_backoff(response, _host), do: response
+
+ # After a redirect, the 429/503 came from the last host, not the requested one.
+ defp response_host(%{url: url}, host) when is_binary(url) do
+ URI.parse(url).host |> downcase() || host
+ end
+
+ defp response_host(_env, host), do: host
+
+ @doc """
+ Turns an Oban job's `{:error, :ratelimit}` result into a snooze for the
+ default backoff, so the job's attempts aren't all spent while the host is
+ backed off. Jobs inserted more than #{@max_job_age} seconds ago get the
+ error back and fail as usual.
+ """
+ @spec snooze_ratelimited(any(), Oban.Job.t()) :: any()
+ def snooze_ratelimited({:error, :ratelimit} = error, %Oban.Job{inserted_at: inserted_at}) do
+ if is_nil(inserted_at) or DateTime.diff(DateTime.utc_now(), inserted_at) < @max_job_age do
+ {:snooze, @default_seconds}
+ else
+ error
+ end
+ end
+
+ def snooze_ratelimited(result, _job), do: result
+
+ @doc """
+ Seconds to back off for, from the first valid `Retry-After` or
+ `X-RateLimit-Reset` header. Missing, invalid or past values give the
+ default of #{@default_seconds} seconds; values are capped at #{@max_seconds}.
+ """
+ @spec backoff_seconds(list(), DateTime.t()) :: pos_integer()
+ def backoff_seconds(headers, now) do
+ [
+ parse_retry_after(header(headers, "retry-after"), now),
+ parse_ratelimit_reset(header(headers, "x-ratelimit-reset"), now)
+ ]
+ |> Enum.find(@default_seconds, &is_integer/1)
+ |> min(@max_seconds)
+ end
+
+ defp header(headers, name) do
+ Enum.find_value(headers, fn
+ {key, value} when is_binary(key) and is_binary(value) ->
+ if String.downcase(key) == name, do: String.trim(value)
+
+ _ ->
+ nil
+ end)
+ end
+
+ # Retry-After is either delta-seconds or an HTTP-date (RFC 9110, 10.2.3)
+ defp parse_retry_after(nil, _now), do: nil
+
+ defp parse_retry_after(value, now) do
+ case Integer.parse(value) do
+ {seconds, ""} when seconds > 0 -> seconds
+ {_, ""} -> nil
+ _ -> parse_timestamp(value, now)
+ end
+ end
+
+ # Servers send X-RateLimit-Reset as a timestamp (ISO 8601 like Mastodon, or
+ # Unix seconds) or as seconds until the reset.
+ @unix_timestamp_min 1_000_000_000
+
+ defp parse_ratelimit_reset(nil, _now), do: nil
+
+ defp parse_ratelimit_reset(value, now) do
+ case Integer.parse(value) do
+ {timestamp, ""} when timestamp >= @unix_timestamp_min ->
+ timestamp - DateTime.to_unix(now)
+
+ {seconds, ""} ->
+ seconds
+
+ _ ->
+ parse_timestamp(value, now)
+ end
+ |> positive()
+ end
+
+ defp positive(seconds) when is_integer(seconds) and seconds > 0, do: seconds
+ defp positive(_), do: nil
+
+ defp parse_timestamp(nil, _now), do: nil
+
+ defp parse_timestamp(value, now) do
+ with {:ok, %DateTime{} = datetime} <- parse_datetime(value),
+ seconds when seconds > 0 <- DateTime.diff(datetime, now) do
+ seconds
+ else
+ _ -> nil
+ end
+ end
+
+ defp parse_datetime(value) do
+ with {:error, _} <- Timex.parse(value, "{RFC1123}"),
+ {:ok, datetime, _offset} <- DateTime.from_iso8601(value) do
+ {:ok, datetime}
+ end
+ end
+end
diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex
index 706ee317c..3313583cc 100644
--- a/lib/pleroma/object/fetcher.ex
+++ b/lib/pleroma/object/fetcher.ex
@@ -216,7 +216,7 @@ defmodule Pleroma.Object.Fetcher do
|> maybe_date_fetch(date)
|> sign_fetch(id, date)
- case HTTP.get(id, headers) do
+ case HTTP.Backoff.get(id, headers) do
{:ok, %{body: body, status: code, headers: headers, url: final_url}}
when code in 200..299 ->
remote_host = if final_url, do: URI.parse(final_url).host, else: nil
diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index 03f8447bb..807e68ffb 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -1956,6 +1956,10 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
Logger.info("Rejected user #{ap_id}: #{inspect(reason)}")
{:error, e}
+ {:error, :ratelimit = e} ->
+ Logger.debug("Backing off from fetching user #{ap_id}")
+ {:error, e}
+
{:error, e} ->
Logger.error("Could not decode user at fetch #{ap_id}, #{inspect(e)}")
{:error, e}
diff --git a/lib/pleroma/web/web_finger.ex b/lib/pleroma/web/web_finger.ex
index 62a2715f5..cb312405c 100644
--- a/lib/pleroma/web/web_finger.ex
+++ b/lib/pleroma/web/web_finger.ex
@@ -158,7 +158,11 @@ defmodule Pleroma.Web.WebFinger do
@cachex Pleroma.Config.get([:cachex, :provider], Cachex)
def find_lrdd_template(domain) do
@cachex.fetch!(:host_meta_cache, domain, fn _ ->
- {:commit, fetch_lrdd_template(domain)}
+ case fetch_lrdd_template(domain) do
+ # retry once the backoff expires instead of caching the miss
+ {:error, :ratelimit} = error -> {:ignore, error}
+ result -> {:commit, result}
+ end
end)
rescue
e -> {:error, "Cachex error: #{inspect(e)}"}
@@ -168,9 +172,13 @@ defmodule Pleroma.Web.WebFinger do
# WebFinger is restricted to HTTPS - https://tools.ietf.org/html/rfc7033#section-9.1
meta_url = "https://#{domain}/.well-known/host-meta"
- with {:ok, %{status: status, body: body}} when status in 200..299 <- HTTP.get(meta_url) do
+ with {:ok, %{status: status, body: body}} when status in 200..299 <-
+ HTTP.Backoff.get(meta_url) do
get_template_from_xml(body)
else
+ {:error, :ratelimit} = error ->
+ error
+
error ->
Logger.warning("Can't find LRDD template in #{inspect(meta_url)}: #{inspect(error)}")
{:error, :lrdd_not_found}
@@ -210,7 +218,7 @@ defmodule Pleroma.Web.WebFinger do
with address when is_binary(address) <- get_address_from_domain(domain, account),
{:ok, %{status: status, body: body, headers: headers}} when status in 200..299 <-
- HTTP.get(
+ HTTP.Backoff.get(
address,
[{"accept", "application/xrd+xml,application/jrd+json"}]
) do
diff --git a/lib/pleroma/workers/receiver_worker.ex b/lib/pleroma/workers/receiver_worker.ex
index 3afbe138d..4d1b5ba92 100644
--- a/lib/pleroma/workers/receiver_worker.ex
+++ b/lib/pleroma/workers/receiver_worker.ex
@@ -3,6 +3,7 @@
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Workers.ReceiverWorker do
+ alias Pleroma.HTTP.Backoff
alias Pleroma.Instances
alias Pleroma.Web.Federator
alias Pleroma.Workers.SignatureRetryWorker
@@ -14,7 +15,9 @@ defmodule Pleroma.Workers.ReceiverWorker do
if signature_retry_job?(args) do
perform_signature_retry(job)
else
- perform_incoming(params)
+ params
+ |> perform_incoming()
+ |> Backoff.snooze_ratelimited(job)
end
end
diff --git a/lib/pleroma/workers/signature_retry_worker.ex b/lib/pleroma/workers/signature_retry_worker.ex
index 28958faff..159d802ad 100644
--- a/lib/pleroma/workers/signature_retry_worker.ex
+++ b/lib/pleroma/workers/signature_retry_worker.ex
@@ -3,6 +3,7 @@
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Workers.SignatureRetryWorker do
+ alias Pleroma.HTTP.Backoff
alias Pleroma.Instances
alias Pleroma.Signature
alias Pleroma.User
@@ -16,16 +17,18 @@ defmodule Pleroma.Workers.SignatureRetryWorker do
use Oban.Worker, queue: :federator_incoming, max_attempts: 5, unique: [period: :infinity]
@impl true
- def perform(%Job{
- args: %{
- "op" => "incoming_failed_signature_ap_doc",
- "method" => method,
- "params" => params,
- "req_headers" => req_headers,
- "request_path" => request_path,
- "query_string" => query_string
- }
- })
+ def perform(
+ %Job{
+ args: %{
+ "op" => "incoming_failed_signature_ap_doc",
+ "method" => method,
+ "params" => params,
+ "req_headers" => req_headers,
+ "request_path" => request_path,
+ "query_string" => query_string
+ }
+ } = job
+ )
when is_binary(method) and is_map(params) and is_list(req_headers) and
is_binary(request_path) and is_binary(query_string) do
case normalize_req_headers(req_headers) do
@@ -58,7 +61,10 @@ defmodule Pleroma.Workers.SignatureRetryWorker do
{:ok, res}
else
- e -> process_errors(e, retry_log_context(params, request_path, signature_actor_result))
+ e ->
+ e
+ |> process_errors(retry_log_context(params, request_path, signature_actor_result))
+ |> Backoff.snooze_ratelimited(job)
end
e ->
diff --git a/test/pleroma/http/backoff_test.exs b/test/pleroma/http/backoff_test.exs
new file mode 100644
index 000000000..b650391dd
--- /dev/null
+++ b/test/pleroma/http/backoff_test.exs
@@ -0,0 +1,195 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors
+# SPDX-License-Identifier: AGPL-3.0-only
+
+# Based on Akkoma's test/pleroma/http/backoff_test.exs
+defmodule Pleroma.HTTP.BackoffTest do
+ use Pleroma.DataCase, async: false
+
+ alias Pleroma.HTTP.Backoff
+
+ @cache :http_backoff_cache
+ @url "https://ratelimited.example.com/users/alice"
+ @host "ratelimited.example.com"
+
+ defp mock_response(status, headers \\ []) do
+ test_pid = self()
+
+ Tesla.Mock.mock(fn %{method: :get} ->
+ send(test_pid, :request)
+ %Tesla.Env{status: status, headers: headers, body: ""}
+ end)
+ end
+
+ defp ttl_seconds do
+ {:ok, ttl} = Cachex.ttl(@cache, @host)
+ div(ttl, 1000)
+ end
+
+ defp assert_ttl_about(expected) do
+ ttl = ttl_seconds()
+ assert ttl in (expected - 15)..expected//1, "expected a TTL near #{expected}, got #{ttl}"
+ end
+
+ describe "get/3" do
+ test "returns the response when the host is not backed off" do
+ mock_response(200)
+
+ assert {:ok, %Tesla.Env{status: 200}} = Backoff.get(@url)
+ assert {:ok, nil} = Cachex.get(@cache, @host)
+ end
+
+ test "does not send a request while the host is backed off" do
+ Cachex.put(@cache, @host, true)
+ mock_response(200)
+
+ assert {:error, :ratelimit} = Backoff.get(@url)
+ refute_received :request
+ end
+
+ for status <- [429, 503] do
+ test "backs off from the host after a #{status} response" do
+ mock_response(unquote(status))
+
+ assert {:error, :ratelimit} = Backoff.get(@url)
+ assert_received :request
+ assert {:ok, true} = Cachex.get(@cache, @host)
+
+ assert {:error, :ratelimit} = Backoff.get(@url <> "/other")
+ refute_received :request
+ end
+ end
+
+ test "uses a default backoff without headers" do
+ mock_response(429)
+
+ assert {:error, :ratelimit} = Backoff.get(@url)
+ assert_ttl_about(300)
+ end
+
+ test "treats host names case-insensitively" do
+ mock_response(429)
+ assert {:error, :ratelimit} = Backoff.get("https://RateLimited.Example.com/users/alice")
+ assert_received :request
+
+ assert {:error, :ratelimit} = Backoff.get(@url)
+ refute_received :request
+ end
+
+ test "only backs off from the host that answered" do
+ mock_response(429)
+ assert {:error, :ratelimit} = Backoff.get(@url)
+
+ mock_response(200)
+ assert {:ok, _} = Backoff.get("https://other.example.com/users/bob")
+ end
+
+ test "backs off from the host that answered after a redirect" do
+ test_pid = self()
+
+ Tesla.Mock.mock(fn
+ %{url: "https://redirector.example.com/users/alice"} = env ->
+ %{env | status: 302, headers: [{"location", @url}], body: ""}
+
+ %{url: @url} = env ->
+ send(test_pid, :request)
+ %{env | status: 429, headers: [], body: ""}
+ end)
+
+ assert {:error, :ratelimit} = Backoff.get("https://redirector.example.com/users/alice")
+ assert {:ok, true} = Cachex.get(@cache, @host)
+ assert {:ok, nil} = Cachex.get(@cache, "redirector.example.com")
+ end
+ end
+
+ describe "backoff headers" do
+ test "parses Retry-After in seconds" do
+ mock_response(429, [{"retry-after", "600"}])
+
+ assert {:error, :ratelimit} = Backoff.get(@url)
+ assert_ttl_about(600)
+ end
+
+ test "parses Retry-After as an HTTP-date" do
+ date =
+ DateTime.utc_now()
+ |> DateTime.add(600)
+ |> Timex.format!("{WDshort}, {0D} {Mshort} {YYYY} {h24}:{m}:{s} GMT")
+
+ mock_response(503, [{"retry-after", date}])
+
+ assert {:error, :ratelimit} = Backoff.get(@url)
+ assert_ttl_about(600)
+ end
+
+ test "parses Retry-After as an ISO 8601 timestamp" do
+ date = DateTime.utc_now() |> DateTime.add(600) |> DateTime.to_iso8601()
+ mock_response(429, [{"retry-after", date}])
+
+ assert {:error, :ratelimit} = Backoff.get(@url)
+ assert_ttl_about(600)
+ end
+
+ test "parses X-RateLimit-Reset as an ISO 8601 timestamp" do
+ date = DateTime.utc_now() |> DateTime.add(600) |> DateTime.to_iso8601()
+ mock_response(429, [{"x-ratelimit-reset", date}])
+
+ assert {:error, :ratelimit} = Backoff.get(@url)
+ assert_ttl_about(600)
+ end
+
+ test "parses X-RateLimit-Reset as a Unix timestamp" do
+ reset = DateTime.utc_now() |> DateTime.add(600) |> DateTime.to_unix() |> to_string()
+ mock_response(429, [{"x-ratelimit-reset", reset}])
+
+ assert {:error, :ratelimit} = Backoff.get(@url)
+ assert_ttl_about(600)
+ end
+
+ test "parses X-RateLimit-Reset in seconds" do
+ mock_response(429, [{"x-ratelimit-reset", "600"}])
+
+ assert {:error, :ratelimit} = Backoff.get(@url)
+ assert_ttl_about(600)
+ end
+
+ test "matches header names case-insensitively" do
+ mock_response(429, [{"Retry-After", "600"}])
+
+ assert {:error, :ratelimit} = Backoff.get(@url)
+ assert_ttl_about(600)
+ end
+
+ test "prefers Retry-After over X-RateLimit-Reset" do
+ date = DateTime.utc_now() |> DateTime.add(1200) |> DateTime.to_iso8601()
+ mock_response(429, [{"x-ratelimit-reset", date}, {"retry-after", "600"}])
+
+ assert {:error, :ratelimit} = Backoff.get(@url)
+ assert_ttl_about(600)
+ end
+
+ test "falls back to the next header when one is invalid" do
+ date = DateTime.utc_now() |> DateTime.add(600) |> DateTime.to_iso8601()
+ mock_response(429, [{"retry-after", "soon"}, {"x-ratelimit-reset", date}])
+
+ assert {:error, :ratelimit} = Backoff.get(@url)
+ assert_ttl_about(600)
+ end
+
+ for value <- ["soon", "-5", "0", "12.5", "", "Wed, 21 Oct 2015 07:28:00 GMT"] do
+ test "ignores the invalid or past Retry-After value #{inspect(value)}" do
+ mock_response(429, [{"retry-after", unquote(value)}])
+
+ assert {:error, :ratelimit} = Backoff.get(@url)
+ assert_ttl_about(300)
+ end
+ end
+
+ test "caps absurd values at one hour" do
+ mock_response(429, [{"retry-after", "99999999999"}])
+
+ assert {:error, :ratelimit} = Backoff.get(@url)
+ assert_ttl_about(3600)
+ end
+ end
+end
diff --git a/test/pleroma/object/fetcher_test.exs b/test/pleroma/object/fetcher_test.exs
index 61a6fb517..80b172a36 100644
--- a/test/pleroma/object/fetcher_test.exs
+++ b/test/pleroma/object/fetcher_test.exs
@@ -94,6 +94,27 @@ defmodule Pleroma.Object.FetcherTest do
end
end
+ describe "rate limiting" do
+ test "a backed-off host gets no further requests" do
+ test_pid = self()
+
+ mock(fn %{method: :get, url: url} ->
+ send(test_pid, {:request, url})
+ %Tesla.Env{status: 429, headers: [{"retry-after", "600"}]}
+ end)
+
+ first = "https://ratelimited.example.com/objects/1"
+ second = "https://ratelimited.example.com/objects/2"
+
+ assert {:fetch, {:error, :ratelimit}} = Fetcher.fetch_object_from_id(first)
+ assert_received {:request, ^first}
+
+ assert {:error, :ratelimit} = Fetcher.fetch_and_contain_remote_object_from_id(second)
+ assert {:fetch, {:error, :ratelimit}} = Fetcher.fetch_object_from_id(second)
+ refute_received {:request, _}
+ end
+ end
+
describe "max thread distance restriction" do
@ap_id "http://mastodon.example.org/@admin/99541947525187367"
setup do: clear_config([:instance, :federation_incoming_replies_max_depth])
diff --git a/test/pleroma/web/web_finger_test.exs b/test/pleroma/web/web_finger_test.exs
index da10abdd2..cf7cfa8b6 100644
--- a/test/pleroma/web/web_finger_test.exs
+++ b/test/pleroma/web/web_finger_test.exs
@@ -305,4 +305,30 @@ defmodule Pleroma.Web.WebFingerTest do
{:ok, _data} = WebFinger.finger("a@sub.mastodon.example")
end
end
+
+ describe "rate limiting" do
+ setup do
+ # the module setup stubs NullCache, the backoff needs a real cache
+ Mox.stub_with(Pleroma.CachexMock, Pleroma.CachexProxy)
+ test_pid = self()
+
+ Tesla.Mock.mock_global(fn %{url: url} ->
+ send(test_pid, {:request, url})
+ %Tesla.Env{status: 429, headers: [{"retry-after", "600"}]}
+ end)
+ end
+
+ test "backs off from a host that rate limits host-meta" do
+ assert {:error, :ratelimit} = WebFinger.finger("alice@ratelimited.example")
+ assert_received {:request, "https://ratelimited.example/.well-known/host-meta"}
+ refute_received {:request, _}
+
+ # a rate-limited host-meta lookup must not be cached as "no LRDD template"
+ assert {:ok, nil} = Cachex.get(:host_meta_cache, "ratelimited.example")
+
+ assert {:error, :ratelimit} = WebFinger.finger("bob@ratelimited.example")
+ assert {:error, :ratelimit} = WebFinger.find_lrdd_template("ratelimited.example")
+ refute_received {:request, _}
+ end
+ end
end
diff --git a/test/pleroma/workers/receiver_worker_test.exs b/test/pleroma/workers/receiver_worker_test.exs
index ea05f38f1..855ae3e2a 100644
--- a/test/pleroma/workers/receiver_worker_test.exs
+++ b/test/pleroma/workers/receiver_worker_test.exs
@@ -133,6 +133,28 @@ defmodule Pleroma.Workers.ReceiverWorkerTest do
end
end
+ describe "rate-limited fetches" do
+ setup do
+ Cachex.put(:http_backoff_cache, "ratelimited.example", true)
+
+ params =
+ insert(:note_activity).data
+ |> Map.put("actor", "https://ratelimited.example/users/alice")
+
+ %{args: %{"op" => "incoming_ap_doc", "params" => params}}
+ end
+
+ test "snooze while the actor's host is backed off", %{args: args} do
+ assert {:snooze, 300} = perform_job(ReceiverWorker, args)
+ end
+
+ test "stop snoozing two hours after the job was inserted", %{args: args} do
+ inserted_at = DateTime.add(DateTime.utc_now(), -121, :minute)
+
+ assert {:error, :ratelimit} = perform_job(ReceiverWorker, args, inserted_at: inserted_at)
+ end
+ end
+
test "cancels due to origin containment" do
params =
insert(:note_activity).data
diff --git a/test/pleroma/workers/remote_fetcher_worker_test.exs b/test/pleroma/workers/remote_fetcher_worker_test.exs
index a3b900a9b..9945ca143 100644
--- a/test/pleroma/workers/remote_fetcher_worker_test.exs
+++ b/test/pleroma/workers/remote_fetcher_worker_test.exs
@@ -13,6 +13,7 @@ defmodule Pleroma.Workers.RemoteFetcherWorkerTest do
@unauthorized_object "https://unauthorized.example.com/"
@depth_object "https://depth.example.com/"
@content_type_object "https://bad_content_type.example.com/"
+ @ratelimited_object "https://ratelimited.example.com/"
describe "RemoteFetcherWorker" do
setup do
@@ -37,6 +38,11 @@ defmodule Pleroma.Workers.RemoteFetcherWorkerTest do
status: 200
}
+ %{method: :get, url: @ratelimited_object} ->
+ %Tesla.Env{
+ status: 429
+ }
+
%{method: :get, url: @content_type_object} ->
%Tesla.Env{
status: 200,
@@ -73,6 +79,14 @@ defmodule Pleroma.Workers.RemoteFetcherWorkerTest do
})
end
+ test "retries jobs for a rate-limited host" do
+ assert {:error, :ratelimit} =
+ perform_job(RemoteFetcherWorker, %{
+ "op" => "fetch_remote",
+ "id" => @ratelimited_object
+ })
+ end
+
test "does not retry jobs for when object returns wrong content type" do
assert {:cancel, _} =
perform_job(RemoteFetcherWorker, %{
diff --git a/test/pleroma/workers/signature_retry_worker_test.exs b/test/pleroma/workers/signature_retry_worker_test.exs
index 3806ecac9..56835bbbc 100644
--- a/test/pleroma/workers/signature_retry_worker_test.exs
+++ b/test/pleroma/workers/signature_retry_worker_test.exs
@@ -306,6 +306,24 @@ defmodule Pleroma.Workers.SignatureRetryWorkerTest do
refute Activity.get_by_ap_id(create["id"])
end
+ test "snoozes while the payload actor's host is backed off" do
+ Cachex.put(:http_backoff_cache, "ratelimited.example", true)
+ alice = build(:user, local: false, ap_id: "https://ratelimited.example/users/alice")
+
+ params = %{
+ "type" => "Create",
+ "actor" => alice.ap_id,
+ "id" => "https://ratelimited.example/activities/1",
+ "object" => "https://ratelimited.example/objects/1"
+ }
+
+ job = failed_signature_job(params, signature_headers_for(alice))
+ assert {:snooze, 300} = SignatureRetryWorker.perform(job)
+
+ old_job = %{job | inserted_at: DateTime.add(DateTime.utc_now(), -121, :minute)}
+ assert {:error, :ratelimit} = SignatureRetryWorker.perform(old_job)
+ end
+
test "processes the activity after refetching a valid matching signature" do
alice = insert(:user, local: false, ap_id: "https://one.com/users/alice")