Merge pull request 'HTTP: back off from hosts that answer 429 or 503' (#8049) from lambadalambda/pleroma:feat/http-backoff into develop

Reviewed-on: https://git.pleroma.social/pleroma/pleroma/pulls/8049

test/pleroma/http/backoff_test.exs

@@ -0,0 +1,195 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+# Based on Akkoma's test/pleroma/http/backoff_test.exs
+defmodule Pleroma.HTTP.BackoffTest do
+  use Pleroma.DataCase, async: false
+
+  alias Pleroma.HTTP.Backoff
+
+  @cache :http_backoff_cache
+  @url "https://ratelimited.example.com/users/alice"
+  @host "ratelimited.example.com"
+
+  defp mock_response(status, headers \\ []) do
+    test_pid = self()
+
+    Tesla.Mock.mock(fn %{method: :get} ->
+      send(test_pid, :request)
+      %Tesla.Env{status: status, headers: headers, body: ""}
+    end)
+  end
+
+  defp ttl_seconds do
+    {:ok, ttl} = Cachex.ttl(@cache, @host)
+    div(ttl, 1000)
+  end
+
+  defp assert_ttl_about(expected) do
+    ttl = ttl_seconds()
+    assert ttl in (expected - 15)..expected//1, "expected a TTL near #{expected}, got #{ttl}"
+  end
+
+  describe "get/3" do
+    test "returns the response when the host is not backed off" do
+      mock_response(200)
+
+      assert {:ok, %Tesla.Env{status: 200}} = Backoff.get(@url)
+      assert {:ok, nil} = Cachex.get(@cache, @host)
+    end
+
+    test "does not send a request while the host is backed off" do
+      Cachex.put(@cache, @host, true)
+      mock_response(200)
+
+      assert {:error, :ratelimit} = Backoff.get(@url)
+      refute_received :request
+    end
+
+    for status <- [429, 503] do
+      test "backs off from the host after a #{status} response" do
+        mock_response(unquote(status))
+
+        assert {:error, :ratelimit} = Backoff.get(@url)
+        assert_received :request
+        assert {:ok, true} = Cachex.get(@cache, @host)
+
+        assert {:error, :ratelimit} = Backoff.get(@url <> "/other")
+        refute_received :request
+      end
+    end
+
+    test "uses a default backoff without headers" do
+      mock_response(429)
+
+      assert {:error, :ratelimit} = Backoff.get(@url)
+      assert_ttl_about(300)
+    end
+
+    test "treats host names case-insensitively" do
+      mock_response(429)
+      assert {:error, :ratelimit} = Backoff.get("https://RateLimited.Example.com/users/alice")
+      assert_received :request
+
+      assert {:error, :ratelimit} = Backoff.get(@url)
+      refute_received :request
+    end
+
+    test "only backs off from the host that answered" do
+      mock_response(429)
+      assert {:error, :ratelimit} = Backoff.get(@url)
+
+      mock_response(200)
+      assert {:ok, _} = Backoff.get("https://other.example.com/users/bob")
+    end
+
+    test "backs off from the host that answered after a redirect" do
+      test_pid = self()
+
+      Tesla.Mock.mock(fn
+        %{url: "https://redirector.example.com/users/alice"} = env ->
+          %{env | status: 302, headers: [{"location", @url}], body: ""}
+
+        %{url: @url} = env ->
+          send(test_pid, :request)
+          %{env | status: 429, headers: [], body: ""}
+      end)
+
+      assert {:error, :ratelimit} = Backoff.get("https://redirector.example.com/users/alice")
+      assert {:ok, true} = Cachex.get(@cache, @host)
+      assert {:ok, nil} = Cachex.get(@cache, "redirector.example.com")
+    end
+  end
+
+  describe "backoff headers" do
+    test "parses Retry-After in seconds" do
+      mock_response(429, [{"retry-after", "600"}])
+
+      assert {:error, :ratelimit} = Backoff.get(@url)
+      assert_ttl_about(600)
+    end
+
+    test "parses Retry-After as an HTTP-date" do
+      date =
+        DateTime.utc_now()
+        |> DateTime.add(600)
+        |> Timex.format!("{WDshort}, {0D} {Mshort} {YYYY} {h24}:{m}:{s} GMT")
+
+      mock_response(503, [{"retry-after", date}])
+
+      assert {:error, :ratelimit} = Backoff.get(@url)
+      assert_ttl_about(600)
+    end
+
+    test "parses Retry-After as an ISO 8601 timestamp" do
+      date = DateTime.utc_now() |> DateTime.add(600) |> DateTime.to_iso8601()
+      mock_response(429, [{"retry-after", date}])
+
+      assert {:error, :ratelimit} = Backoff.get(@url)
+      assert_ttl_about(600)
+    end
+
+    test "parses X-RateLimit-Reset as an ISO 8601 timestamp" do
+      date = DateTime.utc_now() |> DateTime.add(600) |> DateTime.to_iso8601()
+      mock_response(429, [{"x-ratelimit-reset", date}])
+
+      assert {:error, :ratelimit} = Backoff.get(@url)
+      assert_ttl_about(600)
+    end
+
+    test "parses X-RateLimit-Reset as a Unix timestamp" do
+      reset = DateTime.utc_now() |> DateTime.add(600) |> DateTime.to_unix() |> to_string()
+      mock_response(429, [{"x-ratelimit-reset", reset}])
+
+      assert {:error, :ratelimit} = Backoff.get(@url)
+      assert_ttl_about(600)
+    end
+
+    test "parses X-RateLimit-Reset in seconds" do
+      mock_response(429, [{"x-ratelimit-reset", "600"}])
+
+      assert {:error, :ratelimit} = Backoff.get(@url)
+      assert_ttl_about(600)
+    end
+
+    test "matches header names case-insensitively" do
+      mock_response(429, [{"Retry-After", "600"}])
+
+      assert {:error, :ratelimit} = Backoff.get(@url)
+      assert_ttl_about(600)
+    end
+
+    test "prefers Retry-After over X-RateLimit-Reset" do
+      date = DateTime.utc_now() |> DateTime.add(1200) |> DateTime.to_iso8601()
+      mock_response(429, [{"x-ratelimit-reset", date}, {"retry-after", "600"}])
+
+      assert {:error, :ratelimit} = Backoff.get(@url)
+      assert_ttl_about(600)
+    end
+
+    test "falls back to the next header when one is invalid" do
+      date = DateTime.utc_now() |> DateTime.add(600) |> DateTime.to_iso8601()
+      mock_response(429, [{"retry-after", "soon"}, {"x-ratelimit-reset", date}])
+
+      assert {:error, :ratelimit} = Backoff.get(@url)
+      assert_ttl_about(600)
+    end
+
+    for value <- ["soon", "-5", "0", "12.5", "", "Wed, 21 Oct 2015 07:28:00 GMT"] do
+      test "ignores the invalid or past Retry-After value #{inspect(value)}" do
+        mock_response(429, [{"retry-after", unquote(value)}])
+
+        assert {:error, :ratelimit} = Backoff.get(@url)
+        assert_ttl_about(300)
+      end
+    end
+
+    test "caps absurd values at one hour" do
+      mock_response(429, [{"retry-after", "99999999999"}])
+
+      assert {:error, :ratelimit} = Backoff.get(@url)
+      assert_ttl_about(3600)
+    end
+  end
+end

test/pleroma/object/fetcher_test.exs

@@ -94,6 +94,27 @@ defmodule Pleroma.Object.FetcherTest do
     end
   end
 
+  describe "rate limiting" do
+    test "a backed-off host gets no further requests" do
+      test_pid = self()
+
+      mock(fn %{method: :get, url: url} ->
+        send(test_pid, {:request, url})
+        %Tesla.Env{status: 429, headers: [{"retry-after", "600"}]}
+      end)
+
+      first = "https://ratelimited.example.com/objects/1"
+      second = "https://ratelimited.example.com/objects/2"
+
+      assert {:fetch, {:error, :ratelimit}} = Fetcher.fetch_object_from_id(first)
+      assert_received {:request, ^first}
+
+      assert {:error, :ratelimit} = Fetcher.fetch_and_contain_remote_object_from_id(second)
+      assert {:fetch, {:error, :ratelimit}} = Fetcher.fetch_object_from_id(second)
+      refute_received {:request, _}
+    end
+  end
+
   describe "max thread distance restriction" do
     @ap_id "http://mastodon.example.org/@admin/99541947525187367"
     setup do: clear_config([:instance, :federation_incoming_replies_max_depth])

test/pleroma/web/web_finger_test.exs

@@ -305,4 +305,30 @@ defmodule Pleroma.Web.WebFingerTest do
       {:ok, _data} = WebFinger.finger("a@sub.mastodon.example")
     end
   end
+
+  describe "rate limiting" do
+    setup do
+      # the module setup stubs NullCache, the backoff needs a real cache
+      Mox.stub_with(Pleroma.CachexMock, Pleroma.CachexProxy)
+      test_pid = self()
+
+      Tesla.Mock.mock_global(fn %{url: url} ->
+        send(test_pid, {:request, url})
+        %Tesla.Env{status: 429, headers: [{"retry-after", "600"}]}
+      end)
+    end
+
+    test "backs off from a host that rate limits host-meta" do
+      assert {:error, :ratelimit} = WebFinger.finger("alice@ratelimited.example")
+      assert_received {:request, "https://ratelimited.example/.well-known/host-meta"}
+      refute_received {:request, _}
+
+      # a rate-limited host-meta lookup must not be cached as "no LRDD template"
+      assert {:ok, nil} = Cachex.get(:host_meta_cache, "ratelimited.example")
+
+      assert {:error, :ratelimit} = WebFinger.finger("bob@ratelimited.example")
+      assert {:error, :ratelimit} = WebFinger.find_lrdd_template("ratelimited.example")
+      refute_received {:request, _}
+    end
+  end
 end

test/pleroma/workers/receiver_worker_test.exs

@@ -133,6 +133,28 @@ defmodule Pleroma.Workers.ReceiverWorkerTest do
     end
   end
 
+  describe "rate-limited fetches" do
+    setup do
+      Cachex.put(:http_backoff_cache, "ratelimited.example", true)
+
+      params =
+        insert(:note_activity).data
+        |> Map.put("actor", "https://ratelimited.example/users/alice")
+
+      %{args: %{"op" => "incoming_ap_doc", "params" => params}}
+    end
+
+    test "snooze while the actor's host is backed off", %{args: args} do
+      assert {:snooze, 300} = perform_job(ReceiverWorker, args)
+    end
+
+    test "stop snoozing two hours after the job was inserted", %{args: args} do
+      inserted_at = DateTime.add(DateTime.utc_now(), -121, :minute)
+
+      assert {:error, :ratelimit} = perform_job(ReceiverWorker, args, inserted_at: inserted_at)
+    end
+  end
+
   test "cancels due to origin containment" do
     params =
       insert(:note_activity).data

test/pleroma/workers/remote_fetcher_worker_test.exs

@@ -13,6 +13,7 @@ defmodule Pleroma.Workers.RemoteFetcherWorkerTest do
   @unauthorized_object "https://unauthorized.example.com/"
   @depth_object "https://depth.example.com/"
   @content_type_object "https://bad_content_type.example.com/"
+  @ratelimited_object "https://ratelimited.example.com/"
 
   describe "RemoteFetcherWorker" do
     setup do
@@ -37,6 +38,11 @@ defmodule Pleroma.Workers.RemoteFetcherWorkerTest do
             status: 200
           }
 
+        %{method: :get, url: @ratelimited_object} ->
+          %Tesla.Env{
+            status: 429
+          }
+
         %{method: :get, url: @content_type_object} ->
           %Tesla.Env{
             status: 200,
@@ -73,6 +79,14 @@ defmodule Pleroma.Workers.RemoteFetcherWorkerTest do
                })
     end
 
+    test "retries jobs for a rate-limited host" do
+      assert {:error, :ratelimit} =
+               perform_job(RemoteFetcherWorker, %{
+                 "op" => "fetch_remote",
+                 "id" => @ratelimited_object
+               })
+    end
+
     test "does not retry jobs for when object returns wrong content type" do
       assert {:cancel, _} =
                perform_job(RemoteFetcherWorker, %{

test/pleroma/workers/signature_retry_worker_test.exs

@@ -306,6 +306,24 @@ defmodule Pleroma.Workers.SignatureRetryWorkerTest do
     refute Activity.get_by_ap_id(create["id"])
   end
 
+  test "snoozes while the payload actor's host is backed off" do
+    Cachex.put(:http_backoff_cache, "ratelimited.example", true)
+    alice = build(:user, local: false, ap_id: "https://ratelimited.example/users/alice")
+
+    params = %{
+      "type" => "Create",
+      "actor" => alice.ap_id,
+      "id" => "https://ratelimited.example/activities/1",
+      "object" => "https://ratelimited.example/objects/1"
+    }
+
+    job = failed_signature_job(params, signature_headers_for(alice))
+    assert {:snooze, 300} = SignatureRetryWorker.perform(job)
+
+    old_job = %{job | inserted_at: DateTime.add(DateTime.utc_now(), -121, :minute)}
+    assert {:error, :ratelimit} = SignatureRetryWorker.perform(old_job)
+  end
+
   test "processes the activity after refetching a valid matching signature" do
     alice = insert(:user, local: false, ap_id: "https://one.com/users/alice")