Merge branch 'improved-reachability' into 'develop'

Reachability refactor

See merge request pleroma/pleroma!4366

changelog.d/reachability.change

@@ -0,0 +1 @@
+Improved the logic of how we determine if a server is unreachable.

config/config.exs

@@ -194,7 +194,6 @@ config :pleroma, :instance,
   account_approval_required: false,
   federating: true,
   federation_incoming_replies_max_depth: 100,
-  federation_reachability_timeout_days: 7,
   allow_relay: true,
   public: true,
   quarantined_instances: [],

lib/pleroma/instances.ex

@@ -15,25 +15,7 @@ defmodule Pleroma.Instances do
 
   defdelegate set_unreachable(url_or_host, unreachable_since \\ nil), to: Instance
 
-  defdelegate get_consistently_unreachable, to: Instance
+  defdelegate get_unreachable, to: Instance
-
-  def set_consistently_unreachable(url_or_host),
-    do: set_unreachable(url_or_host, reachability_datetime_threshold())
-
-  def reachability_datetime_threshold do
-    federation_reachability_timeout_days =
-      Pleroma.Config.get([:instance, :federation_reachability_timeout_days], 0)
-
-    if federation_reachability_timeout_days > 0 do
-      NaiveDateTime.add(
-        NaiveDateTime.utc_now(),
-        -federation_reachability_timeout_days * 24 * 3600,
-        :second
-      )
-    else
-      ~N[0000-01-01 00:00:00]
-    end
-  end
 
   def host(url_or_host) when is_binary(url_or_host) do
     if url_or_host =~ ~r/^http/i do
@@ -42,4 +24,21 @@ defmodule Pleroma.Instances do
       url_or_host
     end
   end
+
+  @doc "Schedules reachability checks for all unreachable instances"
+  def check_all_unreachable do
+    get_unreachable()
+    |> Enum.each(fn {domain, _} ->
+      Pleroma.Workers.ReachabilityWorker.new(%{"domain" => domain})
+      |> Oban.insert()
+    end)
+  end
+
+  @doc "Deletes all users and activities for unreachable instances"
+  def delete_all_unreachable do
+    get_unreachable()
+    |> Enum.each(fn {domain, _} ->
+      Instance.delete(domain)
+    end)
+  end
 end

lib/pleroma/instances/instance.ex

@@ -50,7 +50,7 @@ defmodule Pleroma.Instances.Instance do
     |> cast(params, [:software_name, :software_version, :software_repository])
   end
 
-  def filter_reachable([]), do: %{}
+  def filter_reachable([]), do: []
 
   def filter_reachable(urls_or_hosts) when is_list(urls_or_hosts) do
     hosts =
@@ -67,19 +67,15 @@ defmodule Pleroma.Instances.Instance do
       )
       |> Map.new(& &1)
 
-    reachability_datetime_threshold = Instances.reachability_datetime_threshold()
-
     for entry <- Enum.filter(urls_or_hosts, &is_binary/1) do
       host = host(entry)
       unreachable_since = unreachable_since_by_host[host]
 
-      if !unreachable_since ||
+      if is_nil(unreachable_since) do
-           NaiveDateTime.compare(unreachable_since, reachability_datetime_threshold) == :gt do
+        entry
-        {entry, unreachable_since}
       end
     end
     |> Enum.filter(& &1)
-    |> Map.new(& &1)
   end
 
   def reachable?(url_or_host) when is_binary(url_or_host) do
@@ -87,7 +83,7 @@ defmodule Pleroma.Instances.Instance do
       from(i in Instance,
         where:
           i.host == ^host(url_or_host) and
-            i.unreachable_since <= ^Instances.reachability_datetime_threshold(),
+            not is_nil(i.unreachable_since),
         select: true
       )
     )
@@ -96,9 +92,16 @@ defmodule Pleroma.Instances.Instance do
   def reachable?(url_or_host) when is_binary(url_or_host), do: true
 
   def set_reachable(url_or_host) when is_binary(url_or_host) do
-    %Instance{host: host(url_or_host)}
+    host = host(url_or_host)
-    |> changeset(%{unreachable_since: nil})
+
-    |> Repo.insert(on_conflict: {:replace, [:unreachable_since]}, conflict_target: :host)
+    result =
+      %Instance{host: host}
+      |> changeset(%{unreachable_since: nil})
+      |> Repo.insert(on_conflict: {:replace, [:unreachable_since]}, conflict_target: :host)
+
+    Pleroma.Workers.ReachabilityWorker.delete_jobs_for_host(host)
+
+    result
   end
 
   def set_reachable(_), do: {:error, nil}
@@ -131,11 +134,9 @@ defmodule Pleroma.Instances.Instance do
 
   def set_unreachable(_, _), do: {:error, nil}
 
-  def get_consistently_unreachable do
+  def get_unreachable do
-    reachability_datetime_threshold = Instances.reachability_datetime_threshold()
-
     from(i in Instance,
-      where: ^reachability_datetime_threshold > i.unreachable_since,
+      where: not is_nil(i.unreachable_since),
       order_by: i.unreachable_since,
       select: {i.host, i.unreachable_since}
     )
@@ -295,8 +296,14 @@ defmodule Pleroma.Instances.Instance do
   Deletes all users from an instance in a background task, thus also deleting
   all of those users' activities and notifications.
   """
-  def delete_users_and_activities(host) when is_binary(host) do
+  def delete(host) when is_binary(host) do
     DeleteWorker.new(%{"op" => "delete_instance", "host" => host})
     |> Oban.insert()
   end
+
+  @doc "Schedules reachability check for instance"
+  def check_unreachable(domain) when is_binary(domain) do
+    Pleroma.Workers.ReachabilityWorker.new(%{"domain" => domain})
+    |> Oban.insert()
+  end
 end

lib/pleroma/object/fetcher.ex

@@ -4,7 +4,6 @@
 
 defmodule Pleroma.Object.Fetcher do
   alias Pleroma.HTTP
-  alias Pleroma.Instances
   alias Pleroma.Maps
   alias Pleroma.Object
   alias Pleroma.Object.Containment
@@ -150,10 +149,6 @@ defmodule Pleroma.Object.Fetcher do
          {:ok, body} <- get_object(id),
          {:ok, data} <- safe_json_decode(body),
          :ok <- Containment.contain_origin_from_id(id, data) do
-      if not Instances.reachable?(id) do
-        Instances.set_reachable(id)
-      end
-
       {:ok, data}
     else
       {:scheme, _} ->

lib/pleroma/web/activity_pub/activity_pub_controller.ex

@@ -53,7 +53,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
   )
 
   plug(:log_inbox_metadata when action in [:inbox])
-  plug(:set_requester_reachable when action in [:inbox])
   plug(:relay_active? when action in [:relay])
 
   defp relay_active?(conn, _) do
@@ -520,15 +519,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
     |> json(dgettext("errors", "error"))
   end
 
-  defp set_requester_reachable(%Plug.Conn{} = conn, _) do
-    with actor <- conn.params["actor"],
-         true <- is_binary(actor) do
-      Pleroma.Instances.set_reachable(actor)
-    end
-
-    conn
-  end
-
   defp log_inbox_metadata(%{params: %{"actor" => actor, "type" => type}} = conn, _) do
     Logger.metadata(actor: actor, type: type)
     conn

lib/pleroma/web/activity_pub/publisher.ex

@@ -161,17 +161,9 @@ defmodule Pleroma.Web.ActivityPub.Publisher do
                {"digest", p.digest}
              ]
            ) do
-      if not is_nil(p.unreachable_since) do
-        Instances.set_reachable(p.inbox)
-      end
-
       result
     else
       {_post_result, %{status: code} = response} = e ->
-        if is_nil(p.unreachable_since) do
-          Instances.set_unreachable(p.inbox)
-        end
-
         Logger.metadata(activity: p.activity_id, inbox: p.inbox, status: code)
         Logger.error("Publisher failed to inbox #{p.inbox} with status #{code}")
 
@@ -192,10 +184,6 @@ defmodule Pleroma.Web.ActivityPub.Publisher do
         connection_pool_snooze()
 
       e ->
-        if is_nil(p.unreachable_since) do
-          Instances.set_unreachable(p.inbox)
-        end
-
         Logger.metadata(activity: p.activity_id, inbox: p.inbox)
         Logger.error("Publisher failed to inbox #{p.inbox} #{inspect(e)}")
         {:error, e}
@@ -307,7 +295,7 @@ defmodule Pleroma.Web.ActivityPub.Publisher do
 
     [priority_recipients, recipients] = recipients(actor, activity)
 
-    inboxes =
+    [priority_inboxes, other_inboxes] =
       [priority_recipients, recipients]
       |> Enum.map(fn recipients ->
         recipients
@@ -320,8 +308,8 @@ defmodule Pleroma.Web.ActivityPub.Publisher do
       end)
 
     Repo.checkout(fn ->
-      Enum.each(inboxes, fn inboxes ->
+      Enum.each([priority_inboxes, other_inboxes], fn inboxes ->
-        Enum.each(inboxes, fn {inbox, unreachable_since} ->
+        Enum.each(inboxes, fn inbox ->
           %User{ap_id: ap_id} = Enum.find(recipients, fn actor -> actor.inbox == inbox end)
 
           # Get all the recipients on the same host and add them to cc. Otherwise, a remote
@@ -331,8 +319,7 @@ defmodule Pleroma.Web.ActivityPub.Publisher do
           __MODULE__.enqueue_one(%{
             inbox: inbox,
             cc: cc,
-            activity_id: activity.id,
+            activity_id: activity.id
-            unreachable_since: unreachable_since
           })
         end)
       end)
@@ -365,12 +352,11 @@ defmodule Pleroma.Web.ActivityPub.Publisher do
     |> Enum.each(fn {inboxes, priority} ->
       inboxes
       |> Instances.filter_reachable()
-      |> Enum.each(fn {inbox, unreachable_since} ->
+      |> Enum.each(fn inbox ->
         __MODULE__.enqueue_one(
           %{
             inbox: inbox,
-            activity_id: activity.id,
+            activity_id: activity.id
-            unreachable_since: unreachable_since
           },
           priority: priority
         )

lib/pleroma/web/admin_api/controllers/instance_controller.ex

@@ -49,7 +49,7 @@ defmodule Pleroma.Web.AdminAPI.InstanceController do
   end
 
   def delete(conn, %{"instance" => instance}) do
-    with {:ok, _job} <- Instance.delete_users_and_activities(instance) do
+    with {:ok, _job} <- Instance.delete(instance) do
       json(conn, instance)
     end
   end

lib/pleroma/web/pleroma_api/controllers/instances_controller.ex

@@ -13,7 +13,7 @@ defmodule Pleroma.Web.PleromaAPI.InstancesController do
 
   def show(conn, _params) do
     unreachable =
-      Instances.get_consistently_unreachable()
+      Instances.get_unreachable()
       |> Map.new(fn {host, date} -> {host, to_string(date)} end)
 
     json(conn, %{"unreachable" => unreachable})

lib/pleroma/workers/delete_worker.ex

@@ -14,6 +14,7 @@ defmodule Pleroma.Workers.DeleteWorker do
   end
 
   def perform(%Job{args: %{"op" => "delete_instance", "host" => host}}) do
+    # Schedule the per-user deletion jobs
     Pleroma.Repo.transaction(fn ->
       User.Query.build(%{nickname: "@#{host}"})
       |> Pleroma.Repo.all()
@@ -22,6 +23,17 @@ defmodule Pleroma.Workers.DeleteWorker do
         |> __MODULE__.new()
         |> Oban.insert()
       end)
+
+      # Delete the instance from the Instances table
+      case Pleroma.Repo.get_by(Pleroma.Instances.Instance, host: host) do
+        nil -> :ok
+        instance -> Pleroma.Repo.delete(instance)
+      end
+
+      # Delete any pending ReachabilityWorker jobs for this domain
+      Pleroma.Workers.ReachabilityWorker.delete_jobs_for_host(host)
+
+      :ok
     end)
   end
 

lib/pleroma/workers/publisher_worker.ex

@@ -4,9 +4,10 @@
 
 defmodule Pleroma.Workers.PublisherWorker do
   alias Pleroma.Activity
+  alias Pleroma.Instances
   alias Pleroma.Web.Federator
 
-  use Oban.Worker, queue: :federator_outgoing, max_attempts: 5
+  use Oban.Worker, queue: :federator_outgoing, max_attempts: 13
 
   @impl true
   def perform(%Job{args: %{"op" => "publish", "activity_id" => activity_id}}) do
@@ -14,9 +15,30 @@ defmodule Pleroma.Workers.PublisherWorker do
     Federator.perform(:publish, activity)
   end
 
-  def perform(%Job{args: %{"op" => "publish_one", "params" => params}}) do
+  def perform(%Job{args: %{"op" => "publish_one", "params" => params}} = job) do
     params = Map.new(params, fn {k, v} -> {String.to_atom(k), v} end)
-    Federator.perform(:publish_one, params)
+
+    # Cancel / skip the job if this server believed to be unreachable now
+    if not Instances.reachable?(params.inbox) do
+      {:cancel, :unreachable}
+    else
+      case Federator.perform(:publish_one, params) do
+        {:ok, _} ->
+          :ok
+
+        {:error, _} = error ->
+          # Only mark as unreachable on final failure
+          if job.attempt == job.max_attempts do
+            Instances.set_unreachable(params.inbox)
+          end
+
+          error
+
+        error ->
+          # Unexpected error, may have been client side
+          error
+      end
+    end
   end
 
   @impl true

lib/pleroma/workers/reachability_worker.ex

@@ -0,0 +1,116 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Workers.ReachabilityWorker do
+  use Oban.Worker,
+    queue: :background,
+    max_attempts: 1,
+    unique: [period: :infinity, states: [:available, :scheduled], keys: [:domain]]
+
+  alias Pleroma.HTTP
+  alias Pleroma.Instances
+
+  import Ecto.Query
+
+  @impl true
+  def perform(%Oban.Job{args: %{"domain" => domain, "phase" => phase, "attempt" => attempt}}) do
+    case check_reachability(domain) do
+      :ok ->
+        Instances.set_reachable("https://#{domain}")
+        :ok
+
+      {:error, _} = error ->
+        handle_failed_attempt(domain, phase, attempt)
+        error
+    end
+  end
+
+  # New jobs enter here and are immediately re-scheduled for the first phase
+  @impl true
+  def perform(%Oban.Job{args: %{"domain" => domain}}) do
+    scheduled_at = DateTime.add(DateTime.utc_now(), 60, :second)
+
+    %{
+      "domain" => domain,
+      "phase" => "phase_1min",
+      "attempt" => 1
+    }
+    |> new(scheduled_at: scheduled_at, replace: true)
+    |> Oban.insert()
+
+    :ok
+  end
+
+  @impl true
+  def timeout(_job), do: :timer.seconds(5)
+
+  @doc "Deletes scheduled jobs to check reachability for specified instance"
+  def delete_jobs_for_host(host) do
+    Oban.Job
+    |> where(worker: "Pleroma.Workers.ReachabilityWorker")
+    |> where([j], j.args["domain"] == ^host)
+    |> Oban.delete_all_jobs()
+  end
+
+  defp check_reachability(domain) do
+    case HTTP.get("https://#{domain}/") do
+      {:ok, %{status: status}} when status in 200..299 ->
+        :ok
+
+      {:ok, %{status: _status}} ->
+        {:error, :unreachable}
+
+      {:error, _} = error ->
+        error
+    end
+  end
+
+  defp handle_failed_attempt(_domain, "final", _attempt), do: :ok
+
+  defp handle_failed_attempt(domain, phase, attempt) do
+    {interval_minutes, max_attempts, next_phase} = get_phase_config(phase)
+
+    if attempt >= max_attempts do
+      # Move to next phase
+      schedule_next_phase(domain, next_phase)
+    else
+      # Retry same phase with incremented attempt
+      schedule_retry(domain, phase, attempt + 1, interval_minutes)
+    end
+  end
+
+  defp get_phase_config("phase_1min"), do: {1, 4, "phase_15min"}
+  defp get_phase_config("phase_15min"), do: {15, 4, "phase_1hour"}
+  defp get_phase_config("phase_1hour"), do: {60, 4, "phase_8hour"}
+  defp get_phase_config("phase_8hour"), do: {480, 4, "phase_24hour"}
+  defp get_phase_config("phase_24hour"), do: {1440, 4, "final"}
+  defp get_phase_config("final"), do: {nil, 0, nil}
+
+  defp schedule_next_phase(_domain, "final"), do: :ok
+
+  defp schedule_next_phase(domain, next_phase) do
+    {interval_minutes, _max_attempts, _next_phase} = get_phase_config(next_phase)
+    scheduled_at = DateTime.add(DateTime.utc_now(), interval_minutes * 60, :second)
+
+    %{
+      "domain" => domain,
+      "phase" => next_phase,
+      "attempt" => 1
+    }
+    |> new(scheduled_at: scheduled_at, replace: true)
+    |> Oban.insert()
+  end
+
+  def schedule_retry(domain, phase, attempt, interval_minutes) do
+    scheduled_at = DateTime.add(DateTime.utc_now(), interval_minutes * 60, :second)
+
+    %{
+      "domain" => domain,
+      "phase" => phase,
+      "attempt" => attempt
+    }
+    |> new(scheduled_at: scheduled_at, replace: true)
+    |> Oban.insert()
+  end
+end

lib/pleroma/workers/receiver_worker.ex

@@ -3,6 +3,7 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Workers.ReceiverWorker do
+  alias Pleroma.Instances
   alias Pleroma.Signature
   alias Pleroma.User
   alias Pleroma.Web.Federator
@@ -37,6 +38,11 @@ defmodule Pleroma.Workers.ReceiverWorker do
          {:ok, _public_key} <- Signature.refetch_public_key(conn_data),
          {:signature, true} <- {:signature, Signature.validate_signature(conn_data)},
          {:ok, res} <- Federator.perform(:incoming_ap_doc, params) do
+      unless Instances.reachable?(params["actor"]) do
+        domain = URI.parse(params["actor"]).host
+        Oban.insert(Pleroma.Workers.ReachabilityWorker.new(%{"domain" => domain}))
+      end
+
       {:ok, res}
     else
       e -> process_errors(e)
@@ -45,6 +51,11 @@ defmodule Pleroma.Workers.ReceiverWorker do
 
   def perform(%Job{args: %{"op" => "incoming_ap_doc", "params" => params}}) do
     with {:ok, res} <- Federator.perform(:incoming_ap_doc, params) do
+      unless Instances.reachable?(params["actor"]) do
+        domain = URI.parse(params["actor"]).host
+        Oban.insert(Pleroma.Workers.ReachabilityWorker.new(%{"domain" => domain}))
+      end
+
       {:ok, res}
     else
       e -> process_errors(e)

lib/pleroma/workers/remote_fetcher_worker.ex

@@ -3,6 +3,7 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Workers.RemoteFetcherWorker do
+  alias Pleroma.Instances
   alias Pleroma.Object.Fetcher
 
   use Oban.Worker, queue: :background, unique: [period: :infinity]
@@ -11,6 +12,11 @@ defmodule Pleroma.Workers.RemoteFetcherWorker do
   def perform(%Job{args: %{"op" => "fetch_remote", "id" => id} = args}) do
     case Fetcher.fetch_object_from_id(id, depth: args["depth"]) do
       {:ok, _object} ->
+        unless Instances.reachable?(id) do
+          # Mark the server as reachable since we successfully fetched an object
+          Instances.set_reachable(id)
+        end
+
         :ok
 
       {:allowed_depth, false} ->

mix.exs

@@ -136,7 +136,7 @@ defmodule Pleroma.Mixfile do
       {:telemetry_poller, "~> 1.0"},
       {:tzdata, "~> 1.0.3"},
       {:plug_cowboy, "~> 2.5"},
-      {:oban, "~> 2.18.0"},
+      {:oban, "~> 2.19.0"},
       {:gettext, "~> 0.20"},
       {:bcrypt_elixir, "~> 2.2"},
       {:trailing_format_plug, "~> 0.0.7"},

mix.lock

@@ -26,8 +26,8 @@
   "credo": {:hex, :credo, "1.7.12", "9e3c20463de4b5f3f23721527fcaf16722ec815e70ff6c60b86412c695d426c1", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "8493d45c656c5427d9c729235b99d498bd133421f3e0a683e5c1b561471291e5"},
   "crontab": {:hex, :crontab, "1.1.8", "2ce0e74777dfcadb28a1debbea707e58b879e6aa0ffbf9c9bb540887bce43617", [:mix], [{:ecto, "~> 1.0 or ~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm"},
   "custom_base": {:hex, :custom_base, "0.2.1", "4a832a42ea0552299d81652aa0b1f775d462175293e99dfbe4d7dbaab785a706", [:mix], [], "hexpm", "8df019facc5ec9603e94f7270f1ac73ddf339f56ade76a721eaa57c1493ba463"},
-  "db_connection": {:hex, :db_connection, "2.7.0", "b99faa9291bb09892c7da373bb82cba59aefa9b36300f6145c5f201c7adf48ec", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "dcf08f31b2701f857dfc787fbad78223d61a32204f217f15e881dd93e4bdd3ff"},
+  "db_connection": {:hex, :db_connection, "2.8.0", "64fd82cfa6d8e25ec6660cea73e92a4cbc6a18b31343910427b702838c4b33b2", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "008399dae5eee1bf5caa6e86d204dcb44242c82b1ed5e22c881f2c34da201b15"},
-  "decimal": {:hex, :decimal, "2.1.1", "5611dca5d4b2c3dd497dec8f68751f1f1a54755e8ed2a966c2633cf885973ad6", [:mix], [], "hexpm", "53cfe5f497ed0e7771ae1a475575603d77425099ba5faef9394932b35020ffcc"},
+  "decimal": {:hex, :decimal, "2.3.0", "3ad6255aa77b4a3c4f818171b12d237500e63525c2fd056699967a3e7ea20f62", [:mix], [], "hexpm", "a4d66355cb29cb47c3cf30e71329e58361cfcb37c34235ef3bf1d7bf3773aeac"},
   "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm", "ce708e5f094b9cd4e8f2be4f00d2f4250c4095be93f8cd6d018c753894885430"},
   "dialyxir": {:hex, :dialyxir, "1.4.3", "edd0124f358f0b9e95bfe53a9fcf806d615d8f838e2202a9f430d59566b6b53b", [:mix], [{:erlex, ">= 0.2.6", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "bf2cfb75cd5c5006bec30141b131663299c661a864ec7fbbc72dfa557487a986"},
   "earmark": {:hex, :earmark, "1.4.46", "8c7287bd3137e99d26ae4643e5b7ef2129a260e3dcf41f251750cb4563c8fb81", [:mix], [], "hexpm", "798d86db3d79964e759ddc0c077d5eb254968ed426399fbf5a62de2b5ff8910a"},
@@ -92,7 +92,7 @@
   "nimble_parsec": {:hex, :nimble_parsec, "0.6.0", "32111b3bf39137144abd7ba1cce0914533b2d16ef35e8abc5ec8be6122944263", [:mix], [], "hexpm", "27eac315a94909d4dc68bc07a4a83e06c8379237c5ea528a9acff4ca1c873c52"},
   "nimble_pool": {:hex, :nimble_pool, "0.2.6", "91f2f4c357da4c4a0a548286c84a3a28004f68f05609b4534526871a22053cde", [:mix], [], "hexpm", "1c715055095d3f2705c4e236c18b618420a35490da94149ff8b580a2144f653f"},
   "nodex": {:git, "https://git.pleroma.social/pleroma/nodex", "cb6730f943cfc6aad674c92161be23a8411f15d1", [ref: "cb6730f943cfc6aad674c92161be23a8411f15d1"]},
-  "oban": {:hex, :oban, "2.18.3", "1608c04f8856c108555c379f2f56bc0759149d35fa9d3b825cb8a6769f8ae926", [:mix], [{:ecto_sql, "~> 3.10", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:ecto_sqlite3, "~> 0.9", [hex: :ecto_sqlite3, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.16", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "36ca6ca84ef6518f9c2c759ea88efd438a3c81d667ba23b02b062a0aa785475e"},
+  "oban": {:hex, :oban, "2.19.4", "045adb10db1161dceb75c254782f97cdc6596e7044af456a59decb6d06da73c1", [:mix], [{:ecto_sql, "~> 3.10", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:ecto_sqlite3, "~> 0.9", [hex: :ecto_sqlite3, repo: "hexpm", optional: true]}, {:igniter, "~> 0.5", [hex: :igniter, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: true]}, {:myxql, "~> 0.7", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.16", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "5fcc6219e6464525b808d97add17896e724131f498444a292071bf8991c99f97"},
   "oban_live_dashboard": {:hex, :oban_live_dashboard, "0.1.1", "8aa4ceaf381c818f7d5c8185cc59942b8ac82ef0cf559881aacf8d3f8ac7bdd3", [:mix], [{:oban, "~> 2.15", [hex: :oban, repo: "hexpm", optional: false]}, {:phoenix_live_dashboard, "~> 0.7", [hex: :phoenix_live_dashboard, repo: "hexpm", optional: false]}], "hexpm", "16dc4ce9c9a95aa2e655e35ed4e675652994a8def61731a18af85e230e1caa63"},
   "octo_fetch": {:hex, :octo_fetch, "0.4.0", "074b5ecbc08be10b05b27e9db08bc20a3060142769436242702931c418695b19", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~> 1.1", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm", "cf8be6f40cd519d7000bb4e84adcf661c32e59369ca2827c4e20042eda7a7fc6"},
   "open_api_spex": {:hex, :open_api_spex, "3.18.2", "8c855e83bfe8bf81603d919d6e892541eafece3720f34d1700b58024dadde247", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:poison, "~> 3.0 or ~> 4.0 or ~> 5.0", [hex: :poison, repo: "hexpm", optional: true]}, {:ymlr, "~> 2.0 or ~> 3.0 or ~> 4.0", [hex: :ymlr, repo: "hexpm", optional: true]}], "hexpm", "aa3e6dcfc0ad6a02596b2172662da21c9dd848dac145ea9e603f54e3d81b8d2b"},

test/pleroma/instances/instance_test.exs

@@ -3,7 +3,6 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Instances.InstanceTest do
-  alias Pleroma.Instances
   alias Pleroma.Instances.Instance
   alias Pleroma.Repo
 
@@ -13,8 +12,6 @@ defmodule Pleroma.Instances.InstanceTest do
   import ExUnit.CaptureLog
   import Pleroma.Factory
 
-  setup_all do: clear_config([:instance, :federation_reachability_timeout_days], 1)
-
   describe "set_reachable/1" do
     test "clears `unreachable_since` of existing matching Instance record having non-nil `unreachable_since`" do
       unreachable_since = NaiveDateTime.to_iso8601(NaiveDateTime.utc_now())
@@ -30,6 +27,32 @@ defmodule Pleroma.Instances.InstanceTest do
       assert {:ok, instance} = Instance.set_reachable(instance.host)
       refute instance.unreachable_since
     end
+
+    test "cancels all ReachabilityWorker jobs for the domain" do
+      domain = "cancelme.example.org"
+      insert(:instance, host: domain, unreachable_since: NaiveDateTime.utc_now())
+
+      # Insert a ReachabilityWorker job for this domain, scheduled 5 minutes in the future
+      scheduled_at = DateTime.add(DateTime.utc_now(), 300, :second)
+
+      {:ok, job} =
+        Pleroma.Workers.ReachabilityWorker.new(
+          %{"domain" => domain, "phase" => "phase_1min", "attempt" => 1},
+          scheduled_at: scheduled_at
+        )
+        |> Oban.insert()
+
+      # Ensure the job is present
+      job = Pleroma.Repo.get(Oban.Job, job.id)
+      assert job
+
+      # Call set_reachable, which should delete the job
+      assert {:ok, _} = Instance.set_reachable(domain)
+
+      # Reload the job and assert it is deleted
+      job = Pleroma.Repo.get(Oban.Job, job.id)
+      refute job
+    end
   end
 
   describe "set_unreachable/1" do
@@ -144,7 +167,11 @@ defmodule Pleroma.Instances.InstanceTest do
     end
 
     test "Doesn't scrapes unreachable instances" do
-      instance = insert(:instance, unreachable_since: Instances.reachability_datetime_threshold())
+      instance =
+        insert(:instance,
+          unreachable_since: NaiveDateTime.utc_now() |> NaiveDateTime.add(-:timer.hours(24))
+        )
+
       url = "https://" <> instance.host
 
       assert capture_log(fn -> assert nil == Instance.get_or_update_favicon(URI.parse(url)) end) =~
@@ -212,14 +239,44 @@ defmodule Pleroma.Instances.InstanceTest do
     end
   end
 
-  test "delete_users_and_activities/1 schedules a job to delete the instance and users" do
+  test "delete/1 schedules a job to delete the instance and users" do
     insert(:user, nickname: "mario@mushroom.kingdom", name: "Mario")
 
-    {:ok, _job} = Instance.delete_users_and_activities("mushroom.kingdom")
+    {:ok, _job} = Instance.delete("mushroom.kingdom")
 
     assert_enqueued(
       worker: Pleroma.Workers.DeleteWorker,
       args: %{"op" => "delete_instance", "host" => "mushroom.kingdom"}
     )
   end
+
+  describe "check_unreachable/1" do
+    test "schedules a ReachabilityWorker job for the given domain" do
+      domain = "test.example.com"
+
+      # Call check_unreachable
+      assert {:ok, _job} = Instance.check_unreachable(domain)
+
+      # Verify that a ReachabilityWorker job was scheduled
+      jobs = all_enqueued(worker: Pleroma.Workers.ReachabilityWorker)
+      assert length(jobs) == 1
+      [job] = jobs
+      assert job.args["domain"] == domain
+    end
+
+    test "handles multiple calls for the same domain (uniqueness enforced)" do
+      domain = "duplicate.example.com"
+
+      assert {:ok, _job1} = Instance.check_unreachable(domain)
+
+      # Second call for the same domain
+      assert {:ok, %Oban.Job{conflict?: true}} = Instance.check_unreachable(domain)
+
+      # Should only have one job due to uniqueness
+      jobs = all_enqueued(worker: Pleroma.Workers.ReachabilityWorker)
+      assert length(jobs) == 1
+      [job] = jobs
+      assert job.args["domain"] == domain
+    end
+  end
 end

test/pleroma/instances_test.exs

@@ -6,74 +6,42 @@ defmodule Pleroma.InstancesTest do
   alias Pleroma.Instances
 
   use Pleroma.DataCase
-
+  use Oban.Testing, repo: Pleroma.Repo
-  setup_all do: clear_config([:instance, :federation_reachability_timeout_days], 1)
 
   describe "reachable?/1" do
     test "returns `true` for host / url with unknown reachability status" do
       assert Instances.reachable?("unknown.site")
       assert Instances.reachable?("http://unknown.site")
     end
-
-    test "returns `false` for host / url marked unreachable for at least `reachability_datetime_threshold()`" do
-      host = "consistently-unreachable.name"
-      Instances.set_consistently_unreachable(host)
-
-      refute Instances.reachable?(host)
-      refute Instances.reachable?("http://#{host}/path")
-    end
-
-    test "returns `true` for host / url marked unreachable for less than `reachability_datetime_threshold()`" do
-      url = "http://eventually-unreachable.name/path"
-
-      Instances.set_unreachable(url)
-
-      assert Instances.reachable?(url)
-      assert Instances.reachable?(URI.parse(url).host)
-    end
-
-    test "raises FunctionClauseError exception on non-binary input" do
-      assert_raise FunctionClauseError, fn -> Instances.reachable?(nil) end
-      assert_raise FunctionClauseError, fn -> Instances.reachable?(1) end
-    end
   end
 
   describe "filter_reachable/1" do
     setup do
-      host = "consistently-unreachable.name"
+      unreachable_host = "consistently-unreachable.name"
-      url1 = "http://eventually-unreachable.com/path"
+      reachable_host = "http://domain.com/path"
-      url2 = "http://domain.com/path"
 
-      Instances.set_consistently_unreachable(host)
+      Instances.set_unreachable(unreachable_host)
-      Instances.set_unreachable(url1)
 
-      result = Instances.filter_reachable([host, url1, url2, nil])
+      result = Instances.filter_reachable([unreachable_host, reachable_host, nil])
-      %{result: result, url1: url1, url2: url2}
+      %{result: result, reachable_host: reachable_host, unreachable_host: unreachable_host}
     end
 
-    test "returns a map with keys containing 'not marked consistently unreachable' elements of supplied list",
+    test "returns a list of only reachable elements",
-         %{result: result, url1: url1, url2: url2} do
+         %{result: result, reachable_host: reachable_host} do
-      assert is_map(result)
+      assert is_list(result)
-      assert Enum.sort([url1, url2]) == result |> Map.keys() |> Enum.sort()
+      assert [reachable_host] == result
     end
 
-    test "returns a map with `unreachable_since` values for keys",
+    test "returns an empty list when provided no data" do
-         %{result: result, url1: url1, url2: url2} do
+      assert [] == Instances.filter_reachable([])
-      assert is_map(result)
+      assert [] == Instances.filter_reachable([nil])
-      assert %NaiveDateTime{} = result[url1]
-      assert is_nil(result[url2])
-    end
-
-    test "returns an empty map for empty list or list containing no hosts / url" do
-      assert %{} == Instances.filter_reachable([])
-      assert %{} == Instances.filter_reachable([nil])
     end
   end
 
   describe "set_reachable/1" do
     test "sets unreachable url or host reachable" do
       host = "domain.com"
-      Instances.set_consistently_unreachable(host)
+      Instances.set_unreachable(host)
       refute Instances.reachable?(host)
 
       Instances.set_reachable(host)
@@ -103,22 +71,68 @@ defmodule Pleroma.InstancesTest do
     end
   end
 
-  describe "set_consistently_unreachable/1" do
+  describe "check_all_unreachable/0" do
-    test "sets reachable url or host unreachable" do
+    test "schedules ReachabilityWorker jobs for all unreachable instances" do
-      url = "http://domain.com?q="
+      domain1 = "unreachable1.example.com"
-      assert Instances.reachable?(url)
+      domain2 = "unreachable2.example.com"
+      domain3 = "unreachable3.example.com"
 
-      Instances.set_consistently_unreachable(url)
+      Instances.set_unreachable(domain1)
-      refute Instances.reachable?(url)
+      Instances.set_unreachable(domain2)
+      Instances.set_unreachable(domain3)
+
+      Instances.check_all_unreachable()
+
+      # Verify that ReachabilityWorker jobs were scheduled for all unreachable domains
+      jobs = all_enqueued(worker: Pleroma.Workers.ReachabilityWorker)
+      assert length(jobs) == 3
+
+      domains = Enum.map(jobs, & &1.args["domain"])
+      assert domain1 in domains
+      assert domain2 in domains
+      assert domain3 in domains
     end
 
-    test "keeps unreachable url or host unreachable" do
+    test "does not schedule jobs for reachable instances" do
-      host = "site.name"
+      unreachable_domain = "unreachable.example.com"
-      Instances.set_consistently_unreachable(host)
+      reachable_domain = "reachable.example.com"
-      refute Instances.reachable?(host)
 
-      Instances.set_consistently_unreachable(host)
+      Instances.set_unreachable(unreachable_domain)
-      refute Instances.reachable?(host)
+      Instances.set_reachable(reachable_domain)
+
+      Instances.check_all_unreachable()
+
+      # Verify that only one job was scheduled (for the unreachable domain)
+      jobs = all_enqueued(worker: Pleroma.Workers.ReachabilityWorker)
+      assert length(jobs) == 1
+      [job] = jobs
+      assert job.args["domain"] == unreachable_domain
     end
   end
+
+  test "delete_all_unreachable/0 schedules DeleteWorker jobs for all unreachable instances" do
+    domain1 = "unreachable1.example.com"
+    domain2 = "unreachable2.example.com"
+    domain3 = "unreachable3.example.com"
+
+    Instances.set_unreachable(domain1)
+    Instances.set_unreachable(domain2)
+    Instances.set_unreachable(domain3)
+
+    Instances.delete_all_unreachable()
+
+    # Verify that DeleteWorker jobs were scheduled for all unreachable domains
+    jobs = all_enqueued(worker: Pleroma.Workers.DeleteWorker)
+    assert length(jobs) == 3
+
+    domains = Enum.map(jobs, & &1.args["host"])
+    assert domain1 in domains
+    assert domain2 in domains
+    assert domain3 in domains
+
+    # Verify all jobs are delete_instance operations
+    Enum.each(jobs, fn job ->
+      assert job.args["op"] == "delete_instance"
+    end)
+  end
 end

test/pleroma/object/fetcher_test.exs

@@ -6,7 +6,6 @@ defmodule Pleroma.Object.FetcherTest do
   use Pleroma.DataCase
 
   alias Pleroma.Activity
-  alias Pleroma.Instances
   alias Pleroma.Object
   alias Pleroma.Object.Fetcher
   alias Pleroma.Web.ActivityPub.ObjectValidator
@@ -250,17 +249,6 @@ defmodule Pleroma.Object.FetcherTest do
       result = Fetcher.fetch_object_from_id("https://example.com/objects/no-content-type")
       assert {:fetch, {:error, nil}} = result
     end
-
-    test "it resets instance reachability on successful fetch" do
-      id = "http://mastodon.example.org/@admin/99541947525187367"
-      Instances.set_consistently_unreachable(id)
-      refute Instances.reachable?(id)
-
-      {:ok, _object} =
-        Fetcher.fetch_object_from_id("http://mastodon.example.org/@admin/99541947525187367")
-
-      assert Instances.reachable?(id)
-    end
   end
 
   describe "implementation quirks" do

test/pleroma/web/activity_pub/activity_pub_controller_test.exs

@@ -8,7 +8,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do
 
   alias Pleroma.Activity
   alias Pleroma.Delivery
-  alias Pleroma.Instances
   alias Pleroma.Object
   alias Pleroma.Tests.ObanHelpers
   alias Pleroma.User
@@ -601,23 +600,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do
       assert Activity.get_by_ap_id(data["id"])
     end
 
-    test "it clears `unreachable` federation status of the sender", %{conn: conn} do
-      data = File.read!("test/fixtures/mastodon-post-activity.json") |> Jason.decode!()
-
-      sender_url = data["actor"]
-      Instances.set_consistently_unreachable(sender_url)
-      refute Instances.reachable?(sender_url)
-
-      conn =
-        conn
-        |> assign(:valid_signature, true)
-        |> put_req_header("content-type", "application/activity+json")
-        |> post("/inbox", data)
-
-      assert "ok" == json_response(conn, 200)
-      assert Instances.reachable?(sender_url)
-    end
-
     test "accept follow activity", %{conn: conn} do
       clear_config([:instance, :federating], true)
       relay = Relay.get_actor()
@@ -1108,24 +1090,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do
       assert response(conn, 200) =~ note_object.data["content"]
     end
 
-    test "it clears `unreachable` federation status of the sender", %{conn: conn, data: data} do
-      user = insert(:user)
-      data = Map.put(data, "bcc", [user.ap_id])
-
-      sender_host = URI.parse(data["actor"]).host
-      Instances.set_consistently_unreachable(sender_host)
-      refute Instances.reachable?(sender_host)
-
-      conn =
-        conn
-        |> assign(:valid_signature, true)
-        |> put_req_header("content-type", "application/activity+json")
-        |> post("/users/#{user.nickname}/inbox", data)
-
-      assert "ok" == json_response(conn, 200)
-      assert Instances.reachable?(sender_host)
-    end
-
     test "it removes all follower collections but actor's", %{conn: conn} do
       [actor, recipient] = insert_pair(:user)
 

test/pleroma/web/activity_pub/publisher_test.exs

@@ -6,13 +6,11 @@ defmodule Pleroma.Web.ActivityPub.PublisherTest do
   use Oban.Testing, repo: Pleroma.Repo
   use Pleroma.Web.ConnCase
 
-  import ExUnit.CaptureLog
   import Pleroma.Factory
   import Tesla.Mock
   import Mock
 
   alias Pleroma.Activity
-  alias Pleroma.Instances
   alias Pleroma.Object
   alias Pleroma.Tests.ObanHelpers
   alias Pleroma.Web.ActivityPub.Publisher
@@ -167,115 +165,6 @@ defmodule Pleroma.Web.ActivityPub.PublisherTest do
                })
                |> Publisher.publish_one()
     end
-
-    test_with_mock "calls `Instances.set_reachable` on successful federation if `unreachable_since` is set",
-                   Instances,
-                   [:passthrough],
-                   [] do
-      _actor = insert(:user)
-      inbox = "http://200.site/users/nick1/inbox"
-      activity = insert(:note_activity)
-
-      assert {:ok, _} =
-               Publisher.prepare_one(%{
-                 inbox: inbox,
-                 activity_id: activity.id,
-                 unreachable_since: NaiveDateTime.utc_now() |> NaiveDateTime.to_string()
-               })
-               |> Publisher.publish_one()
-
-      assert called(Instances.set_reachable(inbox))
-    end
-
-    test_with_mock "does NOT call `Instances.set_reachable` on successful federation if `unreachable_since` is nil",
-                   Instances,
-                   [:passthrough],
-                   [] do
-      _actor = insert(:user)
-      inbox = "http://200.site/users/nick1/inbox"
-      activity = insert(:note_activity)
-
-      assert {:ok, _} =
-               Publisher.prepare_one(%{
-                 inbox: inbox,
-                 activity_id: activity.id,
-                 unreachable_since: nil
-               })
-               |> Publisher.publish_one()
-
-      refute called(Instances.set_reachable(inbox))
-    end
-
-    test_with_mock "calls `Instances.set_unreachable` on target inbox on non-2xx HTTP response code",
-                   Instances,
-                   [:passthrough],
-                   [] do
-      _actor = insert(:user)
-      inbox = "http://404.site/users/nick1/inbox"
-      activity = insert(:note_activity)
-
-      assert {:cancel, _} =
-               Publisher.prepare_one(%{inbox: inbox, activity_id: activity.id})
-               |> Publisher.publish_one()
-
-      assert called(Instances.set_unreachable(inbox))
-    end
-
-    test_with_mock "it calls `Instances.set_unreachable` on target inbox on request error of any kind",
-                   Instances,
-                   [:passthrough],
-                   [] do
-      _actor = insert(:user)
-      inbox = "http://connrefused.site/users/nick1/inbox"
-      activity = insert(:note_activity)
-
-      assert capture_log(fn ->
-               assert {:error, _} =
-                        Publisher.prepare_one(%{
-                          inbox: inbox,
-                          activity_id: activity.id
-                        })
-                        |> Publisher.publish_one()
-             end) =~ "connrefused"
-
-      assert called(Instances.set_unreachable(inbox))
-    end
-
-    test_with_mock "does NOT call `Instances.set_unreachable` if target is reachable",
-                   Instances,
-                   [:passthrough],
-                   [] do
-      _actor = insert(:user)
-      inbox = "http://200.site/users/nick1/inbox"
-      activity = insert(:note_activity)
-
-      assert {:ok, _} =
-               Publisher.prepare_one(%{inbox: inbox, activity_id: activity.id})
-               |> Publisher.publish_one()
-
-      refute called(Instances.set_unreachable(inbox))
-    end
-
-    test_with_mock "does NOT call `Instances.set_unreachable` if target instance has non-nil `unreachable_since`",
-                   Instances,
-                   [:passthrough],
-                   [] do
-      _actor = insert(:user)
-      inbox = "http://connrefused.site/users/nick1/inbox"
-      activity = insert(:note_activity)
-
-      assert capture_log(fn ->
-               assert {:error, _} =
-                        Publisher.prepare_one(%{
-                          inbox: inbox,
-                          activity_id: activity.id,
-                          unreachable_since: NaiveDateTime.utc_now() |> NaiveDateTime.to_string()
-                        })
-                        |> Publisher.publish_one()
-             end) =~ "connrefused"
-
-      refute called(Instances.set_unreachable(inbox))
-    end
   end
 
   describe "publish/2" do

test/pleroma/web/federator_test.exs

@@ -126,22 +126,17 @@ defmodule Pleroma.Web.FederatorTest do
         inbox: inbox2
       })
 
-      dt = NaiveDateTime.utc_now()
+      Instances.set_unreachable(URI.parse(inbox2).host)
-      Instances.set_unreachable(inbox1, dt)
-
-      Instances.set_consistently_unreachable(URI.parse(inbox2).host)
 
       {:ok, _activity} =
         CommonAPI.post(user, %{status: "HI @nick1@domain.com, @nick2@domain2.com!"})
 
-      expected_dt = NaiveDateTime.to_iso8601(dt)
-
       ObanHelpers.perform(all_enqueued(worker: PublisherWorker))
 
       assert ObanHelpers.member?(
                %{
                  "op" => "publish_one",
-                 "params" => %{"inbox" => inbox1, "unreachable_since" => expected_dt}
+                 "params" => %{"inbox" => inbox1}
                },
                all_enqueued(worker: PublisherWorker)
              )

test/pleroma/web/pleroma_api/controllers/instances_controller_test.exs

@@ -7,16 +7,11 @@ defmodule Pleroma.Web.PleromaApi.InstancesControllerTest do
 
   alias Pleroma.Instances
 
-  setup_all do: clear_config([:instance, :federation_reachability_timeout_days], 1)
-
   setup do
     constant = "http://consistently-unreachable.name/"
-    eventual = "http://eventually-unreachable.com/path"
 
     {:ok, %Pleroma.Instances.Instance{unreachable_since: constant_unreachable}} =
-      Instances.set_consistently_unreachable(constant)
+      Instances.set_unreachable(constant)
-
-    _eventual_unreachable = Instances.set_unreachable(eventual)
 
     %{constant_unreachable: constant_unreachable, constant: constant}
   end

test/pleroma/workers/delete_worker_test.exs

@@ -17,7 +17,7 @@ defmodule Pleroma.Workers.DeleteWorkerTest do
       user1 = insert(:user, nickname: "alice@example.com", name: "Alice")
       user2 = insert(:user, nickname: "bob@example.com", name: "Bob")
 
-      {:ok, job} = Instance.delete_users_and_activities("example.com")
+      {:ok, job} = Instance.delete("example.com")
 
       assert_enqueued(
         worker: DeleteWorker,

test/pleroma/workers/publisher_worker_test.exs

@@ -7,7 +7,9 @@ defmodule Pleroma.Workers.PublisherWorkerTest do
   use Oban.Testing, repo: Pleroma.Repo
 
   import Pleroma.Factory
+  import Mock
 
+  alias Pleroma.Instances
   alias Pleroma.Object
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.ActivityPub.Builder
@@ -37,4 +39,85 @@ defmodule Pleroma.Workers.PublisherWorkerTest do
       assert {:ok, %Oban.Job{priority: 0}} = Federator.publish(post)
     end
   end
+
+  describe "Server reachability:" do
+    setup do
+      user = insert(:user)
+      remote_user = insert(:user, local: false, inbox: "https://example.com/inbox")
+      {:ok, _, _} = Pleroma.User.follow(remote_user, user)
+      {:ok, activity} = CommonAPI.post(user, %{status: "Test post"})
+
+      %{
+        user: user,
+        remote_user: remote_user,
+        activity: activity
+      }
+    end
+
+    test "marks server as unreachable only on final failure", %{activity: activity} do
+      with_mock Pleroma.Web.Federator,
+        perform: fn :publish_one, _params -> {:error, :connection_error} end do
+        # First attempt
+        job = %Oban.Job{
+          args: %{
+            "op" => "publish_one",
+            "params" => %{
+              "inbox" => "https://example.com/inbox",
+              "activity_id" => activity.id
+            }
+          },
+          attempt: 1,
+          max_attempts: 5
+        }
+
+        assert {:error, :connection_error} = Pleroma.Workers.PublisherWorker.perform(job)
+        assert Instances.reachable?("https://example.com/inbox")
+
+        # Final attempt
+        job = %{job | attempt: 5}
+        assert {:error, :connection_error} = Pleroma.Workers.PublisherWorker.perform(job)
+        refute Instances.reachable?("https://example.com/inbox")
+      end
+    end
+
+    test "does not mark server as unreachable on successful publish", %{activity: activity} do
+      with_mock Pleroma.Web.Federator,
+        perform: fn :publish_one, _params -> {:ok, %{status: 200}} end do
+        job = %Oban.Job{
+          args: %{
+            "op" => "publish_one",
+            "params" => %{
+              "inbox" => "https://example.com/inbox",
+              "activity_id" => activity.id
+            }
+          },
+          attempt: 1,
+          max_attempts: 5
+        }
+
+        assert :ok = Pleroma.Workers.PublisherWorker.perform(job)
+        assert Instances.reachable?("https://example.com/inbox")
+      end
+    end
+
+    test "cancels job if server is unreachable", %{activity: activity} do
+      # First mark the server as unreachable
+      Instances.set_unreachable("https://example.com/inbox")
+      refute Instances.reachable?("https://example.com/inbox")
+
+      job = %Oban.Job{
+        args: %{
+          "op" => "publish_one",
+          "params" => %{
+            "inbox" => "https://example.com/inbox",
+            "activity_id" => activity.id
+          }
+        },
+        attempt: 1,
+        max_attempts: 5
+      }
+
+      assert {:cancel, :unreachable} = Pleroma.Workers.PublisherWorker.perform(job)
+    end
+  end
 end

test/pleroma/workers/reachability_worker_test.exs

@@ -0,0 +1,226 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Workers.ReachabilityWorkerTest do
+  use Pleroma.DataCase, async: true
+  use Oban.Testing, repo: Pleroma.Repo
+
+  import Mock
+
+  alias Pleroma.Tests.ObanHelpers
+  alias Pleroma.Workers.ReachabilityWorker
+
+  setup do
+    ObanHelpers.wipe_all()
+    :ok
+  end
+
+  describe "progressive backoff phases" do
+    test "starts with phase_1min and progresses through phases on failure" do
+      domain = "example.com"
+
+      with_mocks([
+        {Pleroma.HTTP, [], [get: fn _ -> {:error, :timeout} end]},
+        {Pleroma.Instances, [], [set_reachable: fn _ -> :ok end]}
+      ]) do
+        # Start with phase_1min
+        job = %Oban.Job{
+          args: %{"domain" => domain, "phase" => "phase_1min", "attempt" => 1}
+        }
+
+        # First attempt fails
+        assert {:error, :timeout} = ReachabilityWorker.perform(job)
+
+        # Should schedule retry for phase_1min (attempt 2)
+        retry_jobs = all_enqueued(worker: ReachabilityWorker)
+        assert length(retry_jobs) == 1
+        [retry_job] = retry_jobs
+        assert retry_job.args["phase"] == "phase_1min"
+        assert retry_job.args["attempt"] == 2
+
+        # Clear jobs and simulate second attempt failure
+        ObanHelpers.wipe_all()
+
+        retry_job = %Oban.Job{
+          args: %{"domain" => domain, "phase" => "phase_1min", "attempt" => 2}
+        }
+
+        assert {:error, :timeout} = ReachabilityWorker.perform(retry_job)
+
+        # Should schedule retry for phase_1min (attempt 3)
+        retry_jobs = all_enqueued(worker: ReachabilityWorker)
+        assert length(retry_jobs) == 1
+        [retry_job] = retry_jobs
+        assert retry_job.args["phase"] == "phase_1min"
+        assert retry_job.args["attempt"] == 3
+
+        # Clear jobs and simulate third attempt failure (final attempt for phase_1min)
+        ObanHelpers.wipe_all()
+
+        retry_job = %Oban.Job{
+          args: %{"domain" => domain, "phase" => "phase_1min", "attempt" => 3}
+        }
+
+        assert {:error, :timeout} = ReachabilityWorker.perform(retry_job)
+
+        # Should schedule retry for phase_1min (attempt 4)
+        retry_jobs = all_enqueued(worker: ReachabilityWorker)
+        assert length(retry_jobs) == 1
+        [retry_job] = retry_jobs
+        assert retry_job.args["phase"] == "phase_1min"
+        assert retry_job.args["attempt"] == 4
+
+        # Clear jobs and simulate fourth attempt failure (final attempt for phase_1min)
+        ObanHelpers.wipe_all()
+
+        retry_job = %Oban.Job{
+          args: %{"domain" => domain, "phase" => "phase_1min", "attempt" => 4}
+        }
+
+        assert {:error, :timeout} = ReachabilityWorker.perform(retry_job)
+
+        # Should schedule next phase (phase_15min)
+        next_phase_jobs = all_enqueued(worker: ReachabilityWorker)
+        assert length(next_phase_jobs) == 1
+        [next_phase_job] = next_phase_jobs
+        assert next_phase_job.args["phase"] == "phase_15min"
+        assert next_phase_job.args["attempt"] == 1
+      end
+    end
+
+    test "progresses through all phases correctly" do
+      domain = "example.com"
+
+      with_mocks([
+        {Pleroma.HTTP, [], [get: fn _ -> {:error, :timeout} end]},
+        {Pleroma.Instances, [], [set_reachable: fn _ -> :ok end]}
+      ]) do
+        # Simulate all phases failing
+        phases = ["phase_1min", "phase_15min", "phase_1hour", "phase_8hour", "phase_24hour"]
+
+        Enum.each(phases, fn phase ->
+          {_interval, max_attempts, next_phase} = get_phase_config(phase)
+
+          # Simulate all attempts failing for this phase
+          Enum.each(1..max_attempts, fn attempt ->
+            job = %Oban.Job{args: %{"domain" => domain, "phase" => phase, "attempt" => attempt}}
+            assert {:error, :timeout} = ReachabilityWorker.perform(job)
+
+            if attempt < max_attempts do
+              # Should schedule retry for same phase
+              retry_jobs = all_enqueued(worker: ReachabilityWorker)
+              assert length(retry_jobs) == 1
+              [retry_job] = retry_jobs
+              assert retry_job.args["phase"] == phase
+              assert retry_job.args["attempt"] == attempt + 1
+              ObanHelpers.wipe_all()
+            else
+              # Should schedule next phase (except for final phase)
+              if next_phase != "final" do
+                next_phase_jobs = all_enqueued(worker: ReachabilityWorker)
+                assert length(next_phase_jobs) == 1
+                [next_phase_job] = next_phase_jobs
+                assert next_phase_job.args["phase"] == next_phase
+                assert next_phase_job.args["attempt"] == 1
+                ObanHelpers.wipe_all()
+              else
+                # Final phase - no more jobs should be scheduled
+                next_phase_jobs = all_enqueued(worker: ReachabilityWorker)
+                assert length(next_phase_jobs) == 0
+              end
+            end
+          end)
+        end)
+      end
+    end
+
+    test "succeeds and stops progression when instance becomes reachable" do
+      domain = "example.com"
+
+      with_mocks([
+        {Pleroma.HTTP, [], [get: fn _ -> {:ok, %{status: 200}} end]},
+        {Pleroma.Instances, [], [set_reachable: fn _ -> :ok end]}
+      ]) do
+        job = %Oban.Job{args: %{"domain" => domain, "phase" => "phase_1hour", "attempt" => 2}}
+
+        # Should succeed and not schedule any more jobs
+        assert :ok = ReachabilityWorker.perform(job)
+
+        # Verify set_reachable was called
+        assert_called(Pleroma.Instances.set_reachable("https://#{domain}"))
+
+        # No more jobs should be scheduled
+        next_jobs = all_enqueued(worker: ReachabilityWorker)
+        assert length(next_jobs) == 0
+      end
+    end
+
+    test "enforces uniqueness per domain using Oban's conflict detection" do
+      domain = "example.com"
+
+      # Insert first job for the domain
+      job1 =
+        %{
+          "domain" => domain,
+          "phase" => "phase_1min",
+          "attempt" => 1
+        }
+        |> ReachabilityWorker.new()
+        |> Oban.insert()
+
+      assert {:ok, _} = job1
+
+      # Try to insert a second job for the same domain with different phase/attempt
+      job2 =
+        %{
+          "domain" => domain,
+          "phase" => "phase_15min",
+          "attempt" => 1
+        }
+        |> ReachabilityWorker.new()
+        |> Oban.insert()
+
+      # Should fail due to uniqueness constraint (conflict)
+      assert {:ok, %Oban.Job{conflict?: true}} = job2
+
+      # Verify only one job exists for this domain
+      jobs = all_enqueued(worker: ReachabilityWorker)
+      assert length(jobs) == 1
+      [existing_job] = jobs
+      assert existing_job.args["domain"] == domain
+      assert existing_job.args["phase"] == "phase_1min"
+    end
+
+    test "handles new jobs with only domain argument and transitions them to the first phase" do
+      domain = "legacy.example.com"
+
+      with_mocks([
+        {Pleroma.Instances, [], [set_reachable: fn _ -> :ok end]}
+      ]) do
+        # Create a job with only domain (legacy format)
+        job = %Oban.Job{
+          args: %{"domain" => domain}
+        }
+
+        # Should reschedule with phase_1min and attempt 1
+        assert :ok = ReachabilityWorker.perform(job)
+
+        # Check that a new job was scheduled with the correct format
+        scheduled_jobs = all_enqueued(worker: ReachabilityWorker)
+        assert length(scheduled_jobs) == 1
+        [scheduled_job] = scheduled_jobs
+        assert scheduled_job.args["domain"] == domain
+        assert scheduled_job.args["phase"] == "phase_1min"
+        assert scheduled_job.args["attempt"] == 1
+      end
+    end
+  end
+
+  defp get_phase_config("phase_1min"), do: {1, 4, "phase_15min"}
+  defp get_phase_config("phase_15min"), do: {15, 4, "phase_1hour"}
+  defp get_phase_config("phase_1hour"), do: {60, 4, "phase_8hour"}
+  defp get_phase_config("phase_8hour"), do: {480, 4, "phase_24hour"}
+  defp get_phase_config("phase_24hour"), do: {1440, 4, "final"}
+  defp get_phase_config("final"), do: {nil, 0, nil}
+end

test/pleroma/workers/receiver_worker_test.exs

@@ -3,13 +3,14 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Workers.ReceiverWorkerTest do
-  use Pleroma.DataCase
+  use Pleroma.DataCase, async: true
   use Oban.Testing, repo: Pleroma.Repo
 
   import Mock
   import Pleroma.Factory
 
   alias Pleroma.User
+  alias Pleroma.Web.CommonAPI
   alias Pleroma.Web.Federator
   alias Pleroma.Workers.ReceiverWorker
 
@@ -243,4 +244,62 @@ defmodule Pleroma.Workers.ReceiverWorkerTest do
 
     assert {:cancel, _} = ReceiverWorker.perform(oban_job)
   end
+
+  describe "Server reachability:" do
+    setup do
+      user = insert(:user)
+      remote_user = insert(:user, local: false, ap_id: "https://example.com/users/remote")
+      {:ok, _, _} = Pleroma.User.follow(user, remote_user)
+      {:ok, activity} = CommonAPI.post(remote_user, %{status: "Test post"})
+
+      %{
+        user: user,
+        remote_user: remote_user,
+        activity: activity
+      }
+    end
+
+    test "schedules ReachabilityWorker if host is unreachable", %{activity: activity} do
+      with_mocks [
+        {Pleroma.Web.ActivityPub.Transmogrifier, [],
+         [handle_incoming: fn _ -> {:ok, activity} end]},
+        {Pleroma.Instances, [], [reachable?: fn _ -> false end]},
+        {Pleroma.Web.Federator, [], [perform: fn :incoming_ap_doc, _params -> {:ok, nil} end]}
+      ] do
+        job = %Oban.Job{
+          args: %{
+            "op" => "incoming_ap_doc",
+            "params" => activity.data
+          }
+        }
+
+        Pleroma.Workers.ReceiverWorker.perform(job)
+
+        assert_enqueued(
+          worker: Pleroma.Workers.ReachabilityWorker,
+          args: %{"domain" => "example.com"}
+        )
+      end
+    end
+
+    test "does not schedule ReachabilityWorker if host is reachable", %{activity: activity} do
+      with_mocks [
+        {Pleroma.Web.ActivityPub.Transmogrifier, [],
+         [handle_incoming: fn _ -> {:ok, activity} end]},
+        {Pleroma.Instances, [], [reachable?: fn _ -> true end]},
+        {Pleroma.Web.Federator, [], [perform: fn :incoming_ap_doc, _params -> {:ok, nil} end]}
+      ] do
+        job = %Oban.Job{
+          args: %{
+            "op" => "incoming_ap_doc",
+            "params" => activity.data
+          }
+        }
+
+        Pleroma.Workers.ReceiverWorker.perform(job)
+
+        refute_enqueued(worker: Pleroma.Workers.ReachabilityWorker)
+      end
+    end
+  end
 end

test/pleroma/workers/remote_fetcher_worker_test.exs

@@ -3,7 +3,7 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Workers.RemoteFetcherWorkerTest do
-  use Pleroma.DataCase
+  use Pleroma.DataCase, async: true
   use Oban.Testing, repo: Pleroma.Repo
 
   alias Pleroma.Workers.RemoteFetcherWorker