From 3d422ef3256e9eeef79d0c78743e19d8435dc352 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 5 Jun 2025 16:38:40 -0700 Subject: [PATCH 01/74] Reachability refactor The result of Oban jobs determine the reachability status. Publisher jobs will cancel themselves at execution time if the target server is now unreachable. Receiving activities does not immediately mark a server as reachable, but creates a ReachabilityWorker job to validate. A Cron will execute daily to test all unreachable servers. --- changelog.d/reachabililty.change | 1 + config/config.exs | 4 +- lib/pleroma/instances.ex | 20 +--- lib/pleroma/instances/instance.ex | 18 +-- lib/pleroma/object/fetcher.ex | 5 - .../activity_pub/activity_pub_controller.ex | 10 -- lib/pleroma/web/activity_pub/publisher.ex | 22 +--- .../controllers/instances_controller.ex | 2 +- .../cron/schedule_reachability_worker.ex | 33 ++++++ lib/pleroma/workers/publisher_worker.ex | 26 +++- lib/pleroma/workers/reachability_worker.ex | 31 +++++ lib/pleroma/workers/receiver_worker.ex | 11 ++ lib/pleroma/workers/remote_fetcher_worker.ex | 10 ++ test/pleroma/instances/instance_test.exs | 9 +- test/pleroma/instances_test.exs | 78 ++---------- test/pleroma/object/fetcher_test.exs | 12 -- .../activity_pub_controller_test.exs | 36 ------ .../web/activity_pub/publisher_test.exs | 111 ------------------ test/pleroma/web/federator_test.exs | 9 +- .../controllers/instances_controller_test.exs | 7 +- .../schedule_reachability_worker_test.exs | 52 ++++++++ .../pleroma/workers/publisher_worker_test.exs | 83 +++++++++++++ test/pleroma/workers/receiver_worker_test.exs | 61 +++++++++- .../workers/remote_fetcher_worker_test.exs | 2 +- 24 files changed, 341 insertions(+), 312 deletions(-) create mode 100644 changelog.d/reachabililty.change create mode 100644 lib/pleroma/workers/cron/schedule_reachability_worker.ex create mode 100644 lib/pleroma/workers/reachability_worker.ex create mode 100644 test/pleroma/workers/cron/schedule_reachability_worker_test.exs diff --git a/changelog.d/reachabililty.change b/changelog.d/reachabililty.change new file mode 100644 index 000000000..71b9514be --- /dev/null +++ b/changelog.d/reachabililty.change @@ -0,0 +1 @@ +Improved the logic of how we determine if a server is unreachable. diff --git a/config/config.exs b/config/config.exs index a231c5ba0..d164f8389 100644 --- a/config/config.exs +++ b/config/config.exs @@ -194,7 +194,6 @@ config :pleroma, :instance, account_approval_required: false, federating: true, federation_incoming_replies_max_depth: 100, - federation_reachability_timeout_days: 7, allow_relay: true, public: true, quarantined_instances: [], @@ -603,7 +602,8 @@ config :pleroma, Oban, crontab: [ {"0 0 * * 0", Pleroma.Workers.Cron.DigestEmailsWorker}, {"0 0 * * *", Pleroma.Workers.Cron.NewUsersDigestWorker}, - {"*/10 * * * *", Pleroma.Workers.Cron.AppCleanupWorker} + {"*/10 * * * *", Pleroma.Workers.Cron.AppCleanupWorker}, + {"0 0 * * *", Pleroma.Workers.Cron.ScheduleReachabilityWorker} ] config :pleroma, Pleroma.Formatter, diff --git a/lib/pleroma/instances.ex b/lib/pleroma/instances.ex index b6d83f591..9237e0944 100644 --- a/lib/pleroma/instances.ex +++ b/lib/pleroma/instances.ex @@ -15,25 +15,7 @@ defmodule Pleroma.Instances do defdelegate set_unreachable(url_or_host, unreachable_since \\ nil), to: Instance - defdelegate get_consistently_unreachable, to: Instance - - def set_consistently_unreachable(url_or_host), - do: set_unreachable(url_or_host, reachability_datetime_threshold()) - - def reachability_datetime_threshold do - federation_reachability_timeout_days = - Pleroma.Config.get([:instance, :federation_reachability_timeout_days], 0) - - if federation_reachability_timeout_days > 0 do - NaiveDateTime.add( - NaiveDateTime.utc_now(), - -federation_reachability_timeout_days * 24 * 3600, - :second - ) - else - ~N[0000-01-01 00:00:00] - end - end + defdelegate get_unreachable, to: Instance def host(url_or_host) when is_binary(url_or_host) do if url_or_host =~ ~r/^http/i do diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex index 33f1229d0..baccc314c 100644 --- a/lib/pleroma/instances/instance.ex +++ b/lib/pleroma/instances/instance.ex @@ -51,7 +51,7 @@ defmodule Pleroma.Instances.Instance do |> cast(params, [:software_name, :software_version, :software_repository]) end - def filter_reachable([]), do: %{} + def filter_reachable([]), do: [] def filter_reachable(urls_or_hosts) when is_list(urls_or_hosts) do hosts = @@ -68,19 +68,15 @@ defmodule Pleroma.Instances.Instance do ) |> Map.new(& &1) - reachability_datetime_threshold = Instances.reachability_datetime_threshold() - for entry <- Enum.filter(urls_or_hosts, &is_binary/1) do host = host(entry) unreachable_since = unreachable_since_by_host[host] - if !unreachable_since || - NaiveDateTime.compare(unreachable_since, reachability_datetime_threshold) == :gt do - {entry, unreachable_since} + if is_nil(unreachable_since) do + entry end end |> Enum.filter(& &1) - |> Map.new(& &1) end def reachable?(url_or_host) when is_binary(url_or_host) do @@ -88,7 +84,7 @@ defmodule Pleroma.Instances.Instance do from(i in Instance, where: i.host == ^host(url_or_host) and - i.unreachable_since <= ^Instances.reachability_datetime_threshold(), + i.unreachable_since <= ^NaiveDateTime.utc_now(), select: true ) ) @@ -132,11 +128,9 @@ defmodule Pleroma.Instances.Instance do def set_unreachable(_, _), do: {:error, nil} - def get_consistently_unreachable do - reachability_datetime_threshold = Instances.reachability_datetime_threshold() - + def get_unreachable do from(i in Instance, - where: ^reachability_datetime_threshold > i.unreachable_since, + where: not is_nil(i.unreachable_since), order_by: i.unreachable_since, select: {i.host, i.unreachable_since} ) diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index b54ef9ce5..ea5480a41 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -4,7 +4,6 @@ defmodule Pleroma.Object.Fetcher do alias Pleroma.HTTP - alias Pleroma.Instances alias Pleroma.Maps alias Pleroma.Object alias Pleroma.Object.Containment @@ -152,10 +151,6 @@ defmodule Pleroma.Object.Fetcher do {:ok, body} <- get_object(id), {:ok, data} <- safe_json_decode(body), :ok <- Containment.contain_origin_from_id(id, data) do - if not Instances.reachable?(id) do - Instances.set_reachable(id) - end - {:ok, data} else {:scheme, _} -> diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index 7ac0bbab4..2ee72c49a 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -53,7 +53,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do ) plug(:log_inbox_metadata when action in [:inbox]) - plug(:set_requester_reachable when action in [:inbox]) plug(:relay_active? when action in [:relay]) defp relay_active?(conn, _) do @@ -520,15 +519,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do |> json(dgettext("errors", "error")) end - defp set_requester_reachable(%Plug.Conn{} = conn, _) do - with actor <- conn.params["actor"], - true <- is_binary(actor) do - Pleroma.Instances.set_reachable(actor) - end - - conn - end - defp log_inbox_metadata(%{params: %{"actor" => actor, "type" => type}} = conn, _) do Logger.metadata(actor: actor, type: type) conn diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex index 0de3a0d43..78312b771 100644 --- a/lib/pleroma/web/activity_pub/publisher.ex +++ b/lib/pleroma/web/activity_pub/publisher.ex @@ -148,17 +148,9 @@ defmodule Pleroma.Web.ActivityPub.Publisher do {"digest", p.digest} ] ) do - if not is_nil(p.unreachable_since) do - Instances.set_reachable(p.inbox) - end - result else {_post_result, %{status: code} = response} = e -> - if is_nil(p.unreachable_since) do - Instances.set_unreachable(p.inbox) - end - Logger.metadata(activity: p.activity_id, inbox: p.inbox, status: code) Logger.error("Publisher failed to inbox #{p.inbox} with status #{code}") @@ -179,10 +171,6 @@ defmodule Pleroma.Web.ActivityPub.Publisher do connection_pool_snooze() e -> - if is_nil(p.unreachable_since) do - Instances.set_unreachable(p.inbox) - end - Logger.metadata(activity: p.activity_id, inbox: p.inbox) Logger.error("Publisher failed to inbox #{p.inbox} #{inspect(e)}") {:error, e} @@ -308,7 +296,7 @@ defmodule Pleroma.Web.ActivityPub.Publisher do Repo.checkout(fn -> Enum.each(inboxes, fn inboxes -> - Enum.each(inboxes, fn {inbox, unreachable_since} -> + Enum.each(inboxes, fn inbox -> %User{ap_id: ap_id} = Enum.find(recipients, fn actor -> actor.inbox == inbox end) # Get all the recipients on the same host and add them to cc. Otherwise, a remote @@ -318,8 +306,7 @@ defmodule Pleroma.Web.ActivityPub.Publisher do __MODULE__.enqueue_one(%{ inbox: inbox, cc: cc, - activity_id: activity.id, - unreachable_since: unreachable_since + activity_id: activity.id }) end) end) @@ -352,12 +339,11 @@ defmodule Pleroma.Web.ActivityPub.Publisher do |> Enum.each(fn {inboxes, priority} -> inboxes |> Instances.filter_reachable() - |> Enum.each(fn {inbox, unreachable_since} -> + |> Enum.each(fn inbox -> __MODULE__.enqueue_one( %{ inbox: inbox, - activity_id: activity.id, - unreachable_since: unreachable_since + activity_id: activity.id }, priority: priority ) diff --git a/lib/pleroma/web/pleroma_api/controllers/instances_controller.ex b/lib/pleroma/web/pleroma_api/controllers/instances_controller.ex index 6257e3153..85cfe9f00 100644 --- a/lib/pleroma/web/pleroma_api/controllers/instances_controller.ex +++ b/lib/pleroma/web/pleroma_api/controllers/instances_controller.ex @@ -13,7 +13,7 @@ defmodule Pleroma.Web.PleromaAPI.InstancesController do def show(conn, _params) do unreachable = - Instances.get_consistently_unreachable() + Instances.get_unreachable() |> Map.new(fn {host, date} -> {host, to_string(date)} end) json(conn, %{"unreachable" => unreachable}) diff --git a/lib/pleroma/workers/cron/schedule_reachability_worker.ex b/lib/pleroma/workers/cron/schedule_reachability_worker.ex new file mode 100644 index 000000000..a0b8e261c --- /dev/null +++ b/lib/pleroma/workers/cron/schedule_reachability_worker.ex @@ -0,0 +1,33 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Workers.Cron.ScheduleReachabilityWorker do + use Oban.Worker, + queue: :background, + max_attempts: 2 + + alias Pleroma.Instances + alias Pleroma.Repo + + @impl true + def perform(_job) do + unreachable_servers = Instances.get_unreachable() + + jobs = + unreachable_servers + |> Enum.map(fn {domain, _} -> + Pleroma.Workers.ReachabilityWorker.new(%{"domain" => domain}) + end) + + case Repo.transaction(fn -> + Enum.each(jobs, &Oban.insert/1) + end) do + {:ok, _} -> + :ok + + {:error, reason} -> + {:error, reason} + end + end +end diff --git a/lib/pleroma/workers/publisher_worker.ex b/lib/pleroma/workers/publisher_worker.ex index 7d9b022de..10736bef5 100644 --- a/lib/pleroma/workers/publisher_worker.ex +++ b/lib/pleroma/workers/publisher_worker.ex @@ -4,6 +4,7 @@ defmodule Pleroma.Workers.PublisherWorker do alias Pleroma.Activity + alias Pleroma.Instances alias Pleroma.Web.Federator use Oban.Worker, queue: :federator_outgoing, max_attempts: 5 @@ -14,9 +15,30 @@ defmodule Pleroma.Workers.PublisherWorker do Federator.perform(:publish, activity) end - def perform(%Job{args: %{"op" => "publish_one", "params" => params}}) do + def perform(%Job{args: %{"op" => "publish_one", "params" => params}} = job) do params = Map.new(params, fn {k, v} -> {String.to_atom(k), v} end) - Federator.perform(:publish_one, params) + + # Cancel / skip the job if this server believed to be unreachable now + if not Instances.reachable?(params.inbox) do + {:cancel, :unreachable} + else + case Federator.perform(:publish_one, params) do + {:ok, _} -> + :ok + + {:error, _} = error -> + # Only mark as unreachable on final failure + if job.attempt == job.max_attempts do + Instances.set_unreachable(params.inbox) + end + + error + + error -> + # Unexpected error, may have been client side + error + end + end end @impl true diff --git a/lib/pleroma/workers/reachability_worker.ex b/lib/pleroma/workers/reachability_worker.ex new file mode 100644 index 000000000..3a11dfe2a --- /dev/null +++ b/lib/pleroma/workers/reachability_worker.ex @@ -0,0 +1,31 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Workers.ReachabilityWorker do + use Oban.Worker, + queue: :background, + max_attempts: 3, + unique: [period: :infinity, states: [:available, :scheduled]] + + alias Pleroma.HTTP + alias Pleroma.Instances + + @impl true + def perform(%Oban.Job{args: %{"domain" => domain}}) do + case HTTP.get("https://#{domain}/.well-known/nodeinfo") do + {:ok, %{status: status}} when status in 200..299 -> + Instances.set_reachable("https://#{domain}") + :ok + + {:ok, %{status: _status}} -> + {:error, :unreachable} + + {:error, _} = error -> + error + end + end + + @impl true + def timeout(_job), do: :timer.seconds(5) +end diff --git a/lib/pleroma/workers/receiver_worker.ex b/lib/pleroma/workers/receiver_worker.ex index 11b672bef..e2c950967 100644 --- a/lib/pleroma/workers/receiver_worker.ex +++ b/lib/pleroma/workers/receiver_worker.ex @@ -3,6 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Workers.ReceiverWorker do + alias Pleroma.Instances alias Pleroma.Signature alias Pleroma.User alias Pleroma.Web.Federator @@ -37,6 +38,11 @@ defmodule Pleroma.Workers.ReceiverWorker do {:ok, _public_key} <- Signature.refetch_public_key(conn_data), {:signature, true} <- {:signature, Signature.validate_signature(conn_data)}, {:ok, res} <- Federator.perform(:incoming_ap_doc, params) do + unless Instances.reachable?(params["actor"]) do + domain = URI.parse(params["actor"]).host + Oban.insert(Pleroma.Workers.ReachabilityWorker.new(%{"domain" => domain})) + end + {:ok, res} else e -> process_errors(e) @@ -45,6 +51,11 @@ defmodule Pleroma.Workers.ReceiverWorker do def perform(%Job{args: %{"op" => "incoming_ap_doc", "params" => params}}) do with {:ok, res} <- Federator.perform(:incoming_ap_doc, params) do + unless Instances.reachable?(params["actor"]) do + domain = URI.parse(params["actor"]).host + Oban.insert(Pleroma.Workers.ReachabilityWorker.new(%{"domain" => domain})) + end + {:ok, res} else e -> process_errors(e) diff --git a/lib/pleroma/workers/remote_fetcher_worker.ex b/lib/pleroma/workers/remote_fetcher_worker.ex index aa09362f5..5f57ec2d7 100644 --- a/lib/pleroma/workers/remote_fetcher_worker.ex +++ b/lib/pleroma/workers/remote_fetcher_worker.ex @@ -3,6 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Workers.RemoteFetcherWorker do + alias Pleroma.Instances alias Pleroma.Object.Fetcher use Oban.Worker, queue: :background, unique: [period: :infinity] @@ -11,6 +12,15 @@ defmodule Pleroma.Workers.RemoteFetcherWorker do def perform(%Job{args: %{"op" => "fetch_remote", "id" => id} = args}) do case Fetcher.fetch_object_from_id(id, depth: args["depth"]) do {:ok, _object} -> + # Mark the server as reachable since we successfully fetched an object + case URI.parse(id) do + %URI{host: host} when not is_nil(host) -> + Instances.set_reachable("https://#{host}") + + _ -> + :ok + end + :ok {:allowed_depth, false} -> diff --git a/test/pleroma/instances/instance_test.exs b/test/pleroma/instances/instance_test.exs index 6a718be21..f195d9bd6 100644 --- a/test/pleroma/instances/instance_test.exs +++ b/test/pleroma/instances/instance_test.exs @@ -3,7 +3,6 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Instances.InstanceTest do - alias Pleroma.Instances alias Pleroma.Instances.Instance alias Pleroma.Repo alias Pleroma.Tests.ObanHelpers @@ -14,8 +13,6 @@ defmodule Pleroma.Instances.InstanceTest do import ExUnit.CaptureLog import Pleroma.Factory - setup_all do: clear_config([:instance, :federation_reachability_timeout_days], 1) - describe "set_reachable/1" do test "clears `unreachable_since` of existing matching Instance record having non-nil `unreachable_since`" do unreachable_since = NaiveDateTime.to_iso8601(NaiveDateTime.utc_now()) @@ -145,7 +142,11 @@ defmodule Pleroma.Instances.InstanceTest do end test "Doesn't scrapes unreachable instances" do - instance = insert(:instance, unreachable_since: Instances.reachability_datetime_threshold()) + instance = + insert(:instance, + unreachable_since: NaiveDateTime.utc_now() |> NaiveDateTime.add(-:timer.hours(24)) + ) + url = "https://" <> instance.host assert capture_log(fn -> assert nil == Instance.get_or_update_favicon(URI.parse(url)) end) =~ diff --git a/test/pleroma/instances_test.exs b/test/pleroma/instances_test.exs index 96fa9cffe..cbafbfa44 100644 --- a/test/pleroma/instances_test.exs +++ b/test/pleroma/instances_test.exs @@ -7,73 +7,40 @@ defmodule Pleroma.InstancesTest do use Pleroma.DataCase - setup_all do: clear_config([:instance, :federation_reachability_timeout_days], 1) - describe "reachable?/1" do test "returns `true` for host / url with unknown reachability status" do assert Instances.reachable?("unknown.site") assert Instances.reachable?("http://unknown.site") end - - test "returns `false` for host / url marked unreachable for at least `reachability_datetime_threshold()`" do - host = "consistently-unreachable.name" - Instances.set_consistently_unreachable(host) - - refute Instances.reachable?(host) - refute Instances.reachable?("http://#{host}/path") - end - - test "returns `true` for host / url marked unreachable for less than `reachability_datetime_threshold()`" do - url = "http://eventually-unreachable.name/path" - - Instances.set_unreachable(url) - - assert Instances.reachable?(url) - assert Instances.reachable?(URI.parse(url).host) - end - - test "raises FunctionClauseError exception on non-binary input" do - assert_raise FunctionClauseError, fn -> Instances.reachable?(nil) end - assert_raise FunctionClauseError, fn -> Instances.reachable?(1) end - end end describe "filter_reachable/1" do setup do - host = "consistently-unreachable.name" - url1 = "http://eventually-unreachable.com/path" - url2 = "http://domain.com/path" + unreachable_host = "consistently-unreachable.name" + reachable_host = "http://domain.com/path" - Instances.set_consistently_unreachable(host) - Instances.set_unreachable(url1) + Instances.set_unreachable(unreachable_host) - result = Instances.filter_reachable([host, url1, url2, nil]) - %{result: result, url1: url1, url2: url2} + result = Instances.filter_reachable([unreachable_host, reachable_host, nil]) + %{result: result, reachable_host: reachable_host, unreachable_host: unreachable_host} end - test "returns a map with keys containing 'not marked consistently unreachable' elements of supplied list", - %{result: result, url1: url1, url2: url2} do - assert is_map(result) - assert Enum.sort([url1, url2]) == result |> Map.keys() |> Enum.sort() + test "returns a list of only reachable elements", + %{result: result, reachable_host: reachable_host} do + assert is_list(result) + assert [reachable_host] == result end - test "returns a map with `unreachable_since` values for keys", - %{result: result, url1: url1, url2: url2} do - assert is_map(result) - assert %NaiveDateTime{} = result[url1] - assert is_nil(result[url2]) - end - - test "returns an empty map for empty list or list containing no hosts / url" do - assert %{} == Instances.filter_reachable([]) - assert %{} == Instances.filter_reachable([nil]) + test "returns an empty list when provided no data" do + assert [] == Instances.filter_reachable([]) + assert [] == Instances.filter_reachable([nil]) end end describe "set_reachable/1" do test "sets unreachable url or host reachable" do host = "domain.com" - Instances.set_consistently_unreachable(host) + Instances.set_unreachable(host) refute Instances.reachable?(host) Instances.set_reachable(host) @@ -102,23 +69,4 @@ defmodule Pleroma.InstancesTest do assert {:error, _} = Instances.set_unreachable(1) end end - - describe "set_consistently_unreachable/1" do - test "sets reachable url or host unreachable" do - url = "http://domain.com?q=" - assert Instances.reachable?(url) - - Instances.set_consistently_unreachable(url) - refute Instances.reachable?(url) - end - - test "keeps unreachable url or host unreachable" do - host = "site.name" - Instances.set_consistently_unreachable(host) - refute Instances.reachable?(host) - - Instances.set_consistently_unreachable(host) - refute Instances.reachable?(host) - end - end end diff --git a/test/pleroma/object/fetcher_test.exs b/test/pleroma/object/fetcher_test.exs index 7ba5090e1..9afa34fa2 100644 --- a/test/pleroma/object/fetcher_test.exs +++ b/test/pleroma/object/fetcher_test.exs @@ -6,7 +6,6 @@ defmodule Pleroma.Object.FetcherTest do use Pleroma.DataCase alias Pleroma.Activity - alias Pleroma.Instances alias Pleroma.Object alias Pleroma.Object.Fetcher alias Pleroma.Web.ActivityPub.ObjectValidator @@ -250,17 +249,6 @@ defmodule Pleroma.Object.FetcherTest do result = Fetcher.fetch_object_from_id("https://example.com/objects/no-content-type") assert {:fetch, {:error, nil}} = result end - - test "it resets instance reachability on successful fetch" do - id = "http://mastodon.example.org/@admin/99541947525187367" - Instances.set_consistently_unreachable(id) - refute Instances.reachable?(id) - - {:ok, _object} = - Fetcher.fetch_object_from_id("http://mastodon.example.org/@admin/99541947525187367") - - assert Instances.reachable?(id) - end end describe "implementation quirks" do diff --git a/test/pleroma/web/activity_pub/activity_pub_controller_test.exs b/test/pleroma/web/activity_pub/activity_pub_controller_test.exs index 46b3d5f0d..d9be82e64 100644 --- a/test/pleroma/web/activity_pub/activity_pub_controller_test.exs +++ b/test/pleroma/web/activity_pub/activity_pub_controller_test.exs @@ -8,7 +8,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do alias Pleroma.Activity alias Pleroma.Delivery - alias Pleroma.Instances alias Pleroma.Object alias Pleroma.Tests.ObanHelpers alias Pleroma.User @@ -601,23 +600,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do assert Activity.get_by_ap_id(data["id"]) end - test "it clears `unreachable` federation status of the sender", %{conn: conn} do - data = File.read!("test/fixtures/mastodon-post-activity.json") |> Jason.decode!() - - sender_url = data["actor"] - Instances.set_consistently_unreachable(sender_url) - refute Instances.reachable?(sender_url) - - conn = - conn - |> assign(:valid_signature, true) - |> put_req_header("content-type", "application/activity+json") - |> post("/inbox", data) - - assert "ok" == json_response(conn, 200) - assert Instances.reachable?(sender_url) - end - test "accept follow activity", %{conn: conn} do clear_config([:instance, :federating], true) relay = Relay.get_actor() @@ -1108,24 +1090,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do assert response(conn, 200) =~ note_object.data["content"] end - test "it clears `unreachable` federation status of the sender", %{conn: conn, data: data} do - user = insert(:user) - data = Map.put(data, "bcc", [user.ap_id]) - - sender_host = URI.parse(data["actor"]).host - Instances.set_consistently_unreachable(sender_host) - refute Instances.reachable?(sender_host) - - conn = - conn - |> assign(:valid_signature, true) - |> put_req_header("content-type", "application/activity+json") - |> post("/users/#{user.nickname}/inbox", data) - - assert "ok" == json_response(conn, 200) - assert Instances.reachable?(sender_host) - end - test "it removes all follower collections but actor's", %{conn: conn} do [actor, recipient] = insert_pair(:user) diff --git a/test/pleroma/web/activity_pub/publisher_test.exs b/test/pleroma/web/activity_pub/publisher_test.exs index 99ed42877..a6f25c9a7 100644 --- a/test/pleroma/web/activity_pub/publisher_test.exs +++ b/test/pleroma/web/activity_pub/publisher_test.exs @@ -6,13 +6,11 @@ defmodule Pleroma.Web.ActivityPub.PublisherTest do use Oban.Testing, repo: Pleroma.Repo use Pleroma.Web.ConnCase - import ExUnit.CaptureLog import Pleroma.Factory import Tesla.Mock import Mock alias Pleroma.Activity - alias Pleroma.Instances alias Pleroma.Object alias Pleroma.Tests.ObanHelpers alias Pleroma.Web.ActivityPub.Publisher @@ -167,115 +165,6 @@ defmodule Pleroma.Web.ActivityPub.PublisherTest do }) |> Publisher.publish_one() end - - test_with_mock "calls `Instances.set_reachable` on successful federation if `unreachable_since` is set", - Instances, - [:passthrough], - [] do - _actor = insert(:user) - inbox = "http://200.site/users/nick1/inbox" - activity = insert(:note_activity) - - assert {:ok, _} = - Publisher.prepare_one(%{ - inbox: inbox, - activity_id: activity.id, - unreachable_since: NaiveDateTime.utc_now() |> NaiveDateTime.to_string() - }) - |> Publisher.publish_one() - - assert called(Instances.set_reachable(inbox)) - end - - test_with_mock "does NOT call `Instances.set_reachable` on successful federation if `unreachable_since` is nil", - Instances, - [:passthrough], - [] do - _actor = insert(:user) - inbox = "http://200.site/users/nick1/inbox" - activity = insert(:note_activity) - - assert {:ok, _} = - Publisher.prepare_one(%{ - inbox: inbox, - activity_id: activity.id, - unreachable_since: nil - }) - |> Publisher.publish_one() - - refute called(Instances.set_reachable(inbox)) - end - - test_with_mock "calls `Instances.set_unreachable` on target inbox on non-2xx HTTP response code", - Instances, - [:passthrough], - [] do - _actor = insert(:user) - inbox = "http://404.site/users/nick1/inbox" - activity = insert(:note_activity) - - assert {:cancel, _} = - Publisher.prepare_one(%{inbox: inbox, activity_id: activity.id}) - |> Publisher.publish_one() - - assert called(Instances.set_unreachable(inbox)) - end - - test_with_mock "it calls `Instances.set_unreachable` on target inbox on request error of any kind", - Instances, - [:passthrough], - [] do - _actor = insert(:user) - inbox = "http://connrefused.site/users/nick1/inbox" - activity = insert(:note_activity) - - assert capture_log(fn -> - assert {:error, _} = - Publisher.prepare_one(%{ - inbox: inbox, - activity_id: activity.id - }) - |> Publisher.publish_one() - end) =~ "connrefused" - - assert called(Instances.set_unreachable(inbox)) - end - - test_with_mock "does NOT call `Instances.set_unreachable` if target is reachable", - Instances, - [:passthrough], - [] do - _actor = insert(:user) - inbox = "http://200.site/users/nick1/inbox" - activity = insert(:note_activity) - - assert {:ok, _} = - Publisher.prepare_one(%{inbox: inbox, activity_id: activity.id}) - |> Publisher.publish_one() - - refute called(Instances.set_unreachable(inbox)) - end - - test_with_mock "does NOT call `Instances.set_unreachable` if target instance has non-nil `unreachable_since`", - Instances, - [:passthrough], - [] do - _actor = insert(:user) - inbox = "http://connrefused.site/users/nick1/inbox" - activity = insert(:note_activity) - - assert capture_log(fn -> - assert {:error, _} = - Publisher.prepare_one(%{ - inbox: inbox, - activity_id: activity.id, - unreachable_since: NaiveDateTime.utc_now() |> NaiveDateTime.to_string() - }) - |> Publisher.publish_one() - end) =~ "connrefused" - - refute called(Instances.set_unreachable(inbox)) - end end describe "publish/2" do diff --git a/test/pleroma/web/federator_test.exs b/test/pleroma/web/federator_test.exs index 4a398f239..16fe1066a 100644 --- a/test/pleroma/web/federator_test.exs +++ b/test/pleroma/web/federator_test.exs @@ -126,22 +126,17 @@ defmodule Pleroma.Web.FederatorTest do inbox: inbox2 }) - dt = NaiveDateTime.utc_now() - Instances.set_unreachable(inbox1, dt) - - Instances.set_consistently_unreachable(URI.parse(inbox2).host) + Instances.set_unreachable(URI.parse(inbox2).host) {:ok, _activity} = CommonAPI.post(user, %{status: "HI @nick1@domain.com, @nick2@domain2.com!"}) - expected_dt = NaiveDateTime.to_iso8601(dt) - ObanHelpers.perform(all_enqueued(worker: PublisherWorker)) assert ObanHelpers.member?( %{ "op" => "publish_one", - "params" => %{"inbox" => inbox1, "unreachable_since" => expected_dt} + "params" => %{"inbox" => inbox1} }, all_enqueued(worker: PublisherWorker) ) diff --git a/test/pleroma/web/pleroma_api/controllers/instances_controller_test.exs b/test/pleroma/web/pleroma_api/controllers/instances_controller_test.exs index 0d4951a73..702c05504 100644 --- a/test/pleroma/web/pleroma_api/controllers/instances_controller_test.exs +++ b/test/pleroma/web/pleroma_api/controllers/instances_controller_test.exs @@ -7,16 +7,11 @@ defmodule Pleroma.Web.PleromaApi.InstancesControllerTest do alias Pleroma.Instances - setup_all do: clear_config([:instance, :federation_reachability_timeout_days], 1) - setup do constant = "http://consistently-unreachable.name/" - eventual = "http://eventually-unreachable.com/path" {:ok, %Pleroma.Instances.Instance{unreachable_since: constant_unreachable}} = - Instances.set_consistently_unreachable(constant) - - _eventual_unreachable = Instances.set_unreachable(eventual) + Instances.set_unreachable(constant) %{constant_unreachable: constant_unreachable, constant: constant} end diff --git a/test/pleroma/workers/cron/schedule_reachability_worker_test.exs b/test/pleroma/workers/cron/schedule_reachability_worker_test.exs new file mode 100644 index 000000000..310c2e61a --- /dev/null +++ b/test/pleroma/workers/cron/schedule_reachability_worker_test.exs @@ -0,0 +1,52 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Workers.Cron.ScheduleReachabilityWorkerTest do + use Pleroma.DataCase, async: true + use Oban.Testing, repo: Pleroma.Repo + + alias Pleroma.Instances + alias Pleroma.Workers.Cron.ScheduleReachabilityWorker + + describe "perform/1" do + test "schedules reachability checks for unreachable servers" do + # Mark some servers as unreachable + Instances.set_unreachable("https://example.com") + Instances.set_unreachable("https://test.com") + Instances.set_unreachable("https://another.com") + + # Verify they are marked as unreachable + refute Instances.reachable?("https://example.com") + refute Instances.reachable?("https://test.com") + refute Instances.reachable?("https://another.com") + + # Run the worker + assert :ok = ScheduleReachabilityWorker.perform(%Oban.Job{}) + + # Verify ReachabilityWorker jobs were scheduled for each server + # Note: domains in get_unreachable/0 are without the https:// prefix + assert_enqueued( + worker: Pleroma.Workers.ReachabilityWorker, + args: %{"domain" => "example.com"} + ) + + assert_enqueued( + worker: Pleroma.Workers.ReachabilityWorker, + args: %{"domain" => "test.com"} + ) + + assert_enqueued( + worker: Pleroma.Workers.ReachabilityWorker, + args: %{"domain" => "another.com"} + ) + end + + test "handles empty list of unreachable servers" do + # Ensure no servers are marked as unreachable + assert [] = Instances.get_unreachable() + assert :ok = ScheduleReachabilityWorker.perform(%Oban.Job{}) + refute_enqueued(worker: Pleroma.Workers.ReachabilityWorker) + end + end +end diff --git a/test/pleroma/workers/publisher_worker_test.exs b/test/pleroma/workers/publisher_worker_test.exs index 13372bf49..ca432d9bf 100644 --- a/test/pleroma/workers/publisher_worker_test.exs +++ b/test/pleroma/workers/publisher_worker_test.exs @@ -7,7 +7,9 @@ defmodule Pleroma.Workers.PublisherWorkerTest do use Oban.Testing, repo: Pleroma.Repo import Pleroma.Factory + import Mock + alias Pleroma.Instances alias Pleroma.Object alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Builder @@ -37,4 +39,85 @@ defmodule Pleroma.Workers.PublisherWorkerTest do assert {:ok, %Oban.Job{priority: 0}} = Federator.publish(post) end end + + describe "Server reachability:" do + setup do + user = insert(:user) + remote_user = insert(:user, local: false, inbox: "https://example.com/inbox") + {:ok, _, _} = Pleroma.User.follow(remote_user, user) + {:ok, activity} = CommonAPI.post(user, %{status: "Test post"}) + + %{ + user: user, + remote_user: remote_user, + activity: activity + } + end + + test "marks server as unreachable only on final failure", %{activity: activity} do + with_mock Pleroma.Web.Federator, + perform: fn :publish_one, _params -> {:error, :connection_error} end do + # First attempt + job = %Oban.Job{ + args: %{ + "op" => "publish_one", + "params" => %{ + "inbox" => "https://example.com/inbox", + "activity_id" => activity.id + } + }, + attempt: 1, + max_attempts: 5 + } + + assert {:error, :connection_error} = Pleroma.Workers.PublisherWorker.perform(job) + assert Instances.reachable?("https://example.com/inbox") + + # Final attempt + job = %{job | attempt: 5} + assert {:error, :connection_error} = Pleroma.Workers.PublisherWorker.perform(job) + refute Instances.reachable?("https://example.com/inbox") + end + end + + test "does not mark server as unreachable on successful publish", %{activity: activity} do + with_mock Pleroma.Web.Federator, + perform: fn :publish_one, _params -> {:ok, %{status: 200}} end do + job = %Oban.Job{ + args: %{ + "op" => "publish_one", + "params" => %{ + "inbox" => "https://example.com/inbox", + "activity_id" => activity.id + } + }, + attempt: 1, + max_attempts: 5 + } + + assert :ok = Pleroma.Workers.PublisherWorker.perform(job) + assert Instances.reachable?("https://example.com/inbox") + end + end + + test "cancels job if server is unreachable", %{activity: activity} do + # First mark the server as unreachable + Instances.set_unreachable("https://example.com/inbox") + refute Instances.reachable?("https://example.com/inbox") + + job = %Oban.Job{ + args: %{ + "op" => "publish_one", + "params" => %{ + "inbox" => "https://example.com/inbox", + "activity_id" => activity.id + } + }, + attempt: 1, + max_attempts: 5 + } + + assert {:cancel, :unreachable} = Pleroma.Workers.PublisherWorker.perform(job) + end + end end diff --git a/test/pleroma/workers/receiver_worker_test.exs b/test/pleroma/workers/receiver_worker_test.exs index 4d53c44ed..7f4789f91 100644 --- a/test/pleroma/workers/receiver_worker_test.exs +++ b/test/pleroma/workers/receiver_worker_test.exs @@ -3,13 +3,14 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Workers.ReceiverWorkerTest do - use Pleroma.DataCase + use Pleroma.DataCase, async: true use Oban.Testing, repo: Pleroma.Repo import Mock import Pleroma.Factory alias Pleroma.User + alias Pleroma.Web.CommonAPI alias Pleroma.Web.Federator alias Pleroma.Workers.ReceiverWorker @@ -243,4 +244,62 @@ defmodule Pleroma.Workers.ReceiverWorkerTest do assert {:cancel, _} = ReceiverWorker.perform(oban_job) end + + describe "Server reachability:" do + setup do + user = insert(:user) + remote_user = insert(:user, local: false, ap_id: "https://example.com/users/remote") + {:ok, _, _} = Pleroma.User.follow(user, remote_user) + {:ok, activity} = CommonAPI.post(remote_user, %{status: "Test post"}) + + %{ + user: user, + remote_user: remote_user, + activity: activity + } + end + + test "schedules ReachabilityWorker if host is unreachable", %{activity: activity} do + with_mocks [ + {Pleroma.Web.ActivityPub.Transmogrifier, [], + [handle_incoming: fn _ -> {:ok, activity} end]}, + {Pleroma.Instances, [], [reachable?: fn _ -> false end]}, + {Pleroma.Web.Federator, [], [perform: fn :incoming_ap_doc, _params -> {:ok, nil} end]} + ] do + job = %Oban.Job{ + args: %{ + "op" => "incoming_ap_doc", + "params" => activity.data + } + } + + Pleroma.Workers.ReceiverWorker.perform(job) + + assert_enqueued( + worker: Pleroma.Workers.ReachabilityWorker, + args: %{"domain" => "example.com"} + ) + end + end + + test "does not schedule ReachabilityWorker if host is reachable", %{activity: activity} do + with_mocks [ + {Pleroma.Web.ActivityPub.Transmogrifier, [], + [handle_incoming: fn _ -> {:ok, activity} end]}, + {Pleroma.Instances, [], [reachable?: fn _ -> true end]}, + {Pleroma.Web.Federator, [], [perform: fn :incoming_ap_doc, _params -> {:ok, nil} end]} + ] do + job = %Oban.Job{ + args: %{ + "op" => "incoming_ap_doc", + "params" => activity.data + } + } + + Pleroma.Workers.ReceiverWorker.perform(job) + + refute_enqueued(worker: Pleroma.Workers.ReachabilityWorker) + end + end + end end diff --git a/test/pleroma/workers/remote_fetcher_worker_test.exs b/test/pleroma/workers/remote_fetcher_worker_test.exs index 9caddb600..6eb6932cb 100644 --- a/test/pleroma/workers/remote_fetcher_worker_test.exs +++ b/test/pleroma/workers/remote_fetcher_worker_test.exs @@ -3,7 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Workers.RemoteFetcherWorkerTest do - use Pleroma.DataCase + use Pleroma.DataCase, async: true use Oban.Testing, repo: Pleroma.Repo alias Pleroma.Workers.RemoteFetcherWorker From b87ec4997244fc23d803948eccc778a603cf566f Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 6 Jun 2025 12:55:21 -0700 Subject: [PATCH 02/74] Nodeinfo is not universally implemented --- lib/pleroma/workers/reachability_worker.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/workers/reachability_worker.ex b/lib/pleroma/workers/reachability_worker.ex index 3a11dfe2a..d9f764322 100644 --- a/lib/pleroma/workers/reachability_worker.ex +++ b/lib/pleroma/workers/reachability_worker.ex @@ -13,7 +13,7 @@ defmodule Pleroma.Workers.ReachabilityWorker do @impl true def perform(%Oban.Job{args: %{"domain" => domain}}) do - case HTTP.get("https://#{domain}/.well-known/nodeinfo") do + case HTTP.get("https://#{domain}/") do {:ok, %{status: status}} when status in 200..299 -> Instances.set_reachable("https://#{domain}") :ok From 0f667761a9349a852c549c0bfb846b793607e397 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 6 Jun 2025 13:00:54 -0700 Subject: [PATCH 03/74] The ap_id is a URL, so we can just pass that to set_reachable/1 Also only bother attempting to mark reachable if it was known to be unreachable --- lib/pleroma/workers/remote_fetcher_worker.ex | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/lib/pleroma/workers/remote_fetcher_worker.ex b/lib/pleroma/workers/remote_fetcher_worker.ex index 5f57ec2d7..0cc480c02 100644 --- a/lib/pleroma/workers/remote_fetcher_worker.ex +++ b/lib/pleroma/workers/remote_fetcher_worker.ex @@ -12,13 +12,9 @@ defmodule Pleroma.Workers.RemoteFetcherWorker do def perform(%Job{args: %{"op" => "fetch_remote", "id" => id} = args}) do case Fetcher.fetch_object_from_id(id, depth: args["depth"]) do {:ok, _object} -> - # Mark the server as reachable since we successfully fetched an object - case URI.parse(id) do - %URI{host: host} when not is_nil(host) -> - Instances.set_reachable("https://#{host}") - - _ -> - :ok + unless Instances.reachable?(id) do + # Mark the server as reachable since we successfully fetched an object + Instances.set_reachable(id) end :ok From 0fe03fc4eef0159e3015d68d75ec42ea11f649cf Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 6 Jun 2025 13:44:24 -0700 Subject: [PATCH 04/74] Revert "Nodeinfo is not universally implemented" This reverts commit b87ec4997244fc23d803948eccc778a603cf566f. --- lib/pleroma/workers/reachability_worker.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/workers/reachability_worker.ex b/lib/pleroma/workers/reachability_worker.ex index d9f764322..3a11dfe2a 100644 --- a/lib/pleroma/workers/reachability_worker.ex +++ b/lib/pleroma/workers/reachability_worker.ex @@ -13,7 +13,7 @@ defmodule Pleroma.Workers.ReachabilityWorker do @impl true def perform(%Oban.Job{args: %{"domain" => domain}}) do - case HTTP.get("https://#{domain}/") do + case HTTP.get("https://#{domain}/.well-known/nodeinfo") do {:ok, %{status: status}} when status in 200..299 -> Instances.set_reachable("https://#{domain}") :ok From 83c97568259d5bf34f2117f37c5ec61495f9bc5b Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 6 Jun 2025 17:10:33 -0700 Subject: [PATCH 05/74] Remove unncessary NaiveDateTime call. Every non-nil entry in the database is considered unreachable. --- lib/pleroma/instances/instance.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex index baccc314c..7b7127973 100644 --- a/lib/pleroma/instances/instance.ex +++ b/lib/pleroma/instances/instance.ex @@ -84,7 +84,7 @@ defmodule Pleroma.Instances.Instance do from(i in Instance, where: i.host == ^host(url_or_host) and - i.unreachable_since <= ^NaiveDateTime.utc_now(), + not is_nil(i.unreachable_since), select: true ) ) From 3984ba87217e2a9fdc89c22ff2357c49563c5ad2 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sat, 7 Jun 2025 22:51:26 +0400 Subject: [PATCH 06/74] Fix typo in changelog filename --- changelog.d/fix-public-url-addressing.fix | 1 + changelog.d/{reachabililty.change => reachability.change} | 0 2 files changed, 1 insertion(+) create mode 100644 changelog.d/fix-public-url-addressing.fix rename changelog.d/{reachabililty.change => reachability.change} (100%) diff --git a/changelog.d/fix-public-url-addressing.fix b/changelog.d/fix-public-url-addressing.fix new file mode 100644 index 000000000..810b76905 --- /dev/null +++ b/changelog.d/fix-public-url-addressing.fix @@ -0,0 +1 @@ +- Fixed an issue where the ActivityStreams Public collection URL was being removed from incoming activities' cc fields diff --git a/changelog.d/reachabililty.change b/changelog.d/reachability.change similarity index 100% rename from changelog.d/reachabililty.change rename to changelog.d/reachability.change From 2748891e124ede3619cb2a77e27b83fbb8a724f8 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 7 Jun 2025 12:23:47 -0700 Subject: [PATCH 07/74] Change the inboxes assignment in the Publisher to better indicate it's a list containing two lists This clarifies what is really going on here and removes confusion about the nested Enum.each |> Enum.each which both were using an assignment called "inboxes" --- lib/pleroma/web/activity_pub/publisher.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex index 78312b771..4a5cbd64e 100644 --- a/lib/pleroma/web/activity_pub/publisher.ex +++ b/lib/pleroma/web/activity_pub/publisher.ex @@ -282,7 +282,7 @@ defmodule Pleroma.Web.ActivityPub.Publisher do [priority_recipients, recipients] = recipients(actor, activity) - inboxes = + [priority_inboxes, other_inboxes] = [priority_recipients, recipients] |> Enum.map(fn recipients -> recipients @@ -295,7 +295,7 @@ defmodule Pleroma.Web.ActivityPub.Publisher do end) Repo.checkout(fn -> - Enum.each(inboxes, fn inboxes -> + Enum.each([priority_inboxes, other_inboxes], fn inboxes -> Enum.each(inboxes, fn inbox -> %User{ap_id: ap_id} = Enum.find(recipients, fn actor -> actor.inbox == inbox end) From 8383584d692f56dfdd4b88d328d5d54962f82ed1 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 7 Jun 2025 14:57:34 -0700 Subject: [PATCH 08/74] Reapply "Nodeinfo is not universally implemented" This reverts commit 0fe03fc4eef0159e3015d68d75ec42ea11f649cf. --- lib/pleroma/workers/reachability_worker.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/workers/reachability_worker.ex b/lib/pleroma/workers/reachability_worker.ex index 3a11dfe2a..d9f764322 100644 --- a/lib/pleroma/workers/reachability_worker.ex +++ b/lib/pleroma/workers/reachability_worker.ex @@ -13,7 +13,7 @@ defmodule Pleroma.Workers.ReachabilityWorker do @impl true def perform(%Oban.Job{args: %{"domain" => domain}}) do - case HTTP.get("https://#{domain}/.well-known/nodeinfo") do + case HTTP.get("https://#{domain}/") do {:ok, %{status: status}} when status in 200..299 -> Instances.set_reachable("https://#{domain}") :ok From a46a48fb3f508da67b171b54c91bb027c13aa22b Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 7 Jun 2025 15:13:45 -0700 Subject: [PATCH 09/74] PublisherWorker: change max_attempts to 13 which extends the last delivery attempt to ~4.3 days --- lib/pleroma/workers/publisher_worker.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/workers/publisher_worker.ex b/lib/pleroma/workers/publisher_worker.ex index 10736bef5..f799af77a 100644 --- a/lib/pleroma/workers/publisher_worker.ex +++ b/lib/pleroma/workers/publisher_worker.ex @@ -7,7 +7,7 @@ defmodule Pleroma.Workers.PublisherWorker do alias Pleroma.Instances alias Pleroma.Web.Federator - use Oban.Worker, queue: :federator_outgoing, max_attempts: 5 + use Oban.Worker, queue: :federator_outgoing, max_attempts: 13 @impl true def perform(%Job{args: %{"op" => "publish", "activity_id" => activity_id}}) do From ee37b2d8c64b8fee5f6f1634ad817da64e631b98 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Tue, 17 Jun 2025 21:12:20 +0300 Subject: [PATCH 10/74] Return 404 when an activity is sent to a deactivated user's /inbox Also return 404 when the user who sent the activity is believed to be deactivated. It was already an error, now it just returns a better reason than "Invalid request". Also send proper errors when either user is not known at all. --- changelog.d/deactivated-404-inbox.change | 1 + .../activity_pub/activity_pub_controller.ex | 28 ++++++++- .../activity_pub_controller_test.exs | 61 +++++++++++++------ 3 files changed, 71 insertions(+), 19 deletions(-) create mode 100644 changelog.d/deactivated-404-inbox.change diff --git a/changelog.d/deactivated-404-inbox.change b/changelog.d/deactivated-404-inbox.change new file mode 100644 index 000000000..3912c53ef --- /dev/null +++ b/changelog.d/deactivated-404-inbox.change @@ -0,0 +1 @@ +Return 404 with a better error message instead of 400 when receiving an activity for a deactivated user \ No newline at end of file diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index 7ac0bbab4..71e0a24de 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -274,13 +274,37 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do end def inbox(%{assigns: %{valid_signature: true}} = conn, %{"nickname" => nickname} = params) do - with %User{is_active: true} = recipient <- User.get_cached_by_nickname(nickname), - {:ok, %User{is_active: true} = actor} <- User.get_or_fetch_by_ap_id(params["actor"]), + with {:recipient_exists, %User{} = recipient} <- + {:recipient_exists, User.get_cached_by_nickname(nickname)}, + {:sender_exists, {:ok, %User{} = actor}} <- + {:sender_exists, User.get_or_fetch_by_ap_id(params["actor"])}, + {:recipient_active, true} <- {:recipient_active, recipient.is_active}, + {:sender_active, true} <- {:sender_active, actor.is_active}, true <- Utils.recipient_in_message(recipient, actor, params), params <- Utils.maybe_splice_recipient(recipient.ap_id, params) do Federator.incoming_ap_doc(params) json(conn, "ok") else + {:recipient_exists, _} -> + conn + |> put_status(:not_found) + |> json("User does not exist") + + {:sender_exists, _} -> + conn + |> put_status(:not_found) + |> json("Sender does not exist") + + {:recipient_active, _} -> + conn + |> put_status(:not_found) + |> json("User deactivated") + + {:sender_active, _} -> + conn + |> put_status(:not_found) + |> json("Sender deactivated") + _ -> conn |> put_status(:bad_request) diff --git a/test/pleroma/web/activity_pub/activity_pub_controller_test.exs b/test/pleroma/web/activity_pub/activity_pub_controller_test.exs index 46b3d5f0d..ded5d9d99 100644 --- a/test/pleroma/web/activity_pub/activity_pub_controller_test.exs +++ b/test/pleroma/web/activity_pub/activity_pub_controller_test.exs @@ -941,23 +941,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do assert Activity.get_by_ap_id(data["id"]) end - test "it rejects an invalid incoming activity", %{conn: conn, data: data} do - user = insert(:user, is_active: false) - - data = - data - |> Map.put("bcc", [user.ap_id]) - |> Kernel.put_in(["object", "bcc"], [user.ap_id]) - - conn = - conn - |> assign(:valid_signature, true) - |> put_req_header("content-type", "application/activity+json") - |> post("/users/#{user.nickname}/inbox", data) - - assert "Invalid request." == json_response(conn, 400) - end - test "it accepts messages with to as string instead of array", %{conn: conn, data: data} do user = insert(:user) @@ -1341,6 +1324,50 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do ObanHelpers.perform(all_enqueued(worker: ReceiverWorker)) assert Activity.get_by_ap_id(data["id"]) end + + test "it returns an error when receiving an activity sent to a deactivated user", %{ + conn: conn, + data: data + } do + user = insert(:user) + {:ok, _} = User.set_activation(user, false) + + data = + data + |> Map.put("bcc", [user.ap_id]) + |> Kernel.put_in(["object", "bcc"], [user.ap_id]) + + conn = + conn + |> assign(:valid_signature, true) + |> put_req_header("content-type", "application/activity+json") + |> post("/users/#{user.nickname}/inbox", data) + + assert "User deactivated" == json_response(conn, 404) + end + + test "it returns an error when receiving an activity sent from a deactivated user", %{ + conn: conn, + data: data + } do + sender = insert(:user) + user = insert(:user) + {:ok, _} = User.set_activation(sender, false) + + data = + data + |> Map.put("bcc", [user.ap_id]) + |> Map.put("actor", sender.ap_id) + |> Kernel.put_in(["object", "bcc"], [user.ap_id]) + + conn = + conn + |> assign(:valid_signature, true) + |> put_req_header("content-type", "application/activity+json") + |> post("/users/#{user.nickname}/inbox", data) + + assert "Sender deactivated" == json_response(conn, 404) + end end describe "GET /users/:nickname/outbox" do From 59bfa83c9ce372b6413ff5498cad030b97a7af2d Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 27 Jun 2025 16:04:08 -0700 Subject: [PATCH 11/74] Remove daily reachability scheduling for unreachable instances --- config/config.exs | 3 +- .../cron/schedule_reachability_worker.ex | 33 ------------------- 2 files changed, 1 insertion(+), 35 deletions(-) delete mode 100644 lib/pleroma/workers/cron/schedule_reachability_worker.ex diff --git a/config/config.exs b/config/config.exs index 805cd0d62..372852a7b 100644 --- a/config/config.exs +++ b/config/config.exs @@ -603,8 +603,7 @@ config :pleroma, Oban, crontab: [ {"0 0 * * 0", Pleroma.Workers.Cron.DigestEmailsWorker}, {"0 0 * * *", Pleroma.Workers.Cron.NewUsersDigestWorker}, - {"*/10 * * * *", Pleroma.Workers.Cron.AppCleanupWorker}, - {"0 0 * * *", Pleroma.Workers.Cron.ScheduleReachabilityWorker} + {"*/10 * * * *", Pleroma.Workers.Cron.AppCleanupWorker} ] config :pleroma, Pleroma.Formatter, diff --git a/lib/pleroma/workers/cron/schedule_reachability_worker.ex b/lib/pleroma/workers/cron/schedule_reachability_worker.ex deleted file mode 100644 index a0b8e261c..000000000 --- a/lib/pleroma/workers/cron/schedule_reachability_worker.ex +++ /dev/null @@ -1,33 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2022 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Workers.Cron.ScheduleReachabilityWorker do - use Oban.Worker, - queue: :background, - max_attempts: 2 - - alias Pleroma.Instances - alias Pleroma.Repo - - @impl true - def perform(_job) do - unreachable_servers = Instances.get_unreachable() - - jobs = - unreachable_servers - |> Enum.map(fn {domain, _} -> - Pleroma.Workers.ReachabilityWorker.new(%{"domain" => domain}) - end) - - case Repo.transaction(fn -> - Enum.each(jobs, &Oban.insert/1) - end) do - {:ok, _} -> - :ok - - {:error, reason} -> - {:error, reason} - end - end -end From 77dca7c3e59053505abb4fa757b2d97e227fa4f4 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 27 Jun 2025 16:35:10 -0700 Subject: [PATCH 12/74] Refactor ReachabilityWorker to use a 5-phase reachability testing approach It will check reachability for an instance deemed unreachable at the following intervals: 4 attempts, once a minute 4 attempts, once every 15 minutes 4 attempts, once every 60 minutes 4 attempts, once every 8 hours 4 attempts, once every 24 hours This should be effective and respectful of the resources of instances on the fediverse. We have the Oban Pruner plugin enabled to keep the Oban Jobs table from growing indefinitely. It prunes every 15 minutes, but this will interfere with our ability to enforce uniqueness on the ReachabilityWorker jobs for a time period longer than 15 minutes. The solution is to exclude the ReachabilityWorker from the pruning operation and instead schedule a custom job that will prune the table for us once a day. The ReachabilityPruner cron task will clean up the history of the ReachabilityWorker jobs older than 6 days. --- config/config.exs | 5 +- .../workers/cron/reachability_pruner.ex | 26 +++ lib/pleroma/workers/reachability_worker.ex | 71 +++++- .../schedule_reachability_worker_test.exs | 52 ----- .../workers/reachability_worker_test.exs | 202 ++++++++++++++++++ 5 files changed, 296 insertions(+), 60 deletions(-) create mode 100644 lib/pleroma/workers/cron/reachability_pruner.ex delete mode 100644 test/pleroma/workers/cron/schedule_reachability_worker_test.exs create mode 100644 test/pleroma/workers/reachability_worker_test.exs diff --git a/config/config.exs b/config/config.exs index 372852a7b..f58dfb1af 100644 --- a/config/config.exs +++ b/config/config.exs @@ -599,11 +599,12 @@ config :pleroma, Oban, search_indexing: [limit: 10, paused: true], slow: 5 ], - plugins: [{Oban.Plugins.Pruner, max_age: 900}], + plugins: [{Oban.Plugins.Pruner, max_age: 900, exclude: [Pleroma.Workers.ReachabilityWorker]}], crontab: [ {"0 0 * * 0", Pleroma.Workers.Cron.DigestEmailsWorker}, {"0 0 * * *", Pleroma.Workers.Cron.NewUsersDigestWorker}, - {"*/10 * * * *", Pleroma.Workers.Cron.AppCleanupWorker} + {"*/10 * * * *", Pleroma.Workers.Cron.AppCleanupWorker}, + {"0 2 * * *", Pleroma.Workers.Cron.ReachabilityPruner} ] config :pleroma, Pleroma.Formatter, diff --git a/lib/pleroma/workers/cron/reachability_pruner.ex b/lib/pleroma/workers/cron/reachability_pruner.ex new file mode 100644 index 000000000..6eb671e0e --- /dev/null +++ b/lib/pleroma/workers/cron/reachability_pruner.ex @@ -0,0 +1,26 @@ +defmodule Pleroma.Workers.Cron.ReachabilityPruner do + use Oban.Worker, queue: :background, max_attempts: 1 + + import Ecto.Query + require Logger + + @reachability_worker "Elixir.Pleroma.Workers.ReachabilityWorker" + @prune_days 6 + + @impl true + def perform(_job) do + cutoff = DateTime.utc_now() |> DateTime.add(-@prune_days * 24 * 60 * 60, :second) + + {count, _} = + from(j in Oban.Job, + where: j.worker == @reachability_worker and j.inserted_at < ^cutoff + ) + |> Pleroma.Repo.delete_all() + + if count > 0 do + Logger.debug(fn -> "Pruned #{count} old ReachabilityWorker jobs." end) + end + + :ok + end +end diff --git a/lib/pleroma/workers/reachability_worker.ex b/lib/pleroma/workers/reachability_worker.ex index d9f764322..ba6928dee 100644 --- a/lib/pleroma/workers/reachability_worker.ex +++ b/lib/pleroma/workers/reachability_worker.ex @@ -5,17 +5,31 @@ defmodule Pleroma.Workers.ReachabilityWorker do use Oban.Worker, queue: :background, - max_attempts: 3, - unique: [period: :infinity, states: [:available, :scheduled]] + max_attempts: 1, + unique: [period: :infinity, states: [:available, :scheduled], keys: [:domain]] alias Pleroma.HTTP alias Pleroma.Instances @impl true - def perform(%Oban.Job{args: %{"domain" => domain}}) do + def perform(%Oban.Job{args: %{"domain" => domain, "phase" => phase, "attempt" => attempt}}) do + case check_reachability(domain) do + :ok -> + Instances.set_reachable("https://#{domain}") + :ok + + {:error, _} = error -> + handle_failed_attempt(domain, phase, attempt) + error + end + end + + @impl true + def timeout(_job), do: :timer.seconds(5) + + defp check_reachability(domain) do case HTTP.get("https://#{domain}/") do {:ok, %{status: status}} when status in 200..299 -> - Instances.set_reachable("https://#{domain}") :ok {:ok, %{status: _status}} -> @@ -26,6 +40,51 @@ defmodule Pleroma.Workers.ReachabilityWorker do end end - @impl true - def timeout(_job), do: :timer.seconds(5) + defp handle_failed_attempt(_domain, "final", _attempt), do: :ok + + defp handle_failed_attempt(domain, phase, attempt) do + {interval_minutes, max_attempts, next_phase} = get_phase_config(phase) + + if attempt >= max_attempts do + # Move to next phase + schedule_next_phase(domain, next_phase) + else + # Retry same phase with incremented attempt + schedule_retry(domain, phase, attempt + 1, interval_minutes) + end + end + + defp get_phase_config("phase_1min"), do: {1, 4, "phase_15min"} + defp get_phase_config("phase_15min"), do: {15, 4, "phase_1hour"} + defp get_phase_config("phase_1hour"), do: {60, 4, "phase_8hour"} + defp get_phase_config("phase_8hour"), do: {480, 4, "phase_24hour"} + defp get_phase_config("phase_24hour"), do: {1440, 4, "final"} + defp get_phase_config("final"), do: {nil, 0, nil} + + defp schedule_next_phase(_domain, "final"), do: :ok + + defp schedule_next_phase(domain, next_phase) do + {interval_minutes, _max_attempts, _next_phase} = get_phase_config(next_phase) + scheduled_at = DateTime.add(DateTime.utc_now(), interval_minutes * 60, :second) + + %{ + "domain" => domain, + "phase" => next_phase, + "attempt" => 1 + } + |> new(scheduled_at: scheduled_at, replace: true) + |> Oban.insert() + end + + def schedule_retry(domain, phase, attempt, interval_minutes) do + scheduled_at = DateTime.add(DateTime.utc_now(), interval_minutes * 60, :second) + + %{ + "domain" => domain, + "phase" => phase, + "attempt" => attempt + } + |> new(scheduled_at: scheduled_at, replace: true) + |> Oban.insert() + end end diff --git a/test/pleroma/workers/cron/schedule_reachability_worker_test.exs b/test/pleroma/workers/cron/schedule_reachability_worker_test.exs deleted file mode 100644 index 310c2e61a..000000000 --- a/test/pleroma/workers/cron/schedule_reachability_worker_test.exs +++ /dev/null @@ -1,52 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2022 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Workers.Cron.ScheduleReachabilityWorkerTest do - use Pleroma.DataCase, async: true - use Oban.Testing, repo: Pleroma.Repo - - alias Pleroma.Instances - alias Pleroma.Workers.Cron.ScheduleReachabilityWorker - - describe "perform/1" do - test "schedules reachability checks for unreachable servers" do - # Mark some servers as unreachable - Instances.set_unreachable("https://example.com") - Instances.set_unreachable("https://test.com") - Instances.set_unreachable("https://another.com") - - # Verify they are marked as unreachable - refute Instances.reachable?("https://example.com") - refute Instances.reachable?("https://test.com") - refute Instances.reachable?("https://another.com") - - # Run the worker - assert :ok = ScheduleReachabilityWorker.perform(%Oban.Job{}) - - # Verify ReachabilityWorker jobs were scheduled for each server - # Note: domains in get_unreachable/0 are without the https:// prefix - assert_enqueued( - worker: Pleroma.Workers.ReachabilityWorker, - args: %{"domain" => "example.com"} - ) - - assert_enqueued( - worker: Pleroma.Workers.ReachabilityWorker, - args: %{"domain" => "test.com"} - ) - - assert_enqueued( - worker: Pleroma.Workers.ReachabilityWorker, - args: %{"domain" => "another.com"} - ) - end - - test "handles empty list of unreachable servers" do - # Ensure no servers are marked as unreachable - assert [] = Instances.get_unreachable() - assert :ok = ScheduleReachabilityWorker.perform(%Oban.Job{}) - refute_enqueued(worker: Pleroma.Workers.ReachabilityWorker) - end - end -end diff --git a/test/pleroma/workers/reachability_worker_test.exs b/test/pleroma/workers/reachability_worker_test.exs new file mode 100644 index 000000000..32c39e869 --- /dev/null +++ b/test/pleroma/workers/reachability_worker_test.exs @@ -0,0 +1,202 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Workers.ReachabilityWorkerTest do + use Pleroma.DataCase, async: true + use Oban.Testing, repo: Pleroma.Repo + + import Mock + + alias Pleroma.Tests.ObanHelpers + alias Pleroma.Workers.ReachabilityWorker + + setup do + ObanHelpers.wipe_all() + :ok + end + + describe "progressive backoff phases" do + test "starts with phase_1min and progresses through phases on failure" do + domain = "example.com" + + with_mocks([ + {Pleroma.HTTP, [], [get: fn _ -> {:error, :timeout} end]}, + {Pleroma.Instances, [], [set_reachable: fn _ -> :ok end]} + ]) do + # Start with phase_1min + job = %Oban.Job{ + args: %{"domain" => domain, "phase" => "phase_1min", "attempt" => 1} + } + + # First attempt fails + assert {:error, :timeout} = ReachabilityWorker.perform(job) + + # Should schedule retry for phase_1min (attempt 2) + retry_jobs = all_enqueued(worker: ReachabilityWorker) + assert length(retry_jobs) == 1 + [retry_job] = retry_jobs + assert retry_job.args["phase"] == "phase_1min" + assert retry_job.args["attempt"] == 2 + + # Clear jobs and simulate second attempt failure + ObanHelpers.wipe_all() + + retry_job = %Oban.Job{ + args: %{"domain" => domain, "phase" => "phase_1min", "attempt" => 2} + } + + assert {:error, :timeout} = ReachabilityWorker.perform(retry_job) + + # Should schedule retry for phase_1min (attempt 3) + retry_jobs = all_enqueued(worker: ReachabilityWorker) + assert length(retry_jobs) == 1 + [retry_job] = retry_jobs + assert retry_job.args["phase"] == "phase_1min" + assert retry_job.args["attempt"] == 3 + + # Clear jobs and simulate third attempt failure (final attempt for phase_1min) + ObanHelpers.wipe_all() + + retry_job = %Oban.Job{ + args: %{"domain" => domain, "phase" => "phase_1min", "attempt" => 3} + } + + assert {:error, :timeout} = ReachabilityWorker.perform(retry_job) + + # Should schedule retry for phase_1min (attempt 4) + retry_jobs = all_enqueued(worker: ReachabilityWorker) + assert length(retry_jobs) == 1 + [retry_job] = retry_jobs + assert retry_job.args["phase"] == "phase_1min" + assert retry_job.args["attempt"] == 4 + + # Clear jobs and simulate fourth attempt failure (final attempt for phase_1min) + ObanHelpers.wipe_all() + + retry_job = %Oban.Job{ + args: %{"domain" => domain, "phase" => "phase_1min", "attempt" => 4} + } + + assert {:error, :timeout} = ReachabilityWorker.perform(retry_job) + + # Should schedule next phase (phase_15min) + next_phase_jobs = all_enqueued(worker: ReachabilityWorker) + assert length(next_phase_jobs) == 1 + [next_phase_job] = next_phase_jobs + assert next_phase_job.args["phase"] == "phase_15min" + assert next_phase_job.args["attempt"] == 1 + end + end + + test "progresses through all phases correctly" do + domain = "example.com" + + with_mocks([ + {Pleroma.HTTP, [], [get: fn _ -> {:error, :timeout} end]}, + {Pleroma.Instances, [], [set_reachable: fn _ -> :ok end]} + ]) do + # Simulate all phases failing + phases = ["phase_1min", "phase_15min", "phase_1hour", "phase_8hour", "phase_24hour"] + + Enum.each(phases, fn phase -> + {_interval, max_attempts, next_phase} = get_phase_config(phase) + + # Simulate all attempts failing for this phase + Enum.each(1..max_attempts, fn attempt -> + job = %Oban.Job{args: %{"domain" => domain, "phase" => phase, "attempt" => attempt}} + assert {:error, :timeout} = ReachabilityWorker.perform(job) + + if attempt < max_attempts do + # Should schedule retry for same phase + retry_jobs = all_enqueued(worker: ReachabilityWorker) + assert length(retry_jobs) == 1 + [retry_job] = retry_jobs + assert retry_job.args["phase"] == phase + assert retry_job.args["attempt"] == attempt + 1 + ObanHelpers.wipe_all() + else + # Should schedule next phase (except for final phase) + if next_phase != "final" do + next_phase_jobs = all_enqueued(worker: ReachabilityWorker) + assert length(next_phase_jobs) == 1 + [next_phase_job] = next_phase_jobs + assert next_phase_job.args["phase"] == next_phase + assert next_phase_job.args["attempt"] == 1 + ObanHelpers.wipe_all() + else + # Final phase - no more jobs should be scheduled + next_phase_jobs = all_enqueued(worker: ReachabilityWorker) + assert length(next_phase_jobs) == 0 + end + end + end) + end) + end + end + + test "succeeds and stops progression when instance becomes reachable" do + domain = "example.com" + + with_mocks([ + {Pleroma.HTTP, [], [get: fn _ -> {:ok, %{status: 200}} end]}, + {Pleroma.Instances, [], [set_reachable: fn _ -> :ok end]} + ]) do + job = %Oban.Job{args: %{"domain" => domain, "phase" => "phase_1hour", "attempt" => 2}} + + # Should succeed and not schedule any more jobs + assert :ok = ReachabilityWorker.perform(job) + + # Verify set_reachable was called + assert_called(Pleroma.Instances.set_reachable("https://#{domain}")) + + # No more jobs should be scheduled + next_jobs = all_enqueued(worker: ReachabilityWorker) + assert length(next_jobs) == 0 + end + end + + test "enforces uniqueness per domain using Oban's conflict detection" do + domain = "example.com" + + # Insert first job for the domain + job1 = + %{ + "domain" => domain, + "phase" => "phase_1min", + "attempt" => 1 + } + |> ReachabilityWorker.new() + |> Oban.insert() + + assert {:ok, _} = job1 + + # Try to insert a second job for the same domain with different phase/attempt + job2 = + %{ + "domain" => domain, + "phase" => "phase_15min", + "attempt" => 1 + } + |> ReachabilityWorker.new() + |> Oban.insert() + + # Should fail due to uniqueness constraint (conflict) + assert {:ok, %Oban.Job{conflict?: true}} = job2 + + # Verify only one job exists for this domain + jobs = all_enqueued(worker: ReachabilityWorker) + assert length(jobs) == 1 + [existing_job] = jobs + assert existing_job.args["domain"] == domain + assert existing_job.args["phase"] == "phase_1min" + end + end + + defp get_phase_config("phase_1min"), do: {1, 4, "phase_15min"} + defp get_phase_config("phase_15min"), do: {15, 4, "phase_1hour"} + defp get_phase_config("phase_1hour"), do: {60, 4, "phase_8hour"} + defp get_phase_config("phase_8hour"), do: {480, 4, "phase_24hour"} + defp get_phase_config("phase_24hour"), do: {1440, 4, "final"} + defp get_phase_config("final"), do: {nil, 0, nil} +end From 6e4b5edc257aa7555dd7c82d2884e0beac0c60ac Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 27 Jun 2025 17:13:34 -0700 Subject: [PATCH 13/74] Reduce pruning of history to anything older than 2 days --- lib/pleroma/workers/cron/reachability_pruner.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/workers/cron/reachability_pruner.ex b/lib/pleroma/workers/cron/reachability_pruner.ex index 6eb671e0e..51cfdad3c 100644 --- a/lib/pleroma/workers/cron/reachability_pruner.ex +++ b/lib/pleroma/workers/cron/reachability_pruner.ex @@ -5,7 +5,7 @@ defmodule Pleroma.Workers.Cron.ReachabilityPruner do require Logger @reachability_worker "Elixir.Pleroma.Workers.ReachabilityWorker" - @prune_days 6 + @prune_days 2 @impl true def perform(_job) do From a5e11ad1101dc86309152666b855e1c065a6eabc Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 27 Jun 2025 17:24:02 -0700 Subject: [PATCH 14/74] Custom pruning is not actually needed because an old job cannot exist in the table due to our use of [replace: true] when retrying jobs or walking it through the different phases --- config/config.exs | 5 ++-- .../workers/cron/reachability_pruner.ex | 26 ------------------- 2 files changed, 2 insertions(+), 29 deletions(-) delete mode 100644 lib/pleroma/workers/cron/reachability_pruner.ex diff --git a/config/config.exs b/config/config.exs index f58dfb1af..372852a7b 100644 --- a/config/config.exs +++ b/config/config.exs @@ -599,12 +599,11 @@ config :pleroma, Oban, search_indexing: [limit: 10, paused: true], slow: 5 ], - plugins: [{Oban.Plugins.Pruner, max_age: 900, exclude: [Pleroma.Workers.ReachabilityWorker]}], + plugins: [{Oban.Plugins.Pruner, max_age: 900}], crontab: [ {"0 0 * * 0", Pleroma.Workers.Cron.DigestEmailsWorker}, {"0 0 * * *", Pleroma.Workers.Cron.NewUsersDigestWorker}, - {"*/10 * * * *", Pleroma.Workers.Cron.AppCleanupWorker}, - {"0 2 * * *", Pleroma.Workers.Cron.ReachabilityPruner} + {"*/10 * * * *", Pleroma.Workers.Cron.AppCleanupWorker} ] config :pleroma, Pleroma.Formatter, diff --git a/lib/pleroma/workers/cron/reachability_pruner.ex b/lib/pleroma/workers/cron/reachability_pruner.ex deleted file mode 100644 index 51cfdad3c..000000000 --- a/lib/pleroma/workers/cron/reachability_pruner.ex +++ /dev/null @@ -1,26 +0,0 @@ -defmodule Pleroma.Workers.Cron.ReachabilityPruner do - use Oban.Worker, queue: :background, max_attempts: 1 - - import Ecto.Query - require Logger - - @reachability_worker "Elixir.Pleroma.Workers.ReachabilityWorker" - @prune_days 2 - - @impl true - def perform(_job) do - cutoff = DateTime.utc_now() |> DateTime.add(-@prune_days * 24 * 60 * 60, :second) - - {count, _} = - from(j in Oban.Job, - where: j.worker == @reachability_worker and j.inserted_at < ^cutoff - ) - |> Pleroma.Repo.delete_all() - - if count > 0 do - Logger.debug(fn -> "Pruned #{count} old ReachabilityWorker jobs." end) - end - - :ok - end -end From 13db730659c7abd902cd7d59aecaf1bb36ab58d2 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 27 Jun 2025 17:52:00 -0700 Subject: [PATCH 15/74] Update Oban to 2.19 which gives us the delete_all_jobs/1 and delete_job/1 functions --- mix.exs | 2 +- mix.lock | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mix.exs b/mix.exs index 971084f94..b137802e4 100644 --- a/mix.exs +++ b/mix.exs @@ -136,7 +136,7 @@ defmodule Pleroma.Mixfile do {:telemetry_poller, "~> 1.0"}, {:tzdata, "~> 1.0.3"}, {:plug_cowboy, "~> 2.5"}, - {:oban, "~> 2.18.0"}, + {:oban, "~> 2.19.0"}, {:gettext, "~> 0.20"}, {:bcrypt_elixir, "~> 2.2"}, {:trailing_format_plug, "~> 0.0.7"}, diff --git a/mix.lock b/mix.lock index f7f37b7e1..7e86b5683 100644 --- a/mix.lock +++ b/mix.lock @@ -26,8 +26,8 @@ "credo": {:hex, :credo, "1.7.12", "9e3c20463de4b5f3f23721527fcaf16722ec815e70ff6c60b86412c695d426c1", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "8493d45c656c5427d9c729235b99d498bd133421f3e0a683e5c1b561471291e5"}, "crontab": {:hex, :crontab, "1.1.8", "2ce0e74777dfcadb28a1debbea707e58b879e6aa0ffbf9c9bb540887bce43617", [:mix], [{:ecto, "~> 1.0 or ~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm"}, "custom_base": {:hex, :custom_base, "0.2.1", "4a832a42ea0552299d81652aa0b1f775d462175293e99dfbe4d7dbaab785a706", [:mix], [], "hexpm", "8df019facc5ec9603e94f7270f1ac73ddf339f56ade76a721eaa57c1493ba463"}, - "db_connection": {:hex, :db_connection, "2.7.0", "b99faa9291bb09892c7da373bb82cba59aefa9b36300f6145c5f201c7adf48ec", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "dcf08f31b2701f857dfc787fbad78223d61a32204f217f15e881dd93e4bdd3ff"}, - "decimal": {:hex, :decimal, "2.1.1", "5611dca5d4b2c3dd497dec8f68751f1f1a54755e8ed2a966c2633cf885973ad6", [:mix], [], "hexpm", "53cfe5f497ed0e7771ae1a475575603d77425099ba5faef9394932b35020ffcc"}, + "db_connection": {:hex, :db_connection, "2.8.0", "64fd82cfa6d8e25ec6660cea73e92a4cbc6a18b31343910427b702838c4b33b2", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "008399dae5eee1bf5caa6e86d204dcb44242c82b1ed5e22c881f2c34da201b15"}, + "decimal": {:hex, :decimal, "2.3.0", "3ad6255aa77b4a3c4f818171b12d237500e63525c2fd056699967a3e7ea20f62", [:mix], [], "hexpm", "a4d66355cb29cb47c3cf30e71329e58361cfcb37c34235ef3bf1d7bf3773aeac"}, "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm", "ce708e5f094b9cd4e8f2be4f00d2f4250c4095be93f8cd6d018c753894885430"}, "dialyxir": {:hex, :dialyxir, "1.4.3", "edd0124f358f0b9e95bfe53a9fcf806d615d8f838e2202a9f430d59566b6b53b", [:mix], [{:erlex, ">= 0.2.6", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "bf2cfb75cd5c5006bec30141b131663299c661a864ec7fbbc72dfa557487a986"}, "earmark": {:hex, :earmark, "1.4.46", "8c7287bd3137e99d26ae4643e5b7ef2129a260e3dcf41f251750cb4563c8fb81", [:mix], [], "hexpm", "798d86db3d79964e759ddc0c077d5eb254968ed426399fbf5a62de2b5ff8910a"}, @@ -92,7 +92,7 @@ "nimble_parsec": {:hex, :nimble_parsec, "0.6.0", "32111b3bf39137144abd7ba1cce0914533b2d16ef35e8abc5ec8be6122944263", [:mix], [], "hexpm", "27eac315a94909d4dc68bc07a4a83e06c8379237c5ea528a9acff4ca1c873c52"}, "nimble_pool": {:hex, :nimble_pool, "0.2.6", "91f2f4c357da4c4a0a548286c84a3a28004f68f05609b4534526871a22053cde", [:mix], [], "hexpm", "1c715055095d3f2705c4e236c18b618420a35490da94149ff8b580a2144f653f"}, "nodex": {:git, "https://git.pleroma.social/pleroma/nodex", "cb6730f943cfc6aad674c92161be23a8411f15d1", [ref: "cb6730f943cfc6aad674c92161be23a8411f15d1"]}, - "oban": {:hex, :oban, "2.18.3", "1608c04f8856c108555c379f2f56bc0759149d35fa9d3b825cb8a6769f8ae926", [:mix], [{:ecto_sql, "~> 3.10", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:ecto_sqlite3, "~> 0.9", [hex: :ecto_sqlite3, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.16", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "36ca6ca84ef6518f9c2c759ea88efd438a3c81d667ba23b02b062a0aa785475e"}, + "oban": {:hex, :oban, "2.19.4", "045adb10db1161dceb75c254782f97cdc6596e7044af456a59decb6d06da73c1", [:mix], [{:ecto_sql, "~> 3.10", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:ecto_sqlite3, "~> 0.9", [hex: :ecto_sqlite3, repo: "hexpm", optional: true]}, {:igniter, "~> 0.5", [hex: :igniter, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: true]}, {:myxql, "~> 0.7", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.16", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "5fcc6219e6464525b808d97add17896e724131f498444a292071bf8991c99f97"}, "oban_live_dashboard": {:hex, :oban_live_dashboard, "0.1.1", "8aa4ceaf381c818f7d5c8185cc59942b8ac82ef0cf559881aacf8d3f8ac7bdd3", [:mix], [{:oban, "~> 2.15", [hex: :oban, repo: "hexpm", optional: false]}, {:phoenix_live_dashboard, "~> 0.7", [hex: :phoenix_live_dashboard, repo: "hexpm", optional: false]}], "hexpm", "16dc4ce9c9a95aa2e655e35ed4e675652994a8def61731a18af85e230e1caa63"}, "octo_fetch": {:hex, :octo_fetch, "0.4.0", "074b5ecbc08be10b05b27e9db08bc20a3060142769436242702931c418695b19", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~> 1.1", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm", "cf8be6f40cd519d7000bb4e84adcf661c32e59369ca2827c4e20042eda7a7fc6"}, "open_api_spex": {:hex, :open_api_spex, "3.18.2", "8c855e83bfe8bf81603d919d6e892541eafece3720f34d1700b58024dadde247", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:poison, "~> 3.0 or ~> 4.0 or ~> 5.0", [hex: :poison, repo: "hexpm", optional: true]}, {:ymlr, "~> 2.0 or ~> 3.0 or ~> 4.0", [hex: :ymlr, repo: "hexpm", optional: true]}], "hexpm", "aa3e6dcfc0ad6a02596b2172662da21c9dd848dac145ea9e603f54e3d81b8d2b"}, From ff5f88aae314a61f4c766762056094216e00b89d Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 27 Jun 2025 18:07:46 -0700 Subject: [PATCH 16/74] Instance.set_reachable/1 should delete any existing ReachabilityWorker jobs for that instance --- lib/pleroma/instances/instance.ex | 17 +++++++++++++--- test/pleroma/instances/instance_test.exs | 26 ++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex index fb0b9d7f0..620544134 100644 --- a/lib/pleroma/instances/instance.ex +++ b/lib/pleroma/instances/instance.ex @@ -92,9 +92,20 @@ defmodule Pleroma.Instances.Instance do def reachable?(url_or_host) when is_binary(url_or_host), do: true def set_reachable(url_or_host) when is_binary(url_or_host) do - %Instance{host: host(url_or_host)} - |> changeset(%{unreachable_since: nil}) - |> Repo.insert(on_conflict: {:replace, [:unreachable_since]}, conflict_target: :host) + host = host(url_or_host) + + result = + %Instance{host: host} + |> changeset(%{unreachable_since: nil}) + |> Repo.insert(on_conflict: {:replace, [:unreachable_since]}, conflict_target: :host) + + # Delete any existing reachability testing jobs for this instance + Oban.Job + |> Ecto.Query.where(worker: "Pleroma.Workers.ReachabilityWorker") + |> Ecto.Query.where([j], j.args["domain"] == ^host) + |> Oban.delete_all_jobs() + + result end def set_reachable(_), do: {:error, nil} diff --git a/test/pleroma/instances/instance_test.exs b/test/pleroma/instances/instance_test.exs index ed536c55c..354ba139a 100644 --- a/test/pleroma/instances/instance_test.exs +++ b/test/pleroma/instances/instance_test.exs @@ -27,6 +27,32 @@ defmodule Pleroma.Instances.InstanceTest do assert {:ok, instance} = Instance.set_reachable(instance.host) refute instance.unreachable_since end + + test "cancels all ReachabilityWorker jobs for the domain" do + domain = "cancelme.example.org" + insert(:instance, host: domain, unreachable_since: NaiveDateTime.utc_now()) + + # Insert a ReachabilityWorker job for this domain, scheduled 5 minutes in the future + scheduled_at = DateTime.add(DateTime.utc_now(), 300, :second) + + {:ok, job} = + Pleroma.Workers.ReachabilityWorker.new( + %{"domain" => domain, "phase" => "phase_1min", "attempt" => 1}, + scheduled_at: scheduled_at + ) + |> Oban.insert() + + # Ensure the job is present + job = Pleroma.Repo.get(Oban.Job, job.id) + assert job + + # Call set_reachable, which should delete the job + assert {:ok, _} = Instance.set_reachable(domain) + + # Reload the job and assert it is deleted + job = Pleroma.Repo.get(Oban.Job, job.id) + refute job + end end describe "set_unreachable/1" do From 2267ace10687d9289750932a7809fb7e5c4cc496 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 27 Jun 2025 18:12:18 -0700 Subject: [PATCH 17/74] Ensure ReachabilityWorker jobs can be scheduled without needing awareness of the phase design --- lib/pleroma/workers/reachability_worker.ex | 16 +++++++++++++ .../workers/reachability_worker_test.exs | 24 +++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/lib/pleroma/workers/reachability_worker.ex b/lib/pleroma/workers/reachability_worker.ex index ba6928dee..badfa476c 100644 --- a/lib/pleroma/workers/reachability_worker.ex +++ b/lib/pleroma/workers/reachability_worker.ex @@ -24,6 +24,22 @@ defmodule Pleroma.Workers.ReachabilityWorker do end end + # New jobs enter here and are immediately re-scheduled for the first phase + @impl true + def perform(%Oban.Job{args: %{"domain" => domain}}) do + scheduled_at = DateTime.add(DateTime.utc_now(), 60, :second) + + %{ + "domain" => domain, + "phase" => "phase_1min", + "attempt" => 1 + } + |> new(scheduled_at: scheduled_at, replace: true) + |> Oban.insert() + + :ok + end + @impl true def timeout(_job), do: :timer.seconds(5) diff --git a/test/pleroma/workers/reachability_worker_test.exs b/test/pleroma/workers/reachability_worker_test.exs index 32c39e869..4854aff77 100644 --- a/test/pleroma/workers/reachability_worker_test.exs +++ b/test/pleroma/workers/reachability_worker_test.exs @@ -191,6 +191,30 @@ defmodule Pleroma.Workers.ReachabilityWorkerTest do assert existing_job.args["domain"] == domain assert existing_job.args["phase"] == "phase_1min" end + + test "handles new jobs with only domain argument and transitions them to the first phase" do + domain = "legacy.example.com" + + with_mocks([ + {Pleroma.Instances, [], [set_reachable: fn _ -> :ok end]} + ]) do + # Create a job with only domain (legacy format) + job = %Oban.Job{ + args: %{"domain" => domain} + } + + # Should reschedule with phase_1min and attempt 1 + assert :ok = ReachabilityWorker.perform(job) + + # Check that a new job was scheduled with the correct format + scheduled_jobs = all_enqueued(worker: ReachabilityWorker) + assert length(scheduled_jobs) == 1 + [scheduled_job] = scheduled_jobs + assert scheduled_job.args["domain"] == domain + assert scheduled_job.args["phase"] == "phase_1min" + assert scheduled_job.args["attempt"] == 1 + end + end end defp get_phase_config("phase_1min"), do: {1, 4, "phase_15min"} From 8a0551686238af40ac21a2a9152f2a218c69d04e Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 27 Jun 2025 18:23:24 -0700 Subject: [PATCH 18/74] Remove changelog entry that leaked in via 3984ba87217e2a9fdc89c22ff2357c49563c5ad2 --- changelog.d/fix-public-url-addressing.fix | 1 - 1 file changed, 1 deletion(-) delete mode 100644 changelog.d/fix-public-url-addressing.fix diff --git a/changelog.d/fix-public-url-addressing.fix b/changelog.d/fix-public-url-addressing.fix deleted file mode 100644 index 810b76905..000000000 --- a/changelog.d/fix-public-url-addressing.fix +++ /dev/null @@ -1 +0,0 @@ -- Fixed an issue where the ActivityStreams Public collection URL was being removed from incoming activities' cc fields From 29f76079107f12e14b87e58c804ff10550381478 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 28 Jun 2025 12:51:10 -0700 Subject: [PATCH 19/74] Add Instances.check_all_unreachable/0 and Instance.check_unreachable/1 --- lib/pleroma/instances.ex | 9 ++++++ lib/pleroma/instances/instance.ex | 6 ++++ test/pleroma/instances/instance_test.exs | 30 ++++++++++++++++++ test/pleroma/instances_test.exs | 40 ++++++++++++++++++++++++ 4 files changed, 85 insertions(+) diff --git a/lib/pleroma/instances.ex b/lib/pleroma/instances.ex index 9237e0944..a69554ada 100644 --- a/lib/pleroma/instances.ex +++ b/lib/pleroma/instances.ex @@ -24,4 +24,13 @@ defmodule Pleroma.Instances do url_or_host end end + + @doc "Schedules reachability checks for all unreachable instances" + def check_all_unreachable do + get_unreachable() + |> Enum.each(fn {domain, _} -> + Pleroma.Workers.ReachabilityWorker.new(%{"domain" => domain}) + |> Oban.insert() + end) + end end diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex index 620544134..dca30275b 100644 --- a/lib/pleroma/instances/instance.ex +++ b/lib/pleroma/instances/instance.ex @@ -304,4 +304,10 @@ defmodule Pleroma.Instances.Instance do DeleteWorker.new(%{"op" => "delete_instance", "host" => host}) |> Oban.insert() end + + @doc "Schedules reachability check for instance" + def check_unreachable(domain) when is_binary(domain) do + Pleroma.Workers.ReachabilityWorker.new(%{"domain" => domain}) + |> Oban.insert() + end end diff --git a/test/pleroma/instances/instance_test.exs b/test/pleroma/instances/instance_test.exs index 354ba139a..83e70ac38 100644 --- a/test/pleroma/instances/instance_test.exs +++ b/test/pleroma/instances/instance_test.exs @@ -249,4 +249,34 @@ defmodule Pleroma.Instances.InstanceTest do args: %{"op" => "delete_instance", "host" => "mushroom.kingdom"} ) end + + describe "check_unreachable/1" do + test "schedules a ReachabilityWorker job for the given domain" do + domain = "test.example.com" + + # Call check_unreachable + assert {:ok, _job} = Instance.check_unreachable(domain) + + # Verify that a ReachabilityWorker job was scheduled + jobs = all_enqueued(worker: Pleroma.Workers.ReachabilityWorker) + assert length(jobs) == 1 + [job] = jobs + assert job.args["domain"] == domain + end + + test "handles multiple calls for the same domain (uniqueness enforced)" do + domain = "duplicate.example.com" + + assert {:ok, _job1} = Instance.check_unreachable(domain) + + # Second call for the same domain + assert {:ok, %Oban.Job{conflict?: true}} = Instance.check_unreachable(domain) + + # Should only have one job due to uniqueness + jobs = all_enqueued(worker: Pleroma.Workers.ReachabilityWorker) + assert length(jobs) == 1 + [job] = jobs + assert job.args["domain"] == domain + end + end end diff --git a/test/pleroma/instances_test.exs b/test/pleroma/instances_test.exs index cbafbfa44..8e23fd096 100644 --- a/test/pleroma/instances_test.exs +++ b/test/pleroma/instances_test.exs @@ -6,6 +6,7 @@ defmodule Pleroma.InstancesTest do alias Pleroma.Instances use Pleroma.DataCase + use Oban.Testing, repo: Pleroma.Repo describe "reachable?/1" do test "returns `true` for host / url with unknown reachability status" do @@ -69,4 +70,43 @@ defmodule Pleroma.InstancesTest do assert {:error, _} = Instances.set_unreachable(1) end end + + describe "check_all_unreachable/0" do + test "schedules ReachabilityWorker jobs for all unreachable instances" do + domain1 = "unreachable1.example.com" + domain2 = "unreachable2.example.com" + domain3 = "unreachable3.example.com" + + Instances.set_unreachable(domain1) + Instances.set_unreachable(domain2) + Instances.set_unreachable(domain3) + + Instances.check_all_unreachable() + + # Verify that ReachabilityWorker jobs were scheduled for all unreachable domains + jobs = all_enqueued(worker: Pleroma.Workers.ReachabilityWorker) + assert length(jobs) == 3 + + domains = Enum.map(jobs, & &1.args["domain"]) + assert domain1 in domains + assert domain2 in domains + assert domain3 in domains + end + + test "does not schedule jobs for reachable instances" do + unreachable_domain = "unreachable.example.com" + reachable_domain = "reachable.example.com" + + Instances.set_unreachable(unreachable_domain) + Instances.set_reachable(reachable_domain) + + Instances.check_all_unreachable() + + # Verify that only one job was scheduled (for the unreachable domain) + jobs = all_enqueued(worker: Pleroma.Workers.ReachabilityWorker) + assert length(jobs) == 1 + [job] = jobs + assert job.args["domain"] == unreachable_domain + end + end end From f06f0bedd305706ba8dd7cc38d421e2831f43d0b Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 28 Jun 2025 13:14:01 -0700 Subject: [PATCH 20/74] Clean up ReachabilityWorker jobs and delete from Instances table when deleting all users and activities for an instance --- lib/pleroma/instances/instance.ex | 6 +----- lib/pleroma/workers/delete_worker.ex | 12 ++++++++++++ lib/pleroma/workers/reachability_worker.ex | 10 ++++++++++ 3 files changed, 23 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex index dca30275b..cf896ca08 100644 --- a/lib/pleroma/instances/instance.ex +++ b/lib/pleroma/instances/instance.ex @@ -99,11 +99,7 @@ defmodule Pleroma.Instances.Instance do |> changeset(%{unreachable_since: nil}) |> Repo.insert(on_conflict: {:replace, [:unreachable_since]}, conflict_target: :host) - # Delete any existing reachability testing jobs for this instance - Oban.Job - |> Ecto.Query.where(worker: "Pleroma.Workers.ReachabilityWorker") - |> Ecto.Query.where([j], j.args["domain"] == ^host) - |> Oban.delete_all_jobs() + Pleroma.Workers.ReachabilityWorker.delete_jobs_for_host(host) result end diff --git a/lib/pleroma/workers/delete_worker.ex b/lib/pleroma/workers/delete_worker.ex index 4f52edd28..b83185fff 100644 --- a/lib/pleroma/workers/delete_worker.ex +++ b/lib/pleroma/workers/delete_worker.ex @@ -14,6 +14,7 @@ defmodule Pleroma.Workers.DeleteWorker do end def perform(%Job{args: %{"op" => "delete_instance", "host" => host}}) do + # Schedule the per-user deletion jobs Pleroma.Repo.transaction(fn -> User.Query.build(%{nickname: "@#{host}"}) |> Pleroma.Repo.all() @@ -22,6 +23,17 @@ defmodule Pleroma.Workers.DeleteWorker do |> __MODULE__.new() |> Oban.insert() end) + + # Delete the instance from the Instances table + case Pleroma.Repo.get_by(Pleroma.Instances.Instance, host: host) do + nil -> :ok + instance -> Pleroma.Repo.delete(instance) + end + + # Delete any pending ReachabilityWorker jobs for this domain + Pleroma.Workers.ReachabilityWorker.delete_jobs_for_host(host) + + :ok end) end diff --git a/lib/pleroma/workers/reachability_worker.ex b/lib/pleroma/workers/reachability_worker.ex index badfa476c..41981a2e4 100644 --- a/lib/pleroma/workers/reachability_worker.ex +++ b/lib/pleroma/workers/reachability_worker.ex @@ -11,6 +11,8 @@ defmodule Pleroma.Workers.ReachabilityWorker do alias Pleroma.HTTP alias Pleroma.Instances + import Ecto.Query + @impl true def perform(%Oban.Job{args: %{"domain" => domain, "phase" => phase, "attempt" => attempt}}) do case check_reachability(domain) do @@ -43,6 +45,14 @@ defmodule Pleroma.Workers.ReachabilityWorker do @impl true def timeout(_job), do: :timer.seconds(5) + @doc "Deletes scheduled jobs to check reachability for specified instance" + def delete_jobs_for_host(host) do + Oban.Job + |> where(worker: "Pleroma.Workers.ReachabilityWorker") + |> where([j], j.args["domain"] == ^host) + |> Oban.delete_all_jobs() + end + defp check_reachability(domain) do case HTTP.get("https://#{domain}/") do {:ok, %{status: status}} when status in 200..299 -> From df0880d8d12e557ca79161bc9a942bc8b3655d4e Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 28 Jun 2025 13:23:37 -0700 Subject: [PATCH 21/74] Add Instances.delete_all_unreachable/0 --- lib/pleroma/instances.ex | 8 ++++++++ test/pleroma/instances_test.exs | 26 ++++++++++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/lib/pleroma/instances.ex b/lib/pleroma/instances.ex index a69554ada..79fbd538f 100644 --- a/lib/pleroma/instances.ex +++ b/lib/pleroma/instances.ex @@ -33,4 +33,12 @@ defmodule Pleroma.Instances do |> Oban.insert() end) end + + @doc "Deletes all users and activities for unreachable instances" + def delete_all_unreachable do + get_unreachable() + |> Enum.each(fn {domain, _} -> + Instance.delete_users_and_activities(domain) + end) + end end diff --git a/test/pleroma/instances_test.exs b/test/pleroma/instances_test.exs index 8e23fd096..c8618b748 100644 --- a/test/pleroma/instances_test.exs +++ b/test/pleroma/instances_test.exs @@ -109,4 +109,30 @@ defmodule Pleroma.InstancesTest do assert job.args["domain"] == unreachable_domain end end + + test "delete_all_unreachable/0 schedules DeleteWorker jobs for all unreachable instances" do + domain1 = "unreachable1.example.com" + domain2 = "unreachable2.example.com" + domain3 = "unreachable3.example.com" + + Instances.set_unreachable(domain1) + Instances.set_unreachable(domain2) + Instances.set_unreachable(domain3) + + Instances.delete_all_unreachable() + + # Verify that DeleteWorker jobs were scheduled for all unreachable domains + jobs = all_enqueued(worker: Pleroma.Workers.DeleteWorker) + assert length(jobs) == 3 + + domains = Enum.map(jobs, & &1.args["host"]) + assert domain1 in domains + assert domain2 in domains + assert domain3 in domains + + # Verify all jobs are delete_instance operations + Enum.each(jobs, fn job -> + assert job.args["op"] == "delete_instance" + end) + end end From 59844d020212b1df85101a142b2102d62bdccaef Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 28 Jun 2025 13:37:06 -0700 Subject: [PATCH 22/74] Rename Instance.delete_users_and_activities/1 to Instance.delete/1 --- lib/pleroma/instances.ex | 2 +- lib/pleroma/instances/instance.ex | 2 +- lib/pleroma/web/admin_api/controllers/instance_controller.ex | 2 +- test/pleroma/instances/instance_test.exs | 4 ++-- test/pleroma/workers/delete_worker_test.exs | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/instances.ex b/lib/pleroma/instances.ex index 79fbd538f..52dbba8ad 100644 --- a/lib/pleroma/instances.ex +++ b/lib/pleroma/instances.ex @@ -38,7 +38,7 @@ defmodule Pleroma.Instances do def delete_all_unreachable do get_unreachable() |> Enum.each(fn {domain, _} -> - Instance.delete_users_and_activities(domain) + Instance.delete(domain) end) end end diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex index cf896ca08..3695e0b75 100644 --- a/lib/pleroma/instances/instance.ex +++ b/lib/pleroma/instances/instance.ex @@ -296,7 +296,7 @@ defmodule Pleroma.Instances.Instance do Deletes all users from an instance in a background task, thus also deleting all of those users' activities and notifications. """ - def delete_users_and_activities(host) when is_binary(host) do + def delete(host) when is_binary(host) do DeleteWorker.new(%{"op" => "delete_instance", "host" => host}) |> Oban.insert() end diff --git a/lib/pleroma/web/admin_api/controllers/instance_controller.ex b/lib/pleroma/web/admin_api/controllers/instance_controller.ex index 117a72280..40d4d812e 100644 --- a/lib/pleroma/web/admin_api/controllers/instance_controller.ex +++ b/lib/pleroma/web/admin_api/controllers/instance_controller.ex @@ -49,7 +49,7 @@ defmodule Pleroma.Web.AdminAPI.InstanceController do end def delete(conn, %{"instance" => instance}) do - with {:ok, _job} <- Instance.delete_users_and_activities(instance) do + with {:ok, _job} <- Instance.delete(instance) do json(conn, instance) end end diff --git a/test/pleroma/instances/instance_test.exs b/test/pleroma/instances/instance_test.exs index 83e70ac38..bc3e7993e 100644 --- a/test/pleroma/instances/instance_test.exs +++ b/test/pleroma/instances/instance_test.exs @@ -239,10 +239,10 @@ defmodule Pleroma.Instances.InstanceTest do end end - test "delete_users_and_activities/1 schedules a job to delete the instance and users" do + test "delete/1 schedules a job to delete the instance and users" do insert(:user, nickname: "mario@mushroom.kingdom", name: "Mario") - {:ok, _job} = Instance.delete_users_and_activities("mushroom.kingdom") + {:ok, _job} = Instance.delete("mushroom.kingdom") assert_enqueued( worker: Pleroma.Workers.DeleteWorker, diff --git a/test/pleroma/workers/delete_worker_test.exs b/test/pleroma/workers/delete_worker_test.exs index b914aaee2..1becd0c03 100644 --- a/test/pleroma/workers/delete_worker_test.exs +++ b/test/pleroma/workers/delete_worker_test.exs @@ -17,7 +17,7 @@ defmodule Pleroma.Workers.DeleteWorkerTest do user1 = insert(:user, nickname: "alice@example.com", name: "Alice") user2 = insert(:user, nickname: "bob@example.com", name: "Bob") - {:ok, job} = Instance.delete_users_and_activities("example.com") + {:ok, job} = Instance.delete("example.com") assert_enqueued( worker: DeleteWorker, From 9eb3fc2d3b66bb7865f6c3699a39ca34ee327cf0 Mon Sep 17 00:00:00 2001 From: Phantasm Date: Mon, 30 Jun 2025 23:25:56 +0200 Subject: [PATCH 23/74] Docs: Avoid long DB restore times and update few things Mostly to avoid long restore times thanks to an index not being built before it's needed by restoring the DB schema first. https://blog.freespeechextremist.com/blog/activities-visibility-index-slowness.html Also updates backup command to compress DB backups, removes Pleroma users's home directory, replaces "role" with "user" in PostgreSQL contexts since they are the same now. --- changelog.d/db-restore-docs.change | 1 + docs/administration/backup.md | 64 +++++++++++++++++++++++------- 2 files changed, 51 insertions(+), 14 deletions(-) create mode 100644 changelog.d/db-restore-docs.change diff --git a/changelog.d/db-restore-docs.change b/changelog.d/db-restore-docs.change new file mode 100644 index 000000000..21e0f8e97 --- /dev/null +++ b/changelog.d/db-restore-docs.change @@ -0,0 +1 @@ +Docs: Restore DB schema before data to avoid long restore times diff --git a/docs/administration/backup.md b/docs/administration/backup.md index 93325e702..794e82b19 100644 --- a/docs/administration/backup.md +++ b/docs/administration/backup.md @@ -4,7 +4,10 @@ 1. Stop the Pleroma service. 2. Go to the working directory of Pleroma (default is `/opt/pleroma`) -3. Run `sudo -Hu postgres pg_dump -d --format=custom -f ` (make sure the postgres user has write access to the destination file) +3. Run (make sure the postgres user has write access to the destination file): +``` +# sudo -Hu postgres pg_dump -d -v --format=custom --compress=9 -f +``` 4. Copy `pleroma.pgdump`, `config/prod.secret.exs`, `config/setup_db.psql` (if still available) and the `uploads` folder to your backup destination. If you have other modifications, copy those changes too. 5. Restart the Pleroma service. @@ -14,16 +17,33 @@ 2. Stop the Pleroma service. 3. Go to the working directory of Pleroma (default is `/opt/pleroma`) 4. Copy the above mentioned files back to their original position. -5. Drop the existing database and user if restoring in-place. `sudo -Hu postgres psql -c 'DROP DATABASE ;';` `sudo -Hu postgres psql -c 'DROP USER ;'` -6. Restore the database schema and pleroma postgres role the with the original `setup_db.psql` if you have it: `sudo -Hu postgres psql -f config/setup_db.psql`. +5. Drop the existing database and user if restoring in-place. +``` +# sudo -Hu postgres dropdb +# sudo -Hu postgres dropuser +``` +6. Restore the database schema and pleroma database user the with the original `setup_db.psql` if you have it: +``` +# sudo -Hu postgres psql -f config/setup_db.psql +``` - Alternatively, run the `mix pleroma.instance gen` task again. You can ignore most of the questions, but make the database user, name, and password the same as found in your backup of `config/prod.secret.exs`. Then run the restoration of the pleroma role and schema with of the generated `config/setup_db.psql` as instructed above. You may delete the `config/generated_config.exs` file as it is not needed. + Alternatively, run the `mix pleroma.instance gen` task again. You can ignore most of the questions, but make the database user, name, and password the same as found in your backup of `config/prod.secret.exs`. Then run the restoration of the pleroma user and schema with the generated `config/setup_db.psql` as instructed above. You may delete the `config/generated_config.exs` file as it is not needed. -7. Now restore the Pleroma instance's data into the empty database schema: `sudo -Hu postgres pg_restore -d -v -1 ` -8. If you installed a newer Pleroma version, you should run `mix ecto.migrate`[^1]. This task performs database migrations, if there were any. -9. Restart the Pleroma service. -10. Run `sudo -Hu postgres vacuumdb --all --analyze-in-stages`. This will quickly generate the statistics so that postgres can properly plan queries. -11. If setting up on a new server configure Nginx by using the `installation/pleroma.nginx` config sample or reference the Pleroma installation guide for your OS which contains the Nginx configuration instructions. +7. Now restore the Pleroma instance's schema into the empty database schema: +``` +# sudo -Hu postgres pg_restore -d -v -s -1 +``` +8. Now restore the Pleroma instance's data into the database: +``` +# sudo -Hu postgres pg_restore -d -v -a -1 --disable-triggers +``` +9. If you installed a newer Pleroma version, you should run `mix ecto.migrate`[^1]. This task performs database migrations, if there were any. +10. Generate the statistics so that PostgreSQL can properly plan queries. +``` +# sudo -Hu postgres vacuumdb -v --all --analyze-in-stages +``` +11. Restart the Pleroma service. +12. If setting up on a new server, configure Nginx by using your original configuration or by using the `installation/pleroma.nginx` config sample or reference the Pleroma installation guide for your OS which contains the Nginx configuration instructions. [^1]: Prefix with `MIX_ENV=prod` to run it using the production config file. @@ -32,10 +52,26 @@ 1. Optionally you can remove the users of your instance. This will trigger delete requests for their accounts and posts. Note that this is 'best effort' and doesn't mean that all traces of your instance will be gone from the fediverse. * You can do this from the admin-FE where you can select all local users and delete the accounts using the *Moderate multiple users* dropdown. * You can also list local users and delete them individually using the CLI tasks for [Managing users](./CLI_tasks/user.md). -2. Stop the Pleroma service `systemctl stop pleroma` -3. Disable pleroma from systemd `systemctl disable pleroma` +2. Stop the Pleroma service: +``` +# systemctl stop pleroma +``` +3. Disable pleroma from systemd: +``` +# systemctl disable pleroma +``` 4. Remove the files and folders you created during installation (see installation guide). This includes the pleroma, nginx and systemd files and folders. -5. Reload nginx now that the configuration is removed `systemctl reload nginx` -6. Remove the database and database user `sudo -Hu postgres psql -c 'DROP DATABASE ;';` `sudo -Hu postgres psql -c 'DROP USER ;'` -7. Remove the system user `userdel pleroma` +5. Reload nginx now that the configuration is removed: +``` +# systemctl reload nginx +``` +6. Remove the database and database user: +``` +# sudo -Hu postgres dropdb +# sudo -Hu postgres dropuser +``` +7. Remove the system user: +``` +# userdel -r pleroma +``` 8. Remove the dependencies that you don't need anymore (see installation guide). Make sure you don't remove packages that are still needed for other software that you have running! From d736d313082c5939e3421921ee51ded2ec5aebce Mon Sep 17 00:00:00 2001 From: Phantasm Date: Tue, 15 Jul 2025 16:08:58 +0200 Subject: [PATCH 24/74] Docs: Add systemctl commands to DB backup/restore --- docs/administration/backup.md | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/docs/administration/backup.md b/docs/administration/backup.md index 794e82b19..326f095ec 100644 --- a/docs/administration/backup.md +++ b/docs/administration/backup.md @@ -2,22 +2,31 @@ ## Backup -1. Stop the Pleroma service. -2. Go to the working directory of Pleroma (default is `/opt/pleroma`) +1. Stop the Pleroma service: +``` +# sudo systemctl stop pleroma +``` +2. Go to the working directory of Pleroma (default is `/opt/pleroma`). 3. Run (make sure the postgres user has write access to the destination file): ``` # sudo -Hu postgres pg_dump -d -v --format=custom --compress=9 -f ``` 4. Copy `pleroma.pgdump`, `config/prod.secret.exs`, `config/setup_db.psql` (if still available) and the `uploads` folder to your backup destination. If you have other modifications, copy those changes too. -5. Restart the Pleroma service. +5. Restart the Pleroma service: +``` +# sudo systemctl start pleroma +``` ## Restore/Move 1. Optionally reinstall Pleroma (either on the same server or on another server if you want to move servers). -2. Stop the Pleroma service. -3. Go to the working directory of Pleroma (default is `/opt/pleroma`) +2. Stop the Pleroma service: +``` +# sudo systemctl stop pleroma +``` +3. Go to the working directory of Pleroma (default is `/opt/pleroma`). 4. Copy the above mentioned files back to their original position. -5. Drop the existing database and user if restoring in-place. +5. Drop the existing database and user if restoring in-place: ``` # sudo -Hu postgres dropdb # sudo -Hu postgres dropuser @@ -38,11 +47,14 @@ # sudo -Hu postgres pg_restore -d -v -a -1 --disable-triggers ``` 9. If you installed a newer Pleroma version, you should run `mix ecto.migrate`[^1]. This task performs database migrations, if there were any. -10. Generate the statistics so that PostgreSQL can properly plan queries. +10. Generate the statistics so that PostgreSQL can properly plan queries: ``` # sudo -Hu postgres vacuumdb -v --all --analyze-in-stages ``` -11. Restart the Pleroma service. +11. Restart the Pleroma service: +``` +# sudo systemctl start pleroma +``` 12. If setting up on a new server, configure Nginx by using your original configuration or by using the `installation/pleroma.nginx` config sample or reference the Pleroma installation guide for your OS which contains the Nginx configuration instructions. [^1]: Prefix with `MIX_ENV=prod` to run it using the production config file. @@ -74,4 +86,4 @@ ``` # userdel -r pleroma ``` -8. Remove the dependencies that you don't need anymore (see installation guide). Make sure you don't remove packages that are still needed for other software that you have running! +8. Remove the dependencies that you don't need anymore (see installation guide). **Make sure you don't remove packages that are still needed for other software that you have running!** From 9ea55a38885c4c4142ca9a479ce0eb23886fbaa4 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 22 Jul 2025 11:49:48 -0700 Subject: [PATCH 25/74] Fix dialyzer error in API spec: Use then/2 for OpenApiSpex.resolve_schema_modules/1 call --- lib/pleroma/web/api_spec.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/api_spec.ex b/lib/pleroma/web/api_spec.ex index e5339097f..3e0ac3704 100644 --- a/lib/pleroma/web/api_spec.ex +++ b/lib/pleroma/web/api_spec.ex @@ -158,6 +158,6 @@ defmodule Pleroma.Web.ApiSpec do } } # discover request/response schemas from path specs - |> OpenApiSpex.resolve_schema_modules() + |> then(&OpenApiSpex.resolve_schema_modules/1) end end From daad35aeb95d0367d1525c60213db1ee6cede6bc Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 22 Jul 2025 11:49:56 -0700 Subject: [PATCH 26/74] Fix dialyzer error in scopes compiler: Add error handling for extract_all_scopes/0 --- lib/pleroma/web/api_spec/scopes/compiler.ex | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/api_spec/scopes/compiler.ex b/lib/pleroma/web/api_spec/scopes/compiler.ex index 162edc9a3..a92cf1199 100644 --- a/lib/pleroma/web/api_spec/scopes/compiler.ex +++ b/lib/pleroma/web/api_spec/scopes/compiler.ex @@ -26,7 +26,11 @@ defmodule Pleroma.Web.ApiSpec.Scopes.Compiler do end def extract_all_scopes do - extract_all_scopes_from(Pleroma.Web.ApiSpec.spec()) + try do + extract_all_scopes_from(Pleroma.Web.ApiSpec.spec()) + catch + _, _ -> [] + end end def extract_all_scopes_from(specs) do From 47ebbc4d212c0adffd212c66fc6b14b1a2ec24e8 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 22 Jul 2025 11:50:01 -0700 Subject: [PATCH 27/74] Fix dialyzer error in status controller: Add catch-all pattern for translate function --- lib/pleroma/web/mastodon_api/controllers/status_controller.ex | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex index 10549fb20..32874d464 100644 --- a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex @@ -584,6 +584,9 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do {:error, error} when error in [:unexpected_response, :quota_exceeded, :too_many_requests] -> render_error(conn, :service_unavailable, "Translation service not available") + + _ -> + render_error(conn, :internal_server_error, "Translation failed") end end From 1d4482047f1ab40ae10f8086620658b327d8d64d Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 22 Jul 2025 11:50:23 -0700 Subject: [PATCH 28/74] Fix dialyzer error in translation provider: Change Map.t() to map() in callback spec --- lib/pleroma/language/translation/provider.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/language/translation/provider.ex b/lib/pleroma/language/translation/provider.ex index 533b5355a..4b423247a 100644 --- a/lib/pleroma/language/translation/provider.ex +++ b/lib/pleroma/language/translation/provider.ex @@ -25,7 +25,7 @@ defmodule Pleroma.Language.Translation.Provider do @callback supported_languages(type :: :string | :target) :: {:ok, [String.t()]} | {:error, atom()} - @callback languages_matrix() :: {:ok, Map.t()} | {:error, atom()} + @callback languages_matrix() :: {:ok, map()} | {:error, atom()} @callback name() :: String.t() From e0104132a7fa5f4a913e33e7f654b6cd64df0107 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 22 Jul 2025 11:50:28 -0700 Subject: [PATCH 29/74] Fix dialyzer error in object fetcher: Add proper guard clause for check_crossdomain_redirect/2 Also remove unnecessary and incorrect usage of Mix.env() --- lib/pleroma/object/fetcher.ex | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index b54ef9ce5..c02069ecc 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -19,8 +19,6 @@ defmodule Pleroma.Object.Fetcher do require Logger require Pleroma.Constants - @mix_env Mix.env() - @spec reinject_object(struct(), map()) :: {:ok, Object.t()} | {:error, any()} defp reinject_object(%Object{data: %{}} = object, new_data) do Logger.debug("Reinjecting object #{new_data["id"]}") @@ -178,13 +176,8 @@ defmodule Pleroma.Object.Fetcher do def fetch_and_contain_remote_object_from_id(_id), do: {:error, "id must be a string"} - defp check_crossdomain_redirect(final_host, original_url) - - # Handle the common case in tests where responses don't include URLs - if @mix_env == :test do - defp check_crossdomain_redirect(nil, _) do - {:cross_domain_redirect, false} - end + defp check_crossdomain_redirect(final_host, _original_url) when is_nil(final_host) do + {:cross_domain_redirect, false} end defp check_crossdomain_redirect(final_host, original_url) do From 28146ee7d258a70366bd16e5e099e5c1f4adc25b Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 22 Jul 2025 11:50:38 -0700 Subject: [PATCH 30/74] Fix dialyzer error in safe_zip: Remove impossible pattern match for {:get_type, _e} --- lib/pleroma/safe_zip.ex | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lib/pleroma/safe_zip.ex b/lib/pleroma/safe_zip.ex index 25fe434d6..ece18e645 100644 --- a/lib/pleroma/safe_zip.ex +++ b/lib/pleroma/safe_zip.ex @@ -56,10 +56,6 @@ defmodule Pleroma.SafeZip do {_, true} <- {:safe_path, safe_path?(path)} do {:cont, {:ok, maybe_add_file(type, path, fl)}} else - {:get_type, e} -> - {:halt, - {:error, "Couldn't determine file type of ZIP entry at #{path} (#{inspect(e)})"}} - {:type, _} -> {:halt, {:error, "Potentially unsafe file type in ZIP at: #{path}"}} From 28cff592b134c9a884d7e8dc147743686c523a89 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 22 Jul 2025 11:50:45 -0700 Subject: [PATCH 31/74] Fix dialyzer error in MRF remote report policy: Remove unreachable pattern match --- lib/pleroma/web/activity_pub/mrf/remote_report_policy.ex | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/pleroma/web/activity_pub/mrf/remote_report_policy.ex b/lib/pleroma/web/activity_pub/mrf/remote_report_policy.ex index fa0610bf1..8422832dd 100644 --- a/lib/pleroma/web/activity_pub/mrf/remote_report_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/remote_report_policy.ex @@ -15,7 +15,6 @@ defmodule Pleroma.Web.ActivityPub.MRF.RemoteReportPolicy do else {:local, true} -> {:ok, object} {:reject, message} -> {:reject, message} - error -> {:reject, error} end end From b54b19a0f4fa3c7220994f259e1a0d12757321ff Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 22 Jul 2025 13:22:51 -0700 Subject: [PATCH 32/74] Fix test for mix task Missing assert_receive which would cause the test to randomly fail --- test/mix/tasks/pleroma/app_test.exs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/mix/tasks/pleroma/app_test.exs b/test/mix/tasks/pleroma/app_test.exs index f35447edc..65245eadd 100644 --- a/test/mix/tasks/pleroma/app_test.exs +++ b/test/mix/tasks/pleroma/app_test.exs @@ -42,9 +42,10 @@ defmodule Mix.Tasks.Pleroma.AppTest do test "with errors" do Mix.Tasks.Pleroma.App.run(["create"]) - {:mix_shell, :error, ["Creating failed:"]} - {:mix_shell, :error, ["name: can't be blank"]} - {:mix_shell, :error, ["redirect_uris: can't be blank"]} + + assert_receive {:mix_shell, :error, ["Creating failed:"]} + assert_receive {:mix_shell, :error, ["name: can't be blank"]} + assert_receive {:mix_shell, :error, ["redirect_uris: can't be blank"]} end defp assert_app(name, redirect, scopes) do From 113261146ff5e643c9598189f8157dfd60516016 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 22 Jul 2025 13:28:25 -0700 Subject: [PATCH 33/74] Fix account endorsements test Random failures were caused by the results sometimes being returned out of order. --- .../web/pleroma_api/controllers/account_controller_test.exs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/pleroma/web/pleroma_api/controllers/account_controller_test.exs b/test/pleroma/web/pleroma_api/controllers/account_controller_test.exs index 61880e2c0..d152a44cd 100644 --- a/test/pleroma/web/pleroma_api/controllers/account_controller_test.exs +++ b/test/pleroma/web/pleroma_api/controllers/account_controller_test.exs @@ -292,10 +292,14 @@ defmodule Pleroma.Web.PleromaAPI.AccountControllerTest do User.endorse(user1, user2) User.endorse(user1, user3) - [%{"id" => ^id2}, %{"id" => ^id3}] = + response = conn |> get("/api/v1/pleroma/accounts/#{id1}/endorsements") |> json_response_and_validate_schema(200) + + assert length(response) == 2 + assert Enum.any?(response, fn user -> user["id"] == id2 end) + assert Enum.any?(response, fn user -> user["id"] == id3 end) end test "returns 404 error when specified user is not exist", %{conn: conn} do From 28b69f5c04e3ef05216c73f85696e44fb4a047af Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 22 Jul 2025 14:11:28 -0700 Subject: [PATCH 34/74] Reset Emoji cache between tests This fixes intermittent test failures --- test/pleroma/emoji/pack_test.exs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/pleroma/emoji/pack_test.exs b/test/pleroma/emoji/pack_test.exs index b458401a7..56b17ac31 100644 --- a/test/pleroma/emoji/pack_test.exs +++ b/test/pleroma/emoji/pack_test.exs @@ -13,6 +13,9 @@ defmodule Pleroma.Emoji.PackTest do ) setup do + # Reload emoji to ensure a clean state + Emoji.reload() + pack_path = Path.join(@emoji_path, "dump_pack") File.mkdir(pack_path) From 6da5ca9b2d271179e0ec912f642ec51f98be3e80 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 22 Jul 2025 14:16:17 -0700 Subject: [PATCH 35/74] Prevent test crash if it cannot successfully remove the console Logger backend --- lib/mix/pleroma.ex | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/mix/pleroma.ex b/lib/mix/pleroma.ex index c01cf054d..e7869919c 100644 --- a/lib/mix/pleroma.ex +++ b/lib/mix/pleroma.ex @@ -26,7 +26,11 @@ defmodule Mix.Pleroma do Application.put_env(:phoenix, :serve_endpoints, false, persistent: true) unless System.get_env("DEBUG") do - Logger.remove_backend(:console) + try do + Logger.remove_backend(:console) + catch + :exit, _ -> :ok + end end adapter = Application.get_env(:tesla, :adapter) From a504c28106be8b9c4020ab2abc194ba6a10b076a Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 23 Jul 2025 09:55:03 -0700 Subject: [PATCH 36/74] Not changelog worthy --- changelog.d/noop-fixes.skip | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 changelog.d/noop-fixes.skip diff --git a/changelog.d/noop-fixes.skip b/changelog.d/noop-fixes.skip new file mode 100644 index 000000000..e69de29bb From 8e0f73e45c9807fcf65e4d0b7d81a050b1a706a0 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 28 Jul 2025 17:18:56 -0700 Subject: [PATCH 37/74] Change Oban Notifier to Oban.Notifiers.PG --- changelog.d/oban-notifier.change | 1 + config/config.exs | 1 + 2 files changed, 2 insertions(+) create mode 100644 changelog.d/oban-notifier.change diff --git a/changelog.d/oban-notifier.change b/changelog.d/oban-notifier.change new file mode 100644 index 000000000..a3932a165 --- /dev/null +++ b/changelog.d/oban-notifier.change @@ -0,0 +1 @@ +Oban Notifier was changed to Oban.Notifiers.PG for performance and scalability benefits diff --git a/config/config.exs b/config/config.exs index 31d7258ee..ba55922ad 100644 --- a/config/config.exs +++ b/config/config.exs @@ -590,6 +590,7 @@ config :pleroma, Pleroma.User, # value or it cannot enforce uniqueness. config :pleroma, Oban, repo: Pleroma.Repo, + notifier: Oban.Notifiers.PG, log: false, queues: [ activity_expiration: 10, From 3efb99fdf80e242da31f62e9259aa527a220f3e1 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 29 Jul 2025 16:34:27 -0700 Subject: [PATCH 38/74] Postgrex: Update to 0.20.0 Includes fixes for database reconnection handling --- changelog.d/postgrex.change | 1 + mix.exs | 4 ++-- mix.lock | 10 +++++----- 3 files changed, 8 insertions(+), 7 deletions(-) create mode 100644 changelog.d/postgrex.change diff --git a/changelog.d/postgrex.change b/changelog.d/postgrex.change new file mode 100644 index 000000000..1539f5b8d --- /dev/null +++ b/changelog.d/postgrex.change @@ -0,0 +1 @@ +Updated Postgrex library to 0.20.0 diff --git a/mix.exs b/mix.exs index d5e47275d..a79aaca8f 100644 --- a/mix.exs +++ b/mix.exs @@ -128,7 +128,7 @@ defmodule Pleroma.Mixfile do {:phoenix_ecto, "~> 4.4"}, {:ecto_sql, "~> 3.10"}, {:ecto_enum, "~> 1.4"}, - {:postgrex, ">= 0.0.0"}, + {:postgrex, ">= 0.20.0"}, {:phoenix_html, "~> 3.3"}, {:phoenix_live_view, "~> 0.19.0"}, {:phoenix_live_dashboard, "~> 0.8.0"}, @@ -188,7 +188,7 @@ defmodule Pleroma.Mixfile do {:restarter, path: "./restarter"}, {:majic, "~> 1.0"}, {:open_api_spex, "~> 3.16"}, - {:ecto_psql_extras, "~> 0.6"}, + {:ecto_psql_extras, "~> 0.8"}, {:vix, "~> 0.26.0"}, {:elixir_make, "~> 0.7.7", override: true}, {:blurhash, "~> 0.1.0", hex: :rinpatch_blurhash}, diff --git a/mix.lock b/mix.lock index 35a600b5f..a708087c9 100644 --- a/mix.lock +++ b/mix.lock @@ -33,10 +33,10 @@ "earmark": {:hex, :earmark, "1.4.46", "8c7287bd3137e99d26ae4643e5b7ef2129a260e3dcf41f251750cb4563c8fb81", [:mix], [], "hexpm", "798d86db3d79964e759ddc0c077d5eb254968ed426399fbf5a62de2b5ff8910a"}, "earmark_parser": {:hex, :earmark_parser, "1.4.39", "424642f8335b05bb9eb611aa1564c148a8ee35c9c8a8bba6e129d51a3e3c6769", [:mix], [], "hexpm", "06553a88d1f1846da9ef066b87b57c6f605552cfbe40d20bd8d59cc6bde41944"}, "eblurhash": {:git, "https://github.com/zotonic/eblurhash.git", "bc37ceb426ef021ee9927fb249bb93f7059194ab", [ref: "bc37ceb426ef021ee9927fb249bb93f7059194ab"]}, - "ecto": {:hex, :ecto, "3.11.2", "e1d26be989db350a633667c5cda9c3d115ae779b66da567c68c80cfb26a8c9ee", [:mix], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "3c38bca2c6f8d8023f2145326cc8a80100c3ffe4dcbd9842ff867f7fc6156c65"}, + "ecto": {:hex, :ecto, "3.13.2", "7d0c0863f3fc8d71d17fc3ad3b9424beae13f02712ad84191a826c7169484f01", [:mix], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "669d9291370513ff56e7b7e7081b7af3283d02e046cf3d403053c557894a0b3e"}, "ecto_enum": {:hex, :ecto_enum, "1.4.0", "d14b00e04b974afc69c251632d1e49594d899067ee2b376277efd8233027aec8", [:mix], [{:ecto, ">= 3.0.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:ecto_sql, "> 3.0.0", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:mariaex, ">= 0.0.0", [hex: :mariaex, repo: "hexpm", optional: true]}, {:postgrex, ">= 0.0.0", [hex: :postgrex, repo: "hexpm", optional: true]}], "hexpm", "8fb55c087181c2b15eee406519dc22578fa60dd82c088be376d0010172764ee4"}, - "ecto_psql_extras": {:hex, :ecto_psql_extras, "0.7.15", "0fc29dbae0e444a29bd6abeee4cf3c4c037e692a272478a234a1cc765077dbb1", [:mix], [{:ecto_sql, "~> 3.7", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.16.0 or ~> 0.17.0", [hex: :postgrex, repo: "hexpm", optional: false]}, {:table_rex, "~> 3.1.1 or ~> 4.0.0", [hex: :table_rex, repo: "hexpm", optional: false]}], "hexpm", "b6127f3a5c6fc3d84895e4768cc7c199f22b48b67d6c99b13fbf4a374e73f039"}, - "ecto_sql": {:hex, :ecto_sql, "3.11.3", "4eb7348ff8101fbc4e6bbc5a4404a24fecbe73a3372d16569526b0cf34ebc195", [:mix], [{:db_connection, "~> 2.4.1 or ~> 2.5", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.11.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.6", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.16 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1 or ~> 2.2", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "e5f36e3d736b99c7fee3e631333b8394ade4bafe9d96d35669fca2d81c2be928"}, + "ecto_psql_extras": {:hex, :ecto_psql_extras, "0.8.8", "aa02529c97f69aed5722899f5dc6360128735a92dd169f23c5d50b1f7fdede08", [:mix], [{:ecto_sql, "~> 3.7", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:postgrex, "> 0.16.0", [hex: :postgrex, repo: "hexpm", optional: false]}, {:table_rex, "~> 3.1.1 or ~> 4.0", [hex: :table_rex, repo: "hexpm", optional: false]}], "hexpm", "04c63d92b141723ad6fed2e60a4b461ca00b3594d16df47bbc48f1f4534f2c49"}, + "ecto_sql": {:hex, :ecto_sql, "3.13.2", "a07d2461d84107b3d037097c822ffdd36ed69d1cf7c0f70e12a3d1decf04e2e1", [:mix], [{:db_connection, "~> 2.4.1 or ~> 2.5", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.13.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.7", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.19 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1 or ~> 2.2", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "539274ab0ecf1a0078a6a72ef3465629e4d6018a3028095dc90f60a19c371717"}, "eimp": {:hex, :eimp, "1.0.14", "fc297f0c7e2700457a95a60c7010a5f1dcb768a083b6d53f49cd94ab95a28f22", [:rebar3], [{:p1_utils, "1.0.18", [hex: :p1_utils, repo: "hexpm", optional: false]}], "hexpm", "501133f3112079b92d9e22da8b88bf4f0e13d4d67ae9c15c42c30bd25ceb83b6"}, "elixir_make": {:hex, :elixir_make, "0.7.8", "505026f266552ee5aabca0b9f9c229cbb496c689537c9f922f3eb5431157efc7", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:certifi, "~> 2.0", [hex: :certifi, repo: "hexpm", optional: true]}], "hexpm", "7a71945b913d37ea89b06966e1342c85cfe549b15e6d6d081e8081c493062c07"}, "erlex": {:hex, :erlex, "0.2.6", "c7987d15e899c7a2f34f5420d2a2ea0d659682c06ac607572df55a43753aa12e", [:mix], [], "hexpm", "2ed2e25711feb44d52b17d2780eabf998452f6efda104877a3881c2f8c0c0c75"}, @@ -114,7 +114,7 @@ "plug_static_index_html": {:hex, :plug_static_index_html, "1.0.0", "840123d4d3975585133485ea86af73cb2600afd7f2a976f9f5fd8b3808e636a0", [:mix], [{:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "79fd4fcf34d110605c26560cbae8f23c603ec4158c08298bd4360fdea90bb5cf"}, "poison": {:hex, :poison, "3.1.0", "d9eb636610e096f86f25d9a46f35a9facac35609a7591b3be3326e99a0484665", [:mix], [], "hexpm", "fec8660eb7733ee4117b85f55799fd3833eb769a6df71ccf8903e8dc5447cfce"}, "poolboy": {:hex, :poolboy, "1.5.2", "392b007a1693a64540cead79830443abf5762f5d30cf50bc95cb2c1aaafa006b", [:rebar3], [], "hexpm", "dad79704ce5440f3d5a3681c8590b9dc25d1a561e8f5a9c995281012860901e3"}, - "postgrex": {:hex, :postgrex, "0.17.5", "0483d054938a8dc069b21bdd636bf56c487404c241ce6c319c1f43588246b281", [:mix], [{:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "50b8b11afbb2c4095a3ba675b4f055c416d0f3d7de6633a595fc131a828a67eb"}, + "postgrex": {:hex, :postgrex, "0.20.0", "363ed03ab4757f6bc47942eff7720640795eb557e1935951c1626f0d303a3aed", [:mix], [{:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "d36ef8b36f323d29505314f704e21a1a038e2dc387c6409ee0cd24144e187c0f"}, "pot": {:hex, :pot, "1.0.2", "13abb849139fdc04ab8154986abbcb63bdee5de6ed2ba7e1713527e33df923dd", [:rebar3], [], "hexpm", "78fe127f5a4f5f919d6ea5a2a671827bd53eb9d37e5b4128c0ad3df99856c2e0"}, "prom_ex": {:hex, :prom_ex, "1.9.0", "63e6dda6c05cdeec1f26c48443dcc38ffd2118b3665ae8d2bd0e5b79f2aea03e", [:mix], [{:absinthe, ">= 1.6.0", [hex: :absinthe, repo: "hexpm", optional: true]}, {:broadway, ">= 1.0.2", [hex: :broadway, repo: "hexpm", optional: true]}, {:ecto, ">= 3.5.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:finch, "~> 0.15", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.2", [hex: :jason, repo: "hexpm", optional: false]}, {:oban, ">= 2.4.0", [hex: :oban, repo: "hexpm", optional: true]}, {:octo_fetch, "~> 0.3", [hex: :octo_fetch, repo: "hexpm", optional: false]}, {:phoenix, ">= 1.5.0", [hex: :phoenix, repo: "hexpm", optional: true]}, {:phoenix_live_view, ">= 0.14.0", [hex: :phoenix_live_view, repo: "hexpm", optional: true]}, {:plug, ">= 1.12.1", [hex: :plug, repo: "hexpm", optional: true]}, {:plug_cowboy, "~> 2.5 or ~> 2.6", [hex: :plug_cowboy, repo: "hexpm", optional: false]}, {:telemetry, ">= 1.0.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.6", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}, {:telemetry_metrics_prometheus_core, "~> 1.0", [hex: :telemetry_metrics_prometheus_core, repo: "hexpm", optional: false]}, {:telemetry_poller, "~> 1.0", [hex: :telemetry_poller, repo: "hexpm", optional: false]}], "hexpm", "01f3d4f69ec93068219e686cc65e58a29c42bea5429a8ff4e2121f19db178ee6"}, "prometheus": {:hex, :prometheus, "4.10.0", "792adbf0130ff61b5fa8826f013772af24b6e57b984445c8d602c8a0355704a1", [:mix, :rebar3], [{:quantile_estimator, "~> 0.2.1", [hex: :quantile_estimator, repo: "hexpm", optional: false]}], "hexpm", "2a99bb6dce85e238c7236fde6b0064f9834dc420ddbd962aac4ea2a3c3d59384"}, @@ -134,7 +134,7 @@ "sweet_xml": {:hex, :sweet_xml, "0.7.4", "a8b7e1ce7ecd775c7e8a65d501bc2cd933bff3a9c41ab763f5105688ef485d08", [:mix], [], "hexpm", "e7c4b0bdbf460c928234951def54fe87edf1a170f6896675443279e2dbeba167"}, "swoosh": {:hex, :swoosh, "1.16.9", "20c6a32ea49136a4c19f538e27739bb5070558c0fa76b8a95f4d5d5ca7d319a1", [:mix], [{:bandit, ">= 1.0.0", [hex: :bandit, repo: "hexpm", optional: true]}, {:cowboy, "~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:ex_aws, "~> 2.1", [hex: :ex_aws, repo: "hexpm", optional: true]}, {:finch, "~> 0.6", [hex: :finch, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13 or ~> 1.0", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mua, "~> 0.2.0", [hex: :mua, repo: "hexpm", optional: true]}, {:multipart, "~> 0.4", [hex: :multipart, repo: "hexpm", optional: true]}, {:plug, "~> 1.9", [hex: :plug, repo: "hexpm", optional: true]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:req, "~> 0.5 or ~> 1.0", [hex: :req, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.2 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "878b1a7a6c10ebbf725a3349363f48f79c5e3d792eb621643b0d276a38acc0a6"}, "syslog": {:hex, :syslog, "1.1.0", "6419a232bea84f07b56dc575225007ffe34d9fdc91abe6f1b2f254fd71d8efc2", [:rebar3], [], "hexpm", "4c6a41373c7e20587be33ef841d3de6f3beba08519809329ecc4d27b15b659e1"}, - "table_rex": {:hex, :table_rex, "4.0.0", "3c613a68ebdc6d4d1e731bc973c233500974ec3993c99fcdabb210407b90959b", [:mix], [], "hexpm", "c35c4d5612ca49ebb0344ea10387da4d2afe278387d4019e4d8111e815df8f55"}, + "table_rex": {:hex, :table_rex, "4.1.0", "fbaa8b1ce154c9772012bf445bfb86b587430fb96f3b12022d3f35ee4a68c918", [:mix], [], "hexpm", "95932701df195d43bc2d1c6531178fc8338aa8f38c80f098504d529c43bc2601"}, "telemetry": {:hex, :telemetry, "1.0.0", "0f453a102cdf13d506b7c0ab158324c337c41f1cc7548f0bc0e130bbf0ae9452", [:rebar3], [], "hexpm", "73bc09fa59b4a0284efb4624335583c528e07ec9ae76aca96ea0673850aec57a"}, "telemetry_metrics": {:hex, :telemetry_metrics, "0.6.2", "2caabe9344ec17eafe5403304771c3539f3b6e2f7fb6a6f602558c825d0d0bfb", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "9b43db0dc33863930b9ef9d27137e78974756f5f198cae18409970ed6fa5b561"}, "telemetry_metrics_prometheus_core": {:hex, :telemetry_metrics_prometheus_core, "1.2.0", "b583c3f18508f5c5561b674d16cf5d9afd2ea3c04505b7d92baaeac93c1b8260", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.6", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "9cba950e1c4733468efbe3f821841f34ac05d28e7af7798622f88ecdbbe63ea3"}, From 1d8eafc0d2c38fdcac34af6b71291e9c8833a20b Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 30 Jul 2025 10:13:54 -0700 Subject: [PATCH 39/74] Add failing test case for URL encoding issue --- test/pleroma/http_test.exs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/test/pleroma/http_test.exs b/test/pleroma/http_test.exs index de359e599..058cb96c0 100644 --- a/test/pleroma/http_test.exs +++ b/test/pleroma/http_test.exs @@ -25,6 +25,9 @@ defmodule Pleroma.HTTPTest do %{method: :post, url: "http://example.com/world"} -> %Tesla.Env{status: 200, body: "world"} + + %{method: :get, url: "https://tsundere.love/emoji/Pack%201/koronebless.png"} -> + %Tesla.Env{status: 200, body: "emoji data"} end) :ok @@ -67,4 +70,18 @@ defmodule Pleroma.HTTPTest do } end end + + test "URL encoding properly encodes URLs with spaces" do + url_with_space = "https://tsundere.love/emoji/Pack 1/koronebless.png" + + result = HTTP.get(url_with_space) + + assert result == {:ok, %Tesla.Env{status: 200, body: "emoji data"}} + + properly_encoded_url = "https://tsundere.love/emoji/Pack%201/koronebless.png" + + result = HTTP.get(properly_encoded_url) + + assert result == {:ok, %Tesla.Env{status: 200, body: "emoji data"}} + end end From 11d27349e32d23649dd4e5ba6a3597f62199e6e5 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 30 Jul 2025 11:05:49 -0700 Subject: [PATCH 40/74] Fix HTTP client making invalid requests due to no percent encoding processing or validation. --- changelog.d/url-encoding.fix | 1 + lib/pleroma/http.ex | 10 ++-- lib/pleroma/tesla/middleware/encode_url.ex | 53 ++++++++++++++++++++++ 3 files changed, 61 insertions(+), 3 deletions(-) create mode 100644 changelog.d/url-encoding.fix create mode 100644 lib/pleroma/tesla/middleware/encode_url.ex diff --git a/changelog.d/url-encoding.fix b/changelog.d/url-encoding.fix new file mode 100644 index 000000000..3cca87ded --- /dev/null +++ b/changelog.d/url-encoding.fix @@ -0,0 +1 @@ +Fix HTTP client making invalid requests due to no percent encoding processing or validation. diff --git a/lib/pleroma/http.ex b/lib/pleroma/http.ex index c11317850..1833f5f85 100644 --- a/lib/pleroma/http.ex +++ b/lib/pleroma/http.ex @@ -105,20 +105,24 @@ defmodule Pleroma.HTTP do end defp adapter_middlewares(Tesla.Adapter.Gun, extra_middleware) do - [Tesla.Middleware.FollowRedirects, Pleroma.Tesla.Middleware.ConnectionPool] ++ + default_middleware() ++ + [Pleroma.Tesla.Middleware.ConnectionPool] ++ extra_middleware end defp adapter_middlewares({Tesla.Adapter.Finch, _}, extra_middleware) do - [Tesla.Middleware.FollowRedirects] ++ extra_middleware + default_middleware() ++ extra_middleware end defp adapter_middlewares(_, extra_middleware) do if Pleroma.Config.get(:env) == :test do # Emulate redirects in test env, which are handled by adapters in other environments - [Tesla.Middleware.FollowRedirects] + default_middleware() else extra_middleware end end + + defp default_middleware(), + do: [Tesla.Middleware.FollowRedirects, Pleroma.Tesla.Middleware.EncodeUrl] end diff --git a/lib/pleroma/tesla/middleware/encode_url.ex b/lib/pleroma/tesla/middleware/encode_url.ex new file mode 100644 index 000000000..df10e13c2 --- /dev/null +++ b/lib/pleroma/tesla/middleware/encode_url.ex @@ -0,0 +1,53 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2025 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Tesla.Middleware.EncodeUrl do + @moduledoc """ + Middleware to encode URLs properly + + We must decode and then re-encode to ensure correct encoding. + If we only encode it will re-encode each % as %25 causing a space + already encoded as %20 to be %2520. + + Similar problem for query parameters which need spaces to be the + character + """ + + @behaviour Tesla.Middleware + + @impl Tesla.Middleware + def call(%Tesla.Env{url: url} = env, next, _) do + url = + URI.parse(url) + |> then(fn parsed -> + path = encode_path(parsed.path) + query = encode_query(parsed.query) + + %{parsed | path: path, query: query} + end) + |> URI.to_string() + + env = %{env | url: url} + + case Tesla.run(env, next) do + {:ok, env} -> {:ok, env} + err -> err + end + end + + defp encode_path(nil), do: nil + + defp encode_path(path) when is_binary(path) do + path + |> URI.decode() + |> URI.encode() + end + + defp encode_query(nil), do: nil + + defp encode_query(query) when is_binary(query) do + query + |> URI.decode_query() + |> URI.encode_query() + end +end From 4217ababfc0f75559d48e58cc9d966aae5059476 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 30 Jul 2025 13:17:50 -0700 Subject: [PATCH 41/74] Improve design so existing tests do not break --- lib/pleroma/http.ex | 19 ++++++++++++++----- lib/pleroma/tesla/middleware/encode_url.ex | 21 ++++++++++++--------- test/pleroma/http_test.exs | 2 ++ 3 files changed, 28 insertions(+), 14 deletions(-) diff --git a/lib/pleroma/http.ex b/lib/pleroma/http.ex index 1833f5f85..75570d281 100644 --- a/lib/pleroma/http.ex +++ b/lib/pleroma/http.ex @@ -115,11 +115,20 @@ defmodule Pleroma.HTTP do end defp adapter_middlewares(_, extra_middleware) do - if Pleroma.Config.get(:env) == :test do - # Emulate redirects in test env, which are handled by adapters in other environments - default_middleware() - else - extra_middleware + # A lot of tests are written expecting unencoded URLs + # and the burden of fixing that is high. Also it makes + # them hard to read. Tests will opt-in when we want to validate + # the encoding is being done correctly. + cond do + Pleroma.Config.get(:env) == :test and Pleroma.Config.get(:test_url_encoding) -> + default_middleware() + + Pleroma.Config.get(:env) == :test -> + # Emulate redirects in test env, which are handled by adapters in other environments + [Tesla.Middleware.FollowRedirects] + + true -> + extra_middleware end end diff --git a/lib/pleroma/tesla/middleware/encode_url.ex b/lib/pleroma/tesla/middleware/encode_url.ex index df10e13c2..ee74c41a1 100644 --- a/lib/pleroma/tesla/middleware/encode_url.ex +++ b/lib/pleroma/tesla/middleware/encode_url.ex @@ -17,15 +17,7 @@ defmodule Pleroma.Tesla.Middleware.EncodeUrl do @impl Tesla.Middleware def call(%Tesla.Env{url: url} = env, next, _) do - url = - URI.parse(url) - |> then(fn parsed -> - path = encode_path(parsed.path) - query = encode_query(parsed.query) - - %{parsed | path: path, query: query} - end) - |> URI.to_string() + url = encode_url(url) env = %{env | url: url} @@ -35,6 +27,17 @@ defmodule Pleroma.Tesla.Middleware.EncodeUrl do end end + defp encode_url(url) when is_binary(url) do + URI.parse(url) + |> then(fn parsed -> + path = encode_path(parsed.path) + query = encode_query(parsed.query) + + %{parsed | path: path, query: query} + end) + |> URI.to_string() + end + defp encode_path(nil), do: nil defp encode_path(path) when is_binary(path) do diff --git a/test/pleroma/http_test.exs b/test/pleroma/http_test.exs index 058cb96c0..803bd451a 100644 --- a/test/pleroma/http_test.exs +++ b/test/pleroma/http_test.exs @@ -72,6 +72,8 @@ defmodule Pleroma.HTTPTest do end test "URL encoding properly encodes URLs with spaces" do + clear_config(:test_url_encoding, true) + url_with_space = "https://tsundere.love/emoji/Pack 1/koronebless.png" result = HTTP.get(url_with_space) From 404e09126076dcbd895fb2d17f872c553cc31249 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 30 Jul 2025 13:48:16 -0700 Subject: [PATCH 42/74] Credo --- lib/pleroma/http.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/http.ex b/lib/pleroma/http.ex index 75570d281..f0e01d589 100644 --- a/lib/pleroma/http.ex +++ b/lib/pleroma/http.ex @@ -132,6 +132,6 @@ defmodule Pleroma.HTTP do end end - defp default_middleware(), + defp default_middleware, do: [Tesla.Middleware.FollowRedirects, Pleroma.Tesla.Middleware.EncodeUrl] end From c49dece0ddf7f6704afde2e1fc969537e423a455 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 30 Jul 2025 15:13:43 -0700 Subject: [PATCH 43/74] Update test to also cover query encoding --- test/pleroma/http_test.exs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/pleroma/http_test.exs b/test/pleroma/http_test.exs index 803bd451a..6325ea3c5 100644 --- a/test/pleroma/http_test.exs +++ b/test/pleroma/http_test.exs @@ -26,7 +26,7 @@ defmodule Pleroma.HTTPTest do %{method: :post, url: "http://example.com/world"} -> %Tesla.Env{status: 200, body: "world"} - %{method: :get, url: "https://tsundere.love/emoji/Pack%201/koronebless.png"} -> + %{method: :get, url: "https://tsundere.love/emoji/Pack%201/koronebless.png?foo=bar+baz"} -> %Tesla.Env{status: 200, body: "emoji data"} end) @@ -74,13 +74,13 @@ defmodule Pleroma.HTTPTest do test "URL encoding properly encodes URLs with spaces" do clear_config(:test_url_encoding, true) - url_with_space = "https://tsundere.love/emoji/Pack 1/koronebless.png" + url_with_space = "https://tsundere.love/emoji/Pack 1/koronebless.png?foo=bar baz" result = HTTP.get(url_with_space) assert result == {:ok, %Tesla.Env{status: 200, body: "emoji data"}} - properly_encoded_url = "https://tsundere.love/emoji/Pack%201/koronebless.png" + properly_encoded_url = "https://tsundere.love/emoji/Pack%201/koronebless.png?foo=bar+baz" result = HTTP.get(properly_encoded_url) From 842090945aa5700faa222e47985cad542d375314 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 30 Jul 2025 15:42:49 -0700 Subject: [PATCH 44/74] Ensure Hackney and Finch both get the default middleware --- lib/pleroma/http.ex | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/http.ex b/lib/pleroma/http.ex index f0e01d589..bdeb2171e 100644 --- a/lib/pleroma/http.ex +++ b/lib/pleroma/http.ex @@ -110,10 +110,6 @@ defmodule Pleroma.HTTP do extra_middleware end - defp adapter_middlewares({Tesla.Adapter.Finch, _}, extra_middleware) do - default_middleware() ++ extra_middleware - end - defp adapter_middlewares(_, extra_middleware) do # A lot of tests are written expecting unencoded URLs # and the burden of fixing that is high. Also it makes @@ -127,8 +123,9 @@ defmodule Pleroma.HTTP do # Emulate redirects in test env, which are handled by adapters in other environments [Tesla.Middleware.FollowRedirects] + # Hackney and Finch true -> - extra_middleware + default_middleware() ++ extra_middleware end end From 49ba6c88655669fe83b8c58d4070bd7ca5f215f1 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 30 Jul 2025 17:07:22 -0700 Subject: [PATCH 45/74] Rework the URL encoding so it is a public function: Pleroma.HTTP.encode_url/1 --- lib/pleroma/http.ex | 27 ++++++++++++++++++++ lib/pleroma/tesla/middleware/encode_url.ex | 29 +--------------------- 2 files changed, 28 insertions(+), 28 deletions(-) diff --git a/lib/pleroma/http.ex b/lib/pleroma/http.ex index bdeb2171e..9a0868d33 100644 --- a/lib/pleroma/http.ex +++ b/lib/pleroma/http.ex @@ -131,4 +131,31 @@ defmodule Pleroma.HTTP do defp default_middleware, do: [Tesla.Middleware.FollowRedirects, Pleroma.Tesla.Middleware.EncodeUrl] + + def encode_url(url) when is_binary(url) do + URI.parse(url) + |> then(fn parsed -> + path = encode_path(parsed.path) + query = encode_query(parsed.query) + + %{parsed | path: path, query: query} + end) + |> URI.to_string() + end + + defp encode_path(nil), do: nil + + defp encode_path(path) when is_binary(path) do + path + |> URI.decode() + |> URI.encode() + end + + defp encode_query(nil), do: nil + + defp encode_query(query) when is_binary(query) do + query + |> URI.decode_query() + |> URI.encode_query() + end end diff --git a/lib/pleroma/tesla/middleware/encode_url.ex b/lib/pleroma/tesla/middleware/encode_url.ex index ee74c41a1..32c559d3b 100644 --- a/lib/pleroma/tesla/middleware/encode_url.ex +++ b/lib/pleroma/tesla/middleware/encode_url.ex @@ -17,7 +17,7 @@ defmodule Pleroma.Tesla.Middleware.EncodeUrl do @impl Tesla.Middleware def call(%Tesla.Env{url: url} = env, next, _) do - url = encode_url(url) + url = Pleroma.HTTP.encode_url(url) env = %{env | url: url} @@ -26,31 +26,4 @@ defmodule Pleroma.Tesla.Middleware.EncodeUrl do err -> err end end - - defp encode_url(url) when is_binary(url) do - URI.parse(url) - |> then(fn parsed -> - path = encode_path(parsed.path) - query = encode_query(parsed.query) - - %{parsed | path: path, query: query} - end) - |> URI.to_string() - end - - defp encode_path(nil), do: nil - - defp encode_path(path) when is_binary(path) do - path - |> URI.decode() - |> URI.encode() - end - - defp encode_query(nil), do: nil - - defp encode_query(query) when is_binary(query) do - query - |> URI.decode_query() - |> URI.encode_query() - end end From ab4edf7933f245c9e955d4942471fb938a2f7c0e Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 30 Jul 2025 17:20:42 -0700 Subject: [PATCH 46/74] Add proper ReverseProxy test cases --- test/pleroma/reverse_proxy_test.exs | 36 +++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/test/pleroma/reverse_proxy_test.exs b/test/pleroma/reverse_proxy_test.exs index 85e1d0910..034ab28a5 100644 --- a/test/pleroma/reverse_proxy_test.exs +++ b/test/pleroma/reverse_proxy_test.exs @@ -395,4 +395,40 @@ defmodule Pleroma.ReverseProxyTest do assert Conn.get_resp_header(conn, "content-type") == ["application/octet-stream"] end end + + # Hackey is used for Reverse Proxy when Hackney or Finch is the Tesla Adapter + # Gun is able to proxy through Tesla, so it does not need testing as the + # test cases in the Pleroma.HTTPTest module are sufficient + describe "Hackney URL encoding:" do + setup do + ClientMock + |> expect(:request, fn :get, + "https://example.com/emoji/Pack%201/koronebless.png?foo=bar+baz", + _headers, + _body, + _opts -> + {:ok, 200, [{"content-type", "image/png"}], "It works!"} + end) + |> stub(:stream_body, fn _ -> :done end) + |> stub(:close, fn _ -> :ok end) + + :ok + end + + test "properly encodes URLs with spaces", %{conn: conn} do + url_with_space = "https://example.com/emoji/Pack 1/koronebless.png?foo=bar baz" + + result = ReverseProxy.call(conn, url_with_space) + + assert result.status == 200 + end + + test "properly encoded URL should not be altered", %{conn: conn} do + properly_encoded_url = "https://example.com/emoji/Pack%201/koronebless.png?foo=bar+baz" + + result = ReverseProxy.call(conn, properly_encoded_url) + + assert result.status == 200 + end + end end From 425329bacd59cbf15c017aaa4ab271c768076120 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 30 Jul 2025 17:32:08 -0700 Subject: [PATCH 47/74] Add fix to ensure URL is encoded when reverse proxying --- lib/pleroma/reverse_proxy.ex | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/lib/pleroma/reverse_proxy.ex b/lib/pleroma/reverse_proxy.ex index 3c82f9996..cd58f29e4 100644 --- a/lib/pleroma/reverse_proxy.ex +++ b/lib/pleroma/reverse_proxy.ex @@ -158,6 +158,8 @@ defmodule Pleroma.ReverseProxy do Logger.debug("#{__MODULE__} #{method} #{url} #{inspect(headers)}") method = method |> String.downcase() |> String.to_existing_atom() + url = maybe_encode_url(url) + case client().request(method, url, headers, "", opts) do {:ok, code, headers, client} when code in @valid_resp_codes -> {:ok, code, downcase_headers(headers), client} @@ -449,4 +451,18 @@ defmodule Pleroma.ReverseProxy do _ -> delete_resp_header(conn, "content-length") end end + + # Only when Tesla adapter is Hackney or Finch does the URL + # need encoding before Reverse Proxying as both end up + # using the raw Hackney client and cannot leverage our + # EncodeUrl Tesla middleware + # Also do it for test environment + defp maybe_encode_url(url) do + case Application.get_env(:tesla, :adapter) do + Tesla.Adapter.Hackney -> Pleroma.HTTP.encode_url(url) + {Tesla.Adapter.Finch, _} -> Pleroma.HTTP.encode_url(url) + Tesla.Mock -> Pleroma.HTTP.encode_url(url) + _ -> url + end + end end From 4e6f0af4ce66b17687092746d397c04afc56e281 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 30 Jul 2025 17:32:49 -0700 Subject: [PATCH 48/74] Better assertion logic --- test/pleroma/http_test.exs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/pleroma/http_test.exs b/test/pleroma/http_test.exs index 6325ea3c5..0e2551755 100644 --- a/test/pleroma/http_test.exs +++ b/test/pleroma/http_test.exs @@ -76,14 +76,14 @@ defmodule Pleroma.HTTPTest do url_with_space = "https://tsundere.love/emoji/Pack 1/koronebless.png?foo=bar baz" - result = HTTP.get(url_with_space) + {:ok, result} = HTTP.get(url_with_space) - assert result == {:ok, %Tesla.Env{status: 200, body: "emoji data"}} + assert result.status == 200 properly_encoded_url = "https://tsundere.love/emoji/Pack%201/koronebless.png?foo=bar+baz" - result = HTTP.get(properly_encoded_url) + {:ok, result} = HTTP.get(properly_encoded_url) - assert result == {:ok, %Tesla.Env{status: 200, body: "emoji data"}} + assert result.status == 200 end end From 44e56ed756e19a1de324afebc71103c0c6e7ed31 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 30 Jul 2025 18:26:56 -0700 Subject: [PATCH 49/74] Switch to example domain name --- test/pleroma/http_test.exs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/pleroma/http_test.exs b/test/pleroma/http_test.exs index 0e2551755..61347015d 100644 --- a/test/pleroma/http_test.exs +++ b/test/pleroma/http_test.exs @@ -26,7 +26,7 @@ defmodule Pleroma.HTTPTest do %{method: :post, url: "http://example.com/world"} -> %Tesla.Env{status: 200, body: "world"} - %{method: :get, url: "https://tsundere.love/emoji/Pack%201/koronebless.png?foo=bar+baz"} -> + %{method: :get, url: "https://example.com/emoji/Pack%201/koronebless.png?foo=bar+baz"} -> %Tesla.Env{status: 200, body: "emoji data"} end) @@ -74,13 +74,13 @@ defmodule Pleroma.HTTPTest do test "URL encoding properly encodes URLs with spaces" do clear_config(:test_url_encoding, true) - url_with_space = "https://tsundere.love/emoji/Pack 1/koronebless.png?foo=bar baz" + url_with_space = "https://example.com/emoji/Pack 1/koronebless.png?foo=bar baz" {:ok, result} = HTTP.get(url_with_space) assert result.status == 200 - properly_encoded_url = "https://tsundere.love/emoji/Pack%201/koronebless.png?foo=bar+baz" + properly_encoded_url = "https://example.com/emoji/Pack%201/koronebless.png?foo=bar+baz" {:ok, result} = HTTP.get(properly_encoded_url) From 26fe60494246121b59e40898e6b950e61853452c Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 31 Jul 2025 17:35:11 -0700 Subject: [PATCH 50/74] Hashtag searches now return real results from the database --- lib/pleroma/hashtag.ex | 19 +++ .../controllers/search_controller.ex | 60 +-------- test/pleroma/hashtag_test.exs | 37 ++++++ .../controllers/search_controller_test.exs | 118 ++++++++---------- 4 files changed, 109 insertions(+), 125 deletions(-) diff --git a/lib/pleroma/hashtag.ex b/lib/pleroma/hashtag.ex index 3682f0c14..fdb564fec 100644 --- a/lib/pleroma/hashtag.ex +++ b/lib/pleroma/hashtag.ex @@ -130,4 +130,23 @@ defmodule Pleroma.Hashtag do end def get_recipients_for_activity(_activity), do: [] + + def search(query, options \\ []) do + limit = Keyword.get(options, :limit, 20) + offset = Keyword.get(options, :offset, 0) + + query + |> String.downcase() + |> String.trim() + |> then(fn search_term -> + from(ht in Hashtag, + where: fragment("LOWER(?) LIKE ?", ht.name, ^"%#{search_term}%"), + order_by: [asc: ht.name], + limit: ^limit, + offset: ^offset + ) + |> Repo.all() + |> Enum.map(& &1.name) + end) + end end diff --git a/lib/pleroma/web/mastodon_api/controllers/search_controller.ex b/lib/pleroma/web/mastodon_api/controllers/search_controller.ex index d9a1ba41e..e524a36dd 100644 --- a/lib/pleroma/web/mastodon_api/controllers/search_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/search_controller.ex @@ -7,6 +7,7 @@ defmodule Pleroma.Web.MastodonAPI.SearchController do alias Pleroma.Repo alias Pleroma.User + alias Pleroma.Hashtag alias Pleroma.Web.ControllerHelper alias Pleroma.Web.Endpoint alias Pleroma.Web.MastodonAPI.AccountView @@ -120,69 +121,14 @@ defmodule Pleroma.Web.MastodonAPI.SearchController do defp resource_search(:v2, "hashtags", query, options) do tags_path = Endpoint.url() <> "/tag/" - query - |> prepare_tags(options) + Hashtag.search(query, options) |> Enum.map(fn tag -> %{name: tag, url: tags_path <> tag} end) end defp resource_search(:v1, "hashtags", query, options) do - prepare_tags(query, options) - end - - defp prepare_tags(query, options) do - tags = - query - |> preprocess_uri_query() - |> String.split(~r/[^#\w]+/u, trim: true) - |> Enum.uniq_by(&String.downcase/1) - - explicit_tags = Enum.filter(tags, fn tag -> String.starts_with?(tag, "#") end) - - tags = - if Enum.any?(explicit_tags) do - explicit_tags - else - tags - end - - tags = Enum.map(tags, fn tag -> String.trim_leading(tag, "#") end) - - tags = - if Enum.empty?(explicit_tags) && !options[:skip_joined_tag] do - add_joined_tag(tags) - else - tags - end - - Pleroma.Pagination.paginate_list(tags, options) - end - - defp add_joined_tag(tags) do - tags - |> Kernel.++([joined_tag(tags)]) - |> Enum.uniq_by(&String.downcase/1) - end - - # If `query` is a URI, returns last component of its path, otherwise returns `query` - defp preprocess_uri_query(query) do - if query =~ ~r/https?:\/\// do - query - |> String.trim_trailing("/") - |> URI.parse() - |> Map.get(:path) - |> String.split("/") - |> Enum.at(-1) - else - query - end - end - - defp joined_tag(tags) do - tags - |> Enum.map(fn tag -> String.capitalize(tag) end) - |> Enum.join() + Hashtag.search(query, options) end defp with_fallback(f, fallback \\ []) do diff --git a/test/pleroma/hashtag_test.exs b/test/pleroma/hashtag_test.exs index 8531b1879..a84effc5d 100644 --- a/test/pleroma/hashtag_test.exs +++ b/test/pleroma/hashtag_test.exs @@ -14,4 +14,41 @@ defmodule Pleroma.HashtagTest do assert {:name, {"can't be blank", [validation: :required]}} in changeset.errors end end + + describe "search_hashtags" do + test "searches hashtags by partial match" do + {:ok, _} = Hashtag.get_or_create_by_name("car") + {:ok, _} = Hashtag.get_or_create_by_name("racecar") + {:ok, _} = Hashtag.get_or_create_by_name("nascar") + {:ok, _} = Hashtag.get_or_create_by_name("bicycle") + + results = Hashtag.search("car") + assert "car" in results + assert "racecar" in results + assert "nascar" in results + refute "bicycle" in results + + results = Hashtag.search("race") + assert "racecar" in results + refute "car" in results + refute "nascar" in results + refute "bicycle" in results + + results = Hashtag.search("nonexistent") + assert results == [] + end + + test "supports pagination" do + {:ok, _} = Hashtag.get_or_create_by_name("alpha") + {:ok, _} = Hashtag.get_or_create_by_name("beta") + {:ok, _} = Hashtag.get_or_create_by_name("gamma") + {:ok, _} = Hashtag.get_or_create_by_name("delta") + + results = Hashtag.search("a", limit: 2) + assert length(results) == 2 + + results = Hashtag.search("a", limit: 2, offset: 1) + assert length(results) == 2 + end + end end diff --git a/test/pleroma/web/mastodon_api/controllers/search_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/search_controller_test.exs index d8263dfad..1fbc7c9c6 100644 --- a/test/pleroma/web/mastodon_api/controllers/search_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/search_controller_test.exs @@ -130,84 +130,66 @@ defmodule Pleroma.Web.MastodonAPI.SearchControllerTest do assert [] = results["statuses"] end - test "constructs hashtags from search query", %{conn: conn} do + test "returns empty results when no hashtags match", %{conn: conn} do results = conn - |> get("/api/v2/search?#{URI.encode_query(%{q: "some text with #explicit #hashtags"})}") + |> get("/api/v2/search?#{URI.encode_query(%{q: "nonexistent"})}") |> json_response_and_validate_schema(200) - assert results["hashtags"] == [ - %{"name" => "explicit", "url" => "#{Endpoint.url()}/tag/explicit"}, - %{"name" => "hashtags", "url" => "#{Endpoint.url()}/tag/hashtags"} - ] - - results = - conn - |> get("/api/v2/search?#{URI.encode_query(%{q: "john doe JOHN DOE"})}") - |> json_response_and_validate_schema(200) - - assert results["hashtags"] == [ - %{"name" => "john", "url" => "#{Endpoint.url()}/tag/john"}, - %{"name" => "doe", "url" => "#{Endpoint.url()}/tag/doe"}, - %{"name" => "JohnDoe", "url" => "#{Endpoint.url()}/tag/JohnDoe"} - ] - - results = - conn - |> get("/api/v2/search?#{URI.encode_query(%{q: "accident-prone"})}") - |> json_response_and_validate_schema(200) - - assert results["hashtags"] == [ - %{"name" => "accident", "url" => "#{Endpoint.url()}/tag/accident"}, - %{"name" => "prone", "url" => "#{Endpoint.url()}/tag/prone"}, - %{"name" => "AccidentProne", "url" => "#{Endpoint.url()}/tag/AccidentProne"} - ] - - results = - conn - |> get("/api/v2/search?#{URI.encode_query(%{q: "https://shpposter.club/users/shpuld"})}") - |> json_response_and_validate_schema(200) - - assert results["hashtags"] == [ - %{"name" => "shpuld", "url" => "#{Endpoint.url()}/tag/shpuld"} - ] - - results = - conn - |> get( - "/api/v2/search?#{URI.encode_query(%{q: "https://www.washingtonpost.com/sports/2020/06/10/" <> "nascar-ban-display-confederate-flag-all-events-properties/"})}" - ) - |> json_response_and_validate_schema(200) - - assert results["hashtags"] == [ - %{"name" => "nascar", "url" => "#{Endpoint.url()}/tag/nascar"}, - %{"name" => "ban", "url" => "#{Endpoint.url()}/tag/ban"}, - %{"name" => "display", "url" => "#{Endpoint.url()}/tag/display"}, - %{"name" => "confederate", "url" => "#{Endpoint.url()}/tag/confederate"}, - %{"name" => "flag", "url" => "#{Endpoint.url()}/tag/flag"}, - %{"name" => "all", "url" => "#{Endpoint.url()}/tag/all"}, - %{"name" => "events", "url" => "#{Endpoint.url()}/tag/events"}, - %{"name" => "properties", "url" => "#{Endpoint.url()}/tag/properties"}, - %{ - "name" => "NascarBanDisplayConfederateFlagAllEventsProperties", - "url" => - "#{Endpoint.url()}/tag/NascarBanDisplayConfederateFlagAllEventsProperties" - } - ] + assert results["hashtags"] == [] end test "supports pagination of hashtags search results", %{conn: conn} do + user = insert(:user) + + {:ok, _activity1} = CommonAPI.post(user, %{status: "First #alpha hashtag"}) + {:ok, _activity2} = CommonAPI.post(user, %{status: "Second #beta hashtag"}) + {:ok, _activity3} = CommonAPI.post(user, %{status: "Third #gamma hashtag"}) + {:ok, _activity4} = CommonAPI.post(user, %{status: "Fourth #delta hashtag"}) + results = conn - |> get( - "/api/v2/search?#{URI.encode_query(%{q: "#some #text #with #hashtags", limit: 2, offset: 1})}" - ) + |> get("/api/v2/search?#{URI.encode_query(%{q: "a", limit: 2, offset: 1})}") |> json_response_and_validate_schema(200) - assert results["hashtags"] == [ - %{"name" => "text", "url" => "#{Endpoint.url()}/tag/text"}, - %{"name" => "with", "url" => "#{Endpoint.url()}/tag/with"} - ] + hashtag_names = Enum.map(results["hashtags"], & &1["name"]) + + # Should return 2 hashtags (alpha, beta, gamma, delta all contain 'a') + # With offset 1, we skip the first one, so we get 2 of the remaining 3 + assert length(hashtag_names) == 2 + assert Enum.all?(hashtag_names, &String.contains?(&1, "a")) + end + + test "searches real hashtags from database", %{conn: conn} do + user = insert(:user) + + {:ok, _activity1} = CommonAPI.post(user, %{status: "Check out this #car"}) + {:ok, _activity2} = CommonAPI.post(user, %{status: "Fast #racecar on the track"}) + {:ok, _activity3} = CommonAPI.post(user, %{status: "NASCAR #nascar racing"}) + + results = + conn + |> get("/api/v2/search?#{URI.encode_query(%{q: "car"})}") + |> json_response_and_validate_schema(200) + + hashtag_names = Enum.map(results["hashtags"], & &1["name"]) + + # Should return car, racecar, and nascar since they all contain "car" + assert "car" in hashtag_names + assert "racecar" in hashtag_names + assert "nascar" in hashtag_names + + # Search for "race" - should return racecar + results = + conn + |> get("/api/v2/search?#{URI.encode_query(%{q: "race"})}") + |> json_response_and_validate_schema(200) + + hashtag_names = Enum.map(results["hashtags"], & &1["name"]) + + assert "racecar" in hashtag_names + refute "car" in hashtag_names + refute "nascar" in hashtag_names end test "excludes a blocked users from search results", %{conn: conn} do @@ -314,7 +296,7 @@ defmodule Pleroma.Web.MastodonAPI.SearchControllerTest do [account | _] = results["accounts"] assert account["id"] == to_string(user_three.id) - assert results["hashtags"] == ["2hu"] + assert results["hashtags"] == [] [status] = results["statuses"] assert status["id"] == to_string(activity.id) From 93c144e397d408d7ff1761640e12fb51e333b2ce Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 31 Jul 2025 17:46:32 -0700 Subject: [PATCH 51/74] Improve hashtag search with multi word queries --- changelog.d/hashtag-search.change | 1 + lib/pleroma/hashtag.ex | 22 +++++++--- test/pleroma/hashtag_test.exs | 44 +++++++++++++++++++ .../controllers/search_controller_test.exs | 31 +++++++++++++ 4 files changed, 92 insertions(+), 6 deletions(-) create mode 100644 changelog.d/hashtag-search.change diff --git a/changelog.d/hashtag-search.change b/changelog.d/hashtag-search.change new file mode 100644 index 000000000..f17e711ce --- /dev/null +++ b/changelog.d/hashtag-search.change @@ -0,0 +1 @@ +Hashtag searches return real results based on words in your query diff --git a/lib/pleroma/hashtag.ex b/lib/pleroma/hashtag.ex index fdb564fec..99e6eb39b 100644 --- a/lib/pleroma/hashtag.ex +++ b/lib/pleroma/hashtag.ex @@ -135,18 +135,28 @@ defmodule Pleroma.Hashtag do limit = Keyword.get(options, :limit, 20) offset = Keyword.get(options, :offset, 0) - query - |> String.downcase() - |> String.trim() - |> then(fn search_term -> + search_terms = + query + |> String.downcase() + |> String.trim() + |> String.split(~r/\s+/) + |> Enum.filter(&(&1 != "")) + + if Enum.empty?(search_terms) do + [] + else + # Use PostgreSQL's ANY operator with array for efficient multi-term search + # This is much more efficient than multiple OR clauses + search_patterns = Enum.map(search_terms, &"%#{&1}%") + from(ht in Hashtag, - where: fragment("LOWER(?) LIKE ?", ht.name, ^"%#{search_term}%"), + where: fragment("LOWER(?) LIKE ANY(?)", ht.name, ^search_patterns), order_by: [asc: ht.name], limit: ^limit, offset: ^offset ) |> Repo.all() |> Enum.map(& &1.name) - end) + end end end diff --git a/test/pleroma/hashtag_test.exs b/test/pleroma/hashtag_test.exs index a84effc5d..907c5ff40 100644 --- a/test/pleroma/hashtag_test.exs +++ b/test/pleroma/hashtag_test.exs @@ -38,6 +38,35 @@ defmodule Pleroma.HashtagTest do assert results == [] end + test "searches hashtags by multiple words in query" do + # Create some hashtags + {:ok, _} = Hashtag.get_or_create_by_name("computer") + {:ok, _} = Hashtag.get_or_create_by_name("laptop") + {:ok, _} = Hashtag.get_or_create_by_name("desktop") + {:ok, _} = Hashtag.get_or_create_by_name("phone") + + # Search for "new computer" - should return "computer" + results = Hashtag.search("new computer") + assert "computer" in results + refute "laptop" in results + refute "desktop" in results + refute "phone" in results + + # Search for "computer laptop" - should return both + results = Hashtag.search("computer laptop") + assert "computer" in results + assert "laptop" in results + refute "desktop" in results + refute "phone" in results + + # Search for "new phone" - should return "phone" + results = Hashtag.search("new phone") + assert "phone" in results + refute "computer" in results + refute "laptop" in results + refute "desktop" in results + end + test "supports pagination" do {:ok, _} = Hashtag.get_or_create_by_name("alpha") {:ok, _} = Hashtag.get_or_create_by_name("beta") @@ -50,5 +79,20 @@ defmodule Pleroma.HashtagTest do results = Hashtag.search("a", limit: 2, offset: 1) assert length(results) == 2 end + + test "handles many search terms efficiently" do + # Create hashtags + {:ok, _} = Hashtag.get_or_create_by_name("computer") + {:ok, _} = Hashtag.get_or_create_by_name("laptop") + {:ok, _} = Hashtag.get_or_create_by_name("phone") + {:ok, _} = Hashtag.get_or_create_by_name("tablet") + + # Search with many terms - should be efficient with PostgreSQL ANY operator + results = Hashtag.search("new fast computer laptop phone tablet device") + assert "computer" in results + assert "laptop" in results + assert "phone" in results + assert "tablet" in results + end end end diff --git a/test/pleroma/web/mastodon_api/controllers/search_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/search_controller_test.exs index 1fbc7c9c6..8b4c6add2 100644 --- a/test/pleroma/web/mastodon_api/controllers/search_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/search_controller_test.exs @@ -139,6 +139,37 @@ defmodule Pleroma.Web.MastodonAPI.SearchControllerTest do assert results["hashtags"] == [] end + test "searches hashtags by multiple words in query", %{conn: conn} do + user = insert(:user) + + {:ok, _activity1} = CommonAPI.post(user, %{status: "This is my new #computer"}) + {:ok, _activity2} = CommonAPI.post(user, %{status: "Check out this #laptop"}) + {:ok, _activity3} = CommonAPI.post(user, %{status: "My #desktop setup"}) + {:ok, _activity4} = CommonAPI.post(user, %{status: "New #phone arrived"}) + + results = + conn + |> get("/api/v2/search?#{URI.encode_query(%{q: "new computer"})}") + |> json_response_and_validate_schema(200) + + hashtag_names = Enum.map(results["hashtags"], & &1["name"]) + assert "computer" in hashtag_names + refute "laptop" in hashtag_names + refute "desktop" in hashtag_names + refute "phone" in hashtag_names + + results = + conn + |> get("/api/v2/search?#{URI.encode_query(%{q: "computer laptop"})}") + |> json_response_and_validate_schema(200) + + hashtag_names = Enum.map(results["hashtags"], & &1["name"]) + assert "computer" in hashtag_names + assert "laptop" in hashtag_names + refute "desktop" in hashtag_names + refute "phone" in hashtag_names + end + test "supports pagination of hashtags search results", %{conn: conn} do user = insert(:user) From b1acc9281a69602b71ba35166e787efd000efa50 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 31 Jul 2025 18:02:33 -0700 Subject: [PATCH 52/74] Use ranking to improve order of results --- lib/pleroma/hashtag.ex | 36 ++++++++++++++++++++++++++++++++--- test/pleroma/hashtag_test.exs | 36 +++++++++++++++++++++++++++++++---- 2 files changed, 65 insertions(+), 7 deletions(-) diff --git a/lib/pleroma/hashtag.ex b/lib/pleroma/hashtag.ex index 99e6eb39b..8cffe840f 100644 --- a/lib/pleroma/hashtag.ex +++ b/lib/pleroma/hashtag.ex @@ -149,9 +149,39 @@ defmodule Pleroma.Hashtag do # This is much more efficient than multiple OR clauses search_patterns = Enum.map(search_terms, &"%#{&1}%") - from(ht in Hashtag, - where: fragment("LOWER(?) LIKE ANY(?)", ht.name, ^search_patterns), - order_by: [asc: ht.name], + # Create ranking query that prioritizes exact matches and closer matches + # Use a subquery to properly handle computed columns in ORDER BY + base_query = + from(ht in Hashtag, + where: fragment("LOWER(?) LIKE ANY(?)", ht.name, ^search_patterns), + select: %{ + name: ht.name, + # Ranking: exact matches get highest priority (0), then prefix matches (1), then contains (2) + match_rank: + fragment( + """ + CASE + WHEN LOWER(?) = ANY(?) THEN 0 + WHEN LOWER(?) LIKE ANY(?) THEN 1 + ELSE 2 + END + """, + ht.name, + ^search_terms, + ht.name, + ^Enum.map(search_terms, &"#{&1}%") + ), + # Secondary sort by name length (shorter names first) + name_length: fragment("LENGTH(?)", ht.name) + } + ) + + from(result in subquery(base_query), + order_by: [ + asc: result.match_rank, + asc: result.name_length, + asc: result.name + ], limit: ^limit, offset: ^offset ) diff --git a/test/pleroma/hashtag_test.exs b/test/pleroma/hashtag_test.exs index 907c5ff40..d15c7d1d9 100644 --- a/test/pleroma/hashtag_test.exs +++ b/test/pleroma/hashtag_test.exs @@ -39,7 +39,6 @@ defmodule Pleroma.HashtagTest do end test "searches hashtags by multiple words in query" do - # Create some hashtags {:ok, _} = Hashtag.get_or_create_by_name("computer") {:ok, _} = Hashtag.get_or_create_by_name("laptop") {:ok, _} = Hashtag.get_or_create_by_name("desktop") @@ -80,19 +79,48 @@ defmodule Pleroma.HashtagTest do assert length(results) == 2 end - test "handles many search terms efficiently" do - # Create hashtags + test "handles matching many search terms" do {:ok, _} = Hashtag.get_or_create_by_name("computer") {:ok, _} = Hashtag.get_or_create_by_name("laptop") {:ok, _} = Hashtag.get_or_create_by_name("phone") {:ok, _} = Hashtag.get_or_create_by_name("tablet") - # Search with many terms - should be efficient with PostgreSQL ANY operator results = Hashtag.search("new fast computer laptop phone tablet device") assert "computer" in results assert "laptop" in results assert "phone" in results assert "tablet" in results end + + test "ranks results by match quality" do + {:ok, _} = Hashtag.get_or_create_by_name("my_computer") + {:ok, _} = Hashtag.get_or_create_by_name("computer_science") + {:ok, _} = Hashtag.get_or_create_by_name("computer") + + results = Hashtag.search("computer") + + # Exact match first + assert Enum.at(results, 0) == "computer" + + # Prefix match would be next + assert Enum.at(results, 1) == "computer_science" + + # worst match is last + assert Enum.at(results, 2) == "my_computer" + end + + test "prioritizes shorter names when ranking is equal" do + # Create hashtags with same ranking but different lengths + {:ok, _} = Hashtag.get_or_create_by_name("car") + {:ok, _} = Hashtag.get_or_create_by_name("racecar") + {:ok, _} = Hashtag.get_or_create_by_name("nascar") + + # Search for "car" - shorter names should come first + results = Hashtag.search("car") + # Shortest exact match first + assert Enum.at(results, 0) == "car" + assert "racecar" in results + assert "nascar" in results + end end end From 97e668f4aa5e4ae7b45158b9bc3ff1982baf4089 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 31 Jul 2025 18:07:05 -0700 Subject: [PATCH 53/74] Alpha sort the aliases --- lib/pleroma/web/mastodon_api/controllers/search_controller.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/mastodon_api/controllers/search_controller.ex b/lib/pleroma/web/mastodon_api/controllers/search_controller.ex index e524a36dd..53f1216fd 100644 --- a/lib/pleroma/web/mastodon_api/controllers/search_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/search_controller.ex @@ -5,9 +5,9 @@ defmodule Pleroma.Web.MastodonAPI.SearchController do use Pleroma.Web, :controller + alias Pleroma.Hashtag alias Pleroma.Repo alias Pleroma.User - alias Pleroma.Hashtag alias Pleroma.Web.ControllerHelper alias Pleroma.Web.Endpoint alias Pleroma.Web.MastodonAPI.AccountView From 19f32f7b0981bdfc0da30df011ce1ef3df44dbe5 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 31 Jul 2025 18:17:58 -0700 Subject: [PATCH 54/74] Strip hashtag prefixes Users may actually type in a literal hashtag into the search, so this will ensure it still returns results. --- lib/pleroma/hashtag.ex | 2 ++ test/pleroma/hashtag_test.exs | 20 ++++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/lib/pleroma/hashtag.ex b/lib/pleroma/hashtag.ex index 8cffe840f..507bc09bd 100644 --- a/lib/pleroma/hashtag.ex +++ b/lib/pleroma/hashtag.ex @@ -141,6 +141,8 @@ defmodule Pleroma.Hashtag do |> String.trim() |> String.split(~r/\s+/) |> Enum.filter(&(&1 != "")) + |> Enum.map(&String.trim_leading(&1, "#")) + |> Enum.filter(&(&1 != "")) if Enum.empty?(search_terms) do [] diff --git a/test/pleroma/hashtag_test.exs b/test/pleroma/hashtag_test.exs index d15c7d1d9..0e16b8155 100644 --- a/test/pleroma/hashtag_test.exs +++ b/test/pleroma/hashtag_test.exs @@ -122,5 +122,25 @@ defmodule Pleroma.HashtagTest do assert "racecar" in results assert "nascar" in results end + + test "handles hashtag symbols in search query" do + {:ok, _} = Hashtag.get_or_create_by_name("computer") + {:ok, _} = Hashtag.get_or_create_by_name("laptop") + {:ok, _} = Hashtag.get_or_create_by_name("phone") + + results_with_hash = Hashtag.search("#computer #laptop") + results_without_hash = Hashtag.search("computer laptop") + + assert results_with_hash == results_without_hash + + results_mixed = Hashtag.search("#computer laptop #phone") + assert "computer" in results_mixed + assert "laptop" in results_mixed + assert "phone" in results_mixed + + results_only_hash = Hashtag.search("#computer") + results_no_hash = Hashtag.search("computer") + assert results_only_hash == results_no_hash + end end end From eac8ef79513e9ce425cec30e8f12489eccdf3305 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 1 Aug 2025 10:41:53 -0700 Subject: [PATCH 55/74] Credo --- lib/pleroma/hashtag.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/hashtag.ex b/lib/pleroma/hashtag.ex index 507bc09bd..91c30c6e7 100644 --- a/lib/pleroma/hashtag.ex +++ b/lib/pleroma/hashtag.ex @@ -158,7 +158,8 @@ defmodule Pleroma.Hashtag do where: fragment("LOWER(?) LIKE ANY(?)", ht.name, ^search_patterns), select: %{ name: ht.name, - # Ranking: exact matches get highest priority (0), then prefix matches (1), then contains (2) + # Ranking: exact matches get highest priority (0) + # then prefix matches (1), then contains (2) match_rank: fragment( """ From f66a877af7dcb12f7206f339fd350b265c27e1e9 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 1 Aug 2025 10:52:04 -0700 Subject: [PATCH 56/74] Disable automatic CI jobs for every pushed branch --- .gitlab-ci.yml | 3 ++- changelog.d/gitlabci.skip | 0 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/gitlabci.skip diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bfd9bf414..bd36387c9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -14,9 +14,10 @@ variables: &global_variables workflow: rules: - if: $CI_PIPELINE_SOURCE == "merge_request_event" + - if: $CI_COMMIT_BRANCH == "develop" + - if: $CI_COMMIT_BRANCH == "stable" - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS when: never - - if: $CI_COMMIT_BRANCH cache: &global_cache_policy key: $CI_JOB_IMAGE-$CI_COMMIT_SHORT_SHA diff --git a/changelog.d/gitlabci.skip b/changelog.d/gitlabci.skip new file mode 100644 index 000000000..e69de29bb From 4b01c0f165d1f930291fa1d533dfda599e6e7aab Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 1 Aug 2025 11:41:36 -0700 Subject: [PATCH 57/74] Update Tesla to 1.15.3 --- changelog.d/tesla.change | 1 + mix.lock | 12 ++++++------ 2 files changed, 7 insertions(+), 6 deletions(-) create mode 100644 changelog.d/tesla.change diff --git a/changelog.d/tesla.change b/changelog.d/tesla.change new file mode 100644 index 000000000..bd0ec6e94 --- /dev/null +++ b/changelog.d/tesla.change @@ -0,0 +1 @@ +Updated Tesla to 1.15.3 diff --git a/mix.lock b/mix.lock index a708087c9..2f02ba6f1 100644 --- a/mix.lock +++ b/mix.lock @@ -11,7 +11,7 @@ "cachex": {:hex, :cachex, "3.6.0", "14a1bfbeee060dd9bec25a5b6f4e4691e3670ebda28c8ba2884b12fe30b36bf8", [:mix], [{:eternal, "~> 1.2", [hex: :eternal, repo: "hexpm", optional: false]}, {:jumper, "~> 1.0", [hex: :jumper, repo: "hexpm", optional: false]}, {:sleeplocks, "~> 1.1", [hex: :sleeplocks, repo: "hexpm", optional: false]}, {:unsafe, "~> 1.0", [hex: :unsafe, repo: "hexpm", optional: false]}], "hexpm", "ebf24e373883bc8e0c8d894a63bbe102ae13d918f790121f5cfe6e485cc8e2e2"}, "calendar": {:hex, :calendar, "1.0.0", "f52073a708528482ec33d0a171954ca610fe2bd28f1e871f247dc7f1565fa807", [:mix], [{:tzdata, "~> 0.5.20 or ~> 0.1.201603 or ~> 1.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm", "990e9581920c82912a5ee50e62ff5ef96da6b15949a2ee4734f935fdef0f0a6f"}, "captcha": {:git, "https://git.pleroma.social/pleroma/elixir-libraries/elixir-captcha.git", "e7b7cc34cc16b383461b966484c297e4ec9aeef6", [ref: "e7b7cc34cc16b383461b966484c297e4ec9aeef6"]}, - "castore": {:hex, :castore, "1.0.8", "dedcf20ea746694647f883590b82d9e96014057aff1d44d03ec90f36a5c0dc6e", [:mix], [], "hexpm", "0b2b66d2ee742cb1d9cb8c8be3b43c3a70ee8651f37b75a8b982e036752983f1"}, + "castore": {:hex, :castore, "1.0.14", "4582dd7d630b48cf5e1ca8d3d42494db51e406b7ba704e81fbd401866366896a", [:mix], [], "hexpm", "7bc1b65249d31701393edaaac18ec8398d8974d52c647b7904d01b964137b9f4"}, "cc_precompiler": {:hex, :cc_precompiler, "0.1.9", "e8d3364f310da6ce6463c3dd20cf90ae7bbecbf6c5203b98bf9b48035592649b", [:mix], [{:elixir_make, "~> 0.7", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm", "9dcab3d0f3038621f1601f13539e7a9ee99843862e66ad62827b0c42b2f58a54"}, "certifi": {:hex, :certifi, "2.12.0", "2d1cca2ec95f59643862af91f001478c9863c2ac9cb6e2f89780bfd8de987329", [:rebar3], [], "hexpm", "ee68d85df22e554040cdb4be100f33873ac6051387baf6a8f6ce82272340ff1c"}, "combine": {:hex, :combine, "0.10.0", "eff8224eeb56498a2af13011d142c5e7997a80c8f5b97c499f84c841032e429f", [:mix], [], "hexpm", "1b1dbc1790073076580d0d1d64e42eae2366583e7aecd455d1215b0d16f2451b"}, @@ -22,7 +22,7 @@ "covertool": {:hex, :covertool, "2.0.6", "4a291b4e3449025b0595d8f44c8d7635d4f48f033be2ce88d22a329f36f94a91", [:rebar3], [], "hexpm", "5db3fcd82180d8ea4ad857d4d1ab21a8d31b5aee0d60d2f6c0f9e25a411d1e21"}, "cowboy": {:hex, :cowboy, "2.12.0", "f276d521a1ff88b2b9b4c54d0e753da6c66dd7be6c9fca3d9418b561828a3731", [:make, :rebar3], [{:cowlib, "2.13.0", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "1.8.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "8a7abe6d183372ceb21caa2709bec928ab2b72e18a3911aa1771639bef82651e"}, "cowboy_telemetry": {:hex, :cowboy_telemetry, "0.4.0", "f239f68b588efa7707abce16a84d0d2acf3a0f50571f8bb7f56a15865aae820c", [:rebar3], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "7d98bac1ee4565d31b62d59f8823dfd8356a169e7fcbb83831b8a5397404c9de"}, - "cowlib": {:hex, :cowlib, "2.13.0", "db8f7505d8332d98ef50a3ef34b34c1afddec7506e4ee4dd4a3a266285d282ca", [:make, :rebar3], [], "hexpm", "e1e1284dc3fc030a64b1ad0d8382ae7e99da46c3246b815318a4b848873800a4"}, + "cowlib": {:hex, :cowlib, "2.15.0", "3c97a318a933962d1c12b96ab7c1d728267d2c523c25a5b57b0f93392b6e9e25", [:make, :rebar3], [], "hexpm", "4f00c879a64b4fe7c8fcb42a4281925e9ffdb928820b03c3ad325a617e857532"}, "credo": {:hex, :credo, "1.7.12", "9e3c20463de4b5f3f23721527fcaf16722ec815e70ff6c60b86412c695d426c1", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "8493d45c656c5427d9c729235b99d498bd133421f3e0a683e5c1b561471291e5"}, "crontab": {:hex, :crontab, "1.1.8", "2ce0e74777dfcadb28a1debbea707e58b879e6aa0ffbf9c9bb540887bce43617", [:mix], [{:ecto, "~> 1.0 or ~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm"}, "custom_base": {:hex, :custom_base, "0.2.1", "4a832a42ea0552299d81652aa0b1f775d462175293e99dfbe4d7dbaab785a706", [:mix], [], "hexpm", "8df019facc5ec9603e94f7270f1ac73ddf339f56ade76a721eaa57c1493ba463"}, @@ -80,8 +80,8 @@ "meck": {:hex, :meck, "0.9.2", "85ccbab053f1db86c7ca240e9fc718170ee5bda03810a6292b5306bf31bae5f5", [:rebar3], [], "hexpm", "81344f561357dc40a8344afa53767c32669153355b626ea9fcbc8da6b3045826"}, "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"}, "mime": {:hex, :mime, "1.6.0", "dabde576a497cef4bbdd60aceee8160e02a6c89250d6c0b29e56c0dfb00db3d2", [:mix], [], "hexpm", "31a1a8613f8321143dde1dafc36006a17d28d02bdfecb9e95a880fa7aabd19a7"}, - "mimerl": {:hex, :mimerl, "1.3.0", "d0cd9fc04b9061f82490f6581e0128379830e78535e017f7780f37fea7545726", [:rebar3], [], "hexpm", "a1e15a50d1887217de95f0b9b0793e32853f7c258a5cd227650889b38839fe9d"}, - "mint": {:hex, :mint, "1.6.1", "065e8a5bc9bbd46a41099dfea3e0656436c5cbcb6e741c80bd2bad5cd872446f", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1 or ~> 0.2.0", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "4fc518dcc191d02f433393a72a7ba3f6f94b101d094cb6bf532ea54c89423780"}, + "mimerl": {:hex, :mimerl, "1.4.0", "3882a5ca67fbbe7117ba8947f27643557adec38fa2307490c4c4207624cb213b", [:rebar3], [], "hexpm", "13af15f9f68c65884ecca3a3891d50a7b57d82152792f3e19d88650aa126b144"}, + "mint": {:hex, :mint, "1.7.1", "113fdb2b2f3b59e47c7955971854641c61f378549d73e829e1768de90fc1abf1", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1 or ~> 0.2.0 or ~> 1.0", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "fceba0a4d0f24301ddee3024ae116df1c3f4bb7a563a731f45fdfeb9d39a231b"}, "mochiweb": {:hex, :mochiweb, "2.18.0", "eb55f1db3e6e960fac4e6db4e2db9ec3602cc9f30b86cd1481d56545c3145d2e", [:rebar3], [], "hexpm"}, "mock": {:hex, :mock, "0.3.8", "7046a306b71db2488ef54395eeb74df0a7f335a7caca4a3d3875d1fc81c884dd", [:mix], [{:meck, "~> 0.9.2", [hex: :meck, repo: "hexpm", optional: false]}], "hexpm", "7fa82364c97617d79bb7d15571193fc0c4fe5afd0c932cef09426b3ee6fe2022"}, "mogrify": {:hex, :mogrify, "0.9.3", "238c782f00271dace01369ad35ae2e9dd020feee3443b9299ea5ea6bed559841", [:mix], [], "hexpm", "0189b1e1de27455f2b9ae8cf88239cefd23d38de9276eb5add7159aea51731e6"}, @@ -139,14 +139,14 @@ "telemetry_metrics": {:hex, :telemetry_metrics, "0.6.2", "2caabe9344ec17eafe5403304771c3539f3b6e2f7fb6a6f602558c825d0d0bfb", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "9b43db0dc33863930b9ef9d27137e78974756f5f198cae18409970ed6fa5b561"}, "telemetry_metrics_prometheus_core": {:hex, :telemetry_metrics_prometheus_core, "1.2.0", "b583c3f18508f5c5561b674d16cf5d9afd2ea3c04505b7d92baaeac93c1b8260", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.6", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "9cba950e1c4733468efbe3f821841f34ac05d28e7af7798622f88ecdbbe63ea3"}, "telemetry_poller": {:hex, :telemetry_poller, "1.0.0", "db91bb424e07f2bb6e73926fcafbfcbcb295f0193e0a00e825e589a0a47e8453", [:rebar3], [{:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "b3a24eafd66c3f42da30fc3ca7dda1e9d546c12250a2d60d7b81d264fbec4f6e"}, - "tesla": {:hex, :tesla, "1.11.0", "81b2b10213dddb27105ec6102d9eb0cc93d7097a918a0b1594f2dfd1a4601190", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:exjsx, ">= 3.0.0", [hex: :exjsx, repo: "hexpm", optional: true]}, {:finch, "~> 0.13", [hex: :finch, repo: "hexpm", optional: true]}, {:fuse, "~> 2.4", [hex: :fuse, repo: "hexpm", optional: true]}, {:gun, ">= 1.0.0", [hex: :gun, repo: "hexpm", optional: true]}, {:hackney, "~> 1.6", [hex: :hackney, repo: "hexpm", optional: true]}, {:ibrowse, "4.4.2", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.0", [hex: :mint, repo: "hexpm", optional: true]}, {:msgpax, "~> 2.3", [hex: :msgpax, repo: "hexpm", optional: true]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm", "b83ab5d4c2d202e1ea2b7e17a49f788d49a699513d7c4f08f2aef2c281be69db"}, + "tesla": {:hex, :tesla, "1.15.3", "3a2b5c37f09629b8dcf5d028fbafc9143c0099753559d7fe567eaabfbd9b8663", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:exjsx, ">= 3.0.0", [hex: :exjsx, repo: "hexpm", optional: true]}, {:finch, "~> 0.13", [hex: :finch, repo: "hexpm", optional: true]}, {:fuse, "~> 2.4", [hex: :fuse, repo: "hexpm", optional: true]}, {:gun, ">= 1.0.0", [hex: :gun, repo: "hexpm", optional: true]}, {:hackney, "~> 1.21", [hex: :hackney, repo: "hexpm", optional: true]}, {:ibrowse, "4.4.2", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.0", [hex: :mint, repo: "hexpm", optional: true]}, {:mox, "~> 1.0", [hex: :mox, repo: "hexpm", optional: true]}, {:msgpax, "~> 2.3", [hex: :msgpax, repo: "hexpm", optional: true]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm", "98bb3d4558abc67b92fb7be4cd31bb57ca8d80792de26870d362974b58caeda7"}, "thousand_island": {:hex, :thousand_island, "1.3.5", "6022b6338f1635b3d32406ff98d68b843ba73b3aa95cfc27154223244f3a6ca5", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "2be6954916fdfe4756af3239fb6b6d75d0b8063b5df03ba76fd8a4c87849e180"}, "timex": {:hex, :timex, "3.7.7", "3ed093cae596a410759104d878ad7b38e78b7c2151c6190340835515d4a46b8a", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 1.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm", "0ec4b09f25fe311321f9fc04144a7e3affe48eb29481d7a5583849b6c4dfa0a7"}, "toml": {:hex, :toml, "0.7.0", "fbcd773caa937d0c7a02c301a1feea25612720ac3fa1ccb8bfd9d30d822911de", [:mix], [], "hexpm", "0690246a2478c1defd100b0c9b89b4ea280a22be9a7b313a8a058a2408a2fa70"}, "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "bd4fde4c15f3e993a999e019d64347489b91b7a9096af68b2bdadd192afa693f"}, "tzdata": {:hex, :tzdata, "1.0.5", "69f1ee029a49afa04ad77801febaf69385f3d3e3d1e4b56b9469025677b89a28", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "55519aa2a99e5d2095c1e61cc74c9be69688f8ab75c27da724eb8279ff402a5a"}, "ueberauth": {:hex, :ueberauth, "0.10.7", "5a31cbe11e7ce5c7484d745dc9e1f11948e89662f8510d03c616de03df581ebd", [:mix], [{:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "0bccf73e2ffd6337971340832947ba232877aa8122dba4c95be9f729c8987377"}, - "unicode_util_compat": {:hex, :unicode_util_compat, "0.7.0", "bc84380c9ab48177092f43ac89e4dfa2c6d62b40b8bd132b1059ecc7232f9a78", [:rebar3], [], "hexpm", "25eee6d67df61960cf6a794239566599b09e17e668d3700247bc498638152521"}, + "unicode_util_compat": {:hex, :unicode_util_compat, "0.7.1", "a48703a25c170eedadca83b11e88985af08d35f37c6f664d6dcfb106a97782fc", [:rebar3], [], "hexpm", "b3a917854ce3ae233619744ad1e0102e05673136776fb2fa76234f3e03b23642"}, "unsafe": {:hex, :unsafe, "1.0.2", "23c6be12f6c1605364801f4b47007c0c159497d0446ad378b5cf05f1855c0581", [:mix], [], "hexpm", "b485231683c3ab01a9cd44cb4a79f152c6f3bb87358439c6f68791b85c2df675"}, "vix": {:hex, :vix, "0.26.0", "027f10b6969b759318be84bd0bd8c88af877445e4e41cf96a0460392cea5399c", [:make, :mix], [{:castore, "~> 1.0 or ~> 0.1", [hex: :castore, repo: "hexpm", optional: false]}, {:cc_precompiler, "~> 0.2 or ~> 0.1.4", [hex: :cc_precompiler, repo: "hexpm", optional: false]}, {:elixir_make, "~> 0.8 or ~> 0.7.3", [hex: :elixir_make, repo: "hexpm", optional: false]}, {:kino, "~> 0.7", [hex: :kino, repo: "hexpm", optional: true]}], "hexpm", "71b0a79ae7f199cacfc8e679b0e4ba25ee47dc02e182c5b9097efb29fbe14efd"}, "web_push_encryption": {:hex, :web_push_encryption, "0.3.1", "76d0e7375142dfee67391e7690e89f92578889cbcf2879377900b5620ee4708d", [:mix], [{:httpoison, "~> 1.0", [hex: :httpoison, repo: "hexpm", optional: false]}, {:jose, "~> 1.11.1", [hex: :jose, repo: "hexpm", optional: false]}], "hexpm", "4f82b2e57622fb9337559058e8797cb0df7e7c9790793bdc4e40bc895f70e2a2"}, From 3c36bcfaa6d09436ba06699655934e4d01dba31b Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 1 Aug 2025 12:19:41 -0700 Subject: [PATCH 58/74] Remove deprecated "use Tesla" macro usage --- lib/pleroma/search/qdrant_search.ex | 55 +++++++++++++++++----- test/pleroma/search/qdrant_search_test.exs | 6 +-- 2 files changed, 45 insertions(+), 16 deletions(-) diff --git a/lib/pleroma/search/qdrant_search.ex b/lib/pleroma/search/qdrant_search.ex index b659bb682..5142a273f 100644 --- a/lib/pleroma/search/qdrant_search.ex +++ b/lib/pleroma/search/qdrant_search.ex @@ -157,26 +157,55 @@ defmodule Pleroma.Search.QdrantSearch do end defmodule Pleroma.Search.QdrantSearch.OpenAIClient do - use Tesla alias Pleroma.Config.Getting, as: Config - plug(Tesla.Middleware.BaseUrl, Config.get([Pleroma.Search.QdrantSearch, :openai_url])) - plug(Tesla.Middleware.JSON) + def post(path, body) do + Tesla.post(client(), path, body) + end - plug(Tesla.Middleware.Headers, [ - {"Authorization", - "Bearer #{Pleroma.Config.get([Pleroma.Search.QdrantSearch, :openai_api_key])}"} - ]) + defp client do + Tesla.client(middleware()) + end + + defp middleware do + [ + {Tesla.Middleware.BaseUrl, Config.get([Pleroma.Search.QdrantSearch, :openai_url])}, + Tesla.Middleware.JSON, + {Tesla.Middleware.Headers, + [ + {"Authorization", "Bearer #{Config.get([Pleroma.Search.QdrantSearch, :openai_api_key])}"} + ]} + ] + end end defmodule Pleroma.Search.QdrantSearch.QdrantClient do - use Tesla alias Pleroma.Config.Getting, as: Config - plug(Tesla.Middleware.BaseUrl, Config.get([Pleroma.Search.QdrantSearch, :qdrant_url])) - plug(Tesla.Middleware.JSON) + def delete(path) do + Tesla.delete(client(), path) + end - plug(Tesla.Middleware.Headers, [ - {"api-key", Pleroma.Config.get([Pleroma.Search.QdrantSearch, :qdrant_api_key])} - ]) + def post(path, body) do + Tesla.post(client(), path, body) + end + + def put(path, body) do + Tesla.put(client(), path, body) + end + + defp client do + Tesla.client(middleware()) + end + + defp middleware do + [ + {Tesla.Middleware.BaseUrl, Config.get([Pleroma.Search.QdrantSearch, :qdrant_url])}, + Tesla.Middleware.JSON, + {Tesla.Middleware.Headers, + [ + {"api-key", Pleroma.Config.get([Pleroma.Search.QdrantSearch, :qdrant_api_key])} + ]} + ] + end end diff --git a/test/pleroma/search/qdrant_search_test.exs b/test/pleroma/search/qdrant_search_test.exs index 47a77a391..daf8eeb69 100644 --- a/test/pleroma/search/qdrant_search_test.exs +++ b/test/pleroma/search/qdrant_search_test.exs @@ -51,7 +51,7 @@ defmodule Pleroma.Search.QdrantSearchTest do }) Config - |> expect(:get, 3, fn + |> expect(:get, 4, fn [Pleroma.Search, :module], nil -> QdrantSearch @@ -93,7 +93,7 @@ defmodule Pleroma.Search.QdrantSearchTest do }) Config - |> expect(:get, 3, fn + |> expect(:get, 4, fn [Pleroma.Search, :module], nil -> QdrantSearch @@ -158,7 +158,7 @@ defmodule Pleroma.Search.QdrantSearchTest do end) Config - |> expect(:get, 6, fn + |> expect(:get, 7, fn [Pleroma.Search, :module], nil -> QdrantSearch From 09eb7dbf8eb876b9847d7323fd2b93ee04c5be2a Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Fri, 1 Aug 2025 23:31:54 +0300 Subject: [PATCH 59/74] Change mailer example to use Mua --- changelog.d/smtp-docs.change | 1 + docs/configuration/cheatsheet.md | 8 +++----- 2 files changed, 4 insertions(+), 5 deletions(-) create mode 100644 changelog.d/smtp-docs.change diff --git a/changelog.d/smtp-docs.change b/changelog.d/smtp-docs.change new file mode 100644 index 000000000..fb9925e43 --- /dev/null +++ b/changelog.d/smtp-docs.change @@ -0,0 +1 @@ +Change SMTP example to use the Mua adapter that works with OTP>25 \ No newline at end of file diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index 6e2fddcb6..07414f69a 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -733,13 +733,11 @@ An example for SMTP adapter: ```elixir config :pleroma, Pleroma.Emails.Mailer, enabled: true, - adapter: Swoosh.Adapters.SMTP, + adapter: Swoosh.Adapters.Mua, relay: "smtp.gmail.com", - username: "YOUR_USERNAME@gmail.com", - password: "YOUR_SMTP_PASSWORD", + auth: [username: "YOUR_USERNAME@gmail.com", password: "YOUR_SMTP_PASSWORD"], port: 465, - ssl: true, - auth: :always + protocol: :ssl ``` An example for Mua adapter: From 44898845a69814aad9d09e8af82df0679a8d5f7f Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 1 Aug 2025 12:52:29 -0700 Subject: [PATCH 60/74] Update Plug/Cowboy/Gun --- mix.exs | 6 +++--- mix.lock | 14 +++++++------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/mix.exs b/mix.exs index a79aaca8f..a8b08cf67 100644 --- a/mix.exs +++ b/mix.exs @@ -135,7 +135,7 @@ defmodule Pleroma.Mixfile do {:telemetry_metrics, "~> 0.6"}, {:telemetry_poller, "~> 1.0"}, {:tzdata, "~> 1.0.3"}, - {:plug_cowboy, "~> 2.5"}, + {:plug_cowboy, "~> 2.7"}, {:oban, "~> 2.19.0"}, {:gettext, "~> 0.20"}, {:bcrypt_elixir, "~> 2.2"}, @@ -146,8 +146,8 @@ defmodule Pleroma.Mixfile do {:cachex, "~> 3.2"}, {:tesla, "~> 1.11"}, {:castore, "~> 1.0"}, - {:cowlib, "~> 2.9", override: true}, - {:gun, "~> 2.0.0-rc.1", override: true}, + {:cowlib, "~> 2.15"}, + {:gun, "~> 2.2"}, {:finch, "~> 0.15"}, {:jason, "~> 1.2"}, {:mogrify, "~> 0.9.0", override: "true"}, diff --git a/mix.lock b/mix.lock index a708087c9..93f41bba8 100644 --- a/mix.lock +++ b/mix.lock @@ -20,9 +20,9 @@ "connection": {:hex, :connection, "1.1.0", "ff2a49c4b75b6fb3e674bfc5536451607270aac754ffd1bdfe175abe4a6d7a68", [:mix], [], "hexpm", "722c1eb0a418fbe91ba7bd59a47e28008a189d47e37e0e7bb85585a016b2869c"}, "cors_plug": {:hex, :cors_plug, "2.0.3", "316f806d10316e6d10f09473f19052d20ba0a0ce2a1d910ddf57d663dac402ae", [:mix], [{:plug, "~> 1.8", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "ee4ae1418e6ce117fc42c2ba3e6cbdca4e95ecd2fe59a05ec6884ca16d469aea"}, "covertool": {:hex, :covertool, "2.0.6", "4a291b4e3449025b0595d8f44c8d7635d4f48f033be2ce88d22a329f36f94a91", [:rebar3], [], "hexpm", "5db3fcd82180d8ea4ad857d4d1ab21a8d31b5aee0d60d2f6c0f9e25a411d1e21"}, - "cowboy": {:hex, :cowboy, "2.12.0", "f276d521a1ff88b2b9b4c54d0e753da6c66dd7be6c9fca3d9418b561828a3731", [:make, :rebar3], [{:cowlib, "2.13.0", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "1.8.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "8a7abe6d183372ceb21caa2709bec928ab2b72e18a3911aa1771639bef82651e"}, + "cowboy": {:hex, :cowboy, "2.13.0", "09d770dd5f6a22cc60c071f432cd7cb87776164527f205c5a6b0f24ff6b38990", [:make, :rebar3], [{:cowlib, ">= 2.14.0 and < 3.0.0", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, ">= 1.8.0 and < 3.0.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "e724d3a70995025d654c1992c7b11dbfea95205c047d86ff9bf1cda92ddc5614"}, "cowboy_telemetry": {:hex, :cowboy_telemetry, "0.4.0", "f239f68b588efa7707abce16a84d0d2acf3a0f50571f8bb7f56a15865aae820c", [:rebar3], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "7d98bac1ee4565d31b62d59f8823dfd8356a169e7fcbb83831b8a5397404c9de"}, - "cowlib": {:hex, :cowlib, "2.13.0", "db8f7505d8332d98ef50a3ef34b34c1afddec7506e4ee4dd4a3a266285d282ca", [:make, :rebar3], [], "hexpm", "e1e1284dc3fc030a64b1ad0d8382ae7e99da46c3246b815318a4b848873800a4"}, + "cowlib": {:hex, :cowlib, "2.15.0", "3c97a318a933962d1c12b96ab7c1d728267d2c523c25a5b57b0f93392b6e9e25", [:make, :rebar3], [], "hexpm", "4f00c879a64b4fe7c8fcb42a4281925e9ffdb928820b03c3ad325a617e857532"}, "credo": {:hex, :credo, "1.7.12", "9e3c20463de4b5f3f23721527fcaf16722ec815e70ff6c60b86412c695d426c1", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "8493d45c656c5427d9c729235b99d498bd133421f3e0a683e5c1b561471291e5"}, "crontab": {:hex, :crontab, "1.1.8", "2ce0e74777dfcadb28a1debbea707e58b879e6aa0ffbf9c9bb540887bce43617", [:mix], [{:ecto, "~> 1.0 or ~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm"}, "custom_base": {:hex, :custom_base, "0.2.1", "4a832a42ea0552299d81652aa0b1f775d462175293e99dfbe4d7dbaab785a706", [:mix], [], "hexpm", "8df019facc5ec9603e94f7270f1ac73ddf339f56ade76a721eaa57c1493ba463"}, @@ -58,7 +58,7 @@ "floki": {:hex, :floki, "0.35.2", "87f8c75ed8654b9635b311774308b2760b47e9a579dabf2e4d5f1e1d42c39e0b", [:mix], [], "hexpm", "6b05289a8e9eac475f644f09c2e4ba7e19201fd002b89c28c1293e7bd16773d9"}, "gen_smtp": {:hex, :gen_smtp, "0.15.0", "9f51960c17769b26833b50df0b96123605a8024738b62db747fece14eb2fbfcc", [:rebar3], [], "hexpm", "29bd14a88030980849c7ed2447b8db6d6c9278a28b11a44cafe41b791205440f"}, "gettext": {:hex, :gettext, "0.24.0", "6f4d90ac5f3111673cbefc4ebee96fe5f37a114861ab8c7b7d5b30a1108ce6d8", [:mix], [{:expo, "~> 0.5.1", [hex: :expo, repo: "hexpm", optional: false]}], "hexpm", "bdf75cdfcbe9e4622dd18e034b227d77dd17f0f133853a1c73b97b3d6c770e8b"}, - "gun": {:hex, :gun, "2.0.1", "160a9a5394800fcba41bc7e6d421295cf9a7894c2252c0678244948e3336ad73", [:make, :rebar3], [{:cowlib, "2.12.1", [hex: :cowlib, repo: "hexpm", optional: false]}], "hexpm", "a10bc8d6096b9502205022334f719cc9a08d9adcfbfc0dbee9ef31b56274a20b"}, + "gun": {:hex, :gun, "2.2.0", "b8f6b7d417e277d4c2b0dc3c07dfdf892447b087f1cc1caff9c0f556b884e33d", [:make, :rebar3], [{:cowlib, ">= 2.15.0 and < 3.0.0", [hex: :cowlib, repo: "hexpm", optional: false]}], "hexpm", "76022700c64287feb4df93a1795cff6741b83fb37415c40c34c38d2a4645261a"}, "hackney": {:hex, :hackney, "1.18.2", "d7ff544ddae5e1cb49e9cf7fa4e356d7f41b283989a1c304bfc47a8cc1cf966f", [:rebar3], [{:certifi, "~>2.12.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~>6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~>1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.4.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~>1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~>0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "af94d5c9f97857db257090a4a10e5426ecb6f4918aa5cc666798566ae14b65fd"}, "hpax": {:hex, :hpax, "0.2.0", "5a58219adcb75977b2edce5eb22051de9362f08236220c9e859a47111c194ff5", [:mix], [], "hexpm", "bea06558cdae85bed075e6c036993d43cd54d447f76d8190a8db0dc5893fa2f1"}, "html_entities": {:hex, :html_entities, "0.5.2", "9e47e70598da7de2a9ff6af8758399251db6dbb7eebe2b013f2bbd2515895c3c", [:mix], [], "hexpm", "c53ba390403485615623b9531e97696f076ed415e8d8058b1dbaa28181f4fdcc"}, @@ -108,9 +108,9 @@ "phoenix_swoosh": {:hex, :phoenix_swoosh, "1.2.1", "b74ccaa8046fbc388a62134360ee7d9742d5a8ae74063f34eb050279de7a99e1", [:mix], [{:finch, "~> 0.8", [hex: :finch, repo: "hexpm", optional: true]}, {:hackney, "~> 1.10", [hex: :hackney, repo: "hexpm", optional: true]}, {:phoenix, "~> 1.6", [hex: :phoenix, repo: "hexpm", optional: true]}, {:phoenix_html, "~> 3.0 or ~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:phoenix_view, "~> 1.0 or ~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: false]}, {:swoosh, "~> 1.5", [hex: :swoosh, repo: "hexpm", optional: false]}], "hexpm", "4000eeba3f9d7d1a6bf56d2bd56733d5cadf41a7f0d8ffe5bb67e7d667e204a2"}, "phoenix_template": {:hex, :phoenix_template, "1.0.4", "e2092c132f3b5e5b2d49c96695342eb36d0ed514c5b252a77048d5969330d639", [:mix], [{:phoenix_html, "~> 2.14.2 or ~> 3.0 or ~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}], "hexpm", "2c0c81f0e5c6753faf5cca2f229c9709919aba34fab866d3bc05060c9c444206"}, "phoenix_view": {:hex, :phoenix_view, "2.0.4", "b45c9d9cf15b3a1af5fb555c674b525391b6a1fe975f040fb4d913397b31abf4", [:mix], [{:phoenix_html, "~> 2.14.2 or ~> 3.0 or ~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}], "hexpm", "4e992022ce14f31fe57335db27a28154afcc94e9983266835bb3040243eb620b"}, - "plug": {:hex, :plug, "1.16.1", "40c74619c12f82736d2214557dedec2e9762029b2438d6d175c5074c933edc9d", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "a13ff6b9006b03d7e33874945b2755253841b238c34071ed85b0e86057f8cddc"}, - "plug_cowboy": {:hex, :plug_cowboy, "2.7.1", "87677ffe3b765bc96a89be7960f81703223fe2e21efa42c125fcd0127dd9d6b2", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowboy_telemetry, "~> 0.3", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "02dbd5f9ab571b864ae39418db7811618506256f6d13b4a45037e5fe78dc5de3"}, - "plug_crypto": {:hex, :plug_crypto, "2.1.0", "f44309c2b06d249c27c8d3f65cfe08158ade08418cf540fd4f72d4d6863abb7b", [:mix], [], "hexpm", "131216a4b030b8f8ce0f26038bc4421ae60e4bb95c5cf5395e1421437824c4fa"}, + "plug": {:hex, :plug, "1.18.1", "5067f26f7745b7e31bc3368bc1a2b818b9779faa959b49c934c17730efc911cf", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "57a57db70df2b422b564437d2d33cf8d33cd16339c1edb190cd11b1a3a546cc2"}, + "plug_cowboy": {:hex, :plug_cowboy, "2.7.4", "729c752d17cf364e2b8da5bdb34fb5804f56251e88bb602aff48ae0bd8673d11", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowboy_telemetry, "~> 0.3", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "9b85632bd7012615bae0a5d70084deb1b25d2bcbb32cab82d1e9a1e023168aa3"}, + "plug_crypto": {:hex, :plug_crypto, "2.1.1", "19bda8184399cb24afa10be734f84a16ea0a2bc65054e23a62bb10f06bc89491", [:mix], [], "hexpm", "6470bce6ffe41c8bd497612ffde1a7e4af67f36a15eea5f921af71cf3e11247c"}, "plug_static_index_html": {:hex, :plug_static_index_html, "1.0.0", "840123d4d3975585133485ea86af73cb2600afd7f2a976f9f5fd8b3808e636a0", [:mix], [{:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "79fd4fcf34d110605c26560cbae8f23c603ec4158c08298bd4360fdea90bb5cf"}, "poison": {:hex, :poison, "3.1.0", "d9eb636610e096f86f25d9a46f35a9facac35609a7591b3be3326e99a0484665", [:mix], [], "hexpm", "fec8660eb7733ee4117b85f55799fd3833eb769a6df71ccf8903e8dc5447cfce"}, "poolboy": {:hex, :poolboy, "1.5.2", "392b007a1693a64540cead79830443abf5762f5d30cf50bc95cb2c1aaafa006b", [:rebar3], [], "hexpm", "dad79704ce5440f3d5a3681c8590b9dc25d1a561e8f5a9c995281012860901e3"}, @@ -124,7 +124,7 @@ "prometheus_phx": {:git, "https://git.pleroma.social/pleroma/elixir-libraries/prometheus-phx.git", "9cd8f248c9381ffedc799905050abce194a97514", [branch: "no-logging"]}, "prometheus_plugs": {:hex, :prometheus_plugs, "1.1.5", "25933d48f8af3a5941dd7b621c889749894d8a1082a6ff7c67cc99dec26377c5", [:mix], [{:accept, "~> 0.1", [hex: :accept, repo: "hexpm", optional: false]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.1 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}, {:prometheus_process_collector, "~> 1.1", [hex: :prometheus_process_collector, repo: "hexpm", optional: true]}], "hexpm", "0273a6483ccb936d79ca19b0ab629aef0dba958697c94782bb728b920dfc6a79"}, "quantile_estimator": {:hex, :quantile_estimator, "0.2.1", "ef50a361f11b5f26b5f16d0696e46a9e4661756492c981f7b2229ef42ff1cd15", [:rebar3], [], "hexpm", "282a8a323ca2a845c9e6f787d166348f776c1d4a41ede63046d72d422e3da946"}, - "ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"}, + "ranch": {:hex, :ranch, "2.2.0", "25528f82bc8d7c6152c57666ca99ec716510fe0925cb188172f41ce93117b1b0", [:make, :rebar3], [], "hexpm", "fa0b99a1780c80218a4197a59ea8d3bdae32fbff7e88527d7d8a4787eff4f8e7"}, "recon": {:hex, :recon, "2.5.4", "05dd52a119ee4059fa9daa1ab7ce81bc7a8161a2f12e9d42e9d551ffd2ba901c", [:mix, :rebar3], [], "hexpm", "e9ab01ac7fc8572e41eb59385efeb3fb0ff5bf02103816535bacaedf327d0263"}, "remote_ip": {:git, "https://git.pleroma.social/pleroma/remote_ip.git", "b647d0deecaa3acb140854fe4bda5b7e1dc6d1c8", [ref: "b647d0deecaa3acb140854fe4bda5b7e1dc6d1c8"]}, "rustler": {:hex, :rustler, "0.30.0", "cefc49922132b072853fa9b0ca4dc2ffcb452f68fb73b779042b02d545e097fb", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:toml, "~> 0.6", [hex: :toml, repo: "hexpm", optional: false]}], "hexpm", "9ef1abb6a7dda35c47cfc649e6a5a61663af6cf842a55814a554a84607dee389"}, From 7b8d6eca65c660ae3fb8e606649d927231cc1a53 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 1 Aug 2025 12:53:15 -0700 Subject: [PATCH 61/74] Remove deprecated "use Plug.Test" --- test/pleroma/web/plugs/cache_test.exs | 3 ++- test/pleroma/web/plugs/digest_plug_test.exs | 3 ++- test/pleroma/web/plugs/idempotency_plug_test.exs | 3 ++- test/pleroma/web/plugs/remote_ip_test.exs | 3 ++- test/pleroma/web/plugs/set_format_plug_test.exs | 3 ++- test/pleroma/web/plugs/set_locale_plug_test.exs | 2 +- 6 files changed, 11 insertions(+), 6 deletions(-) diff --git a/test/pleroma/web/plugs/cache_test.exs b/test/pleroma/web/plugs/cache_test.exs index 0c119528d..09065e94e 100644 --- a/test/pleroma/web/plugs/cache_test.exs +++ b/test/pleroma/web/plugs/cache_test.exs @@ -5,7 +5,8 @@ defmodule Pleroma.Web.Plugs.CacheTest do # Relies on Cachex, has to stay synchronous use Pleroma.DataCase - use Plug.Test + import Plug.Conn + import Plug.Test alias Pleroma.Web.Plugs.Cache diff --git a/test/pleroma/web/plugs/digest_plug_test.exs b/test/pleroma/web/plugs/digest_plug_test.exs index 19f8a6f49..21ccddd88 100644 --- a/test/pleroma/web/plugs/digest_plug_test.exs +++ b/test/pleroma/web/plugs/digest_plug_test.exs @@ -4,7 +4,8 @@ defmodule Pleroma.Web.Plugs.DigestPlugTest do use ExUnit.Case, async: true - use Plug.Test + import Plug.Conn + import Plug.Test test "digest algorithm is taken from digest header" do body = "{\"hello\": \"world\"}" diff --git a/test/pleroma/web/plugs/idempotency_plug_test.exs b/test/pleroma/web/plugs/idempotency_plug_test.exs index cc55d341f..3b0131596 100644 --- a/test/pleroma/web/plugs/idempotency_plug_test.exs +++ b/test/pleroma/web/plugs/idempotency_plug_test.exs @@ -5,7 +5,8 @@ defmodule Pleroma.Web.Plugs.IdempotencyPlugTest do # Relies on Cachex, has to stay synchronous use Pleroma.DataCase - use Plug.Test + import Plug.Conn + import Plug.Test alias Pleroma.Web.Plugs.IdempotencyPlug alias Plug.Conn diff --git a/test/pleroma/web/plugs/remote_ip_test.exs b/test/pleroma/web/plugs/remote_ip_test.exs index aea0940f4..37b751370 100644 --- a/test/pleroma/web/plugs/remote_ip_test.exs +++ b/test/pleroma/web/plugs/remote_ip_test.exs @@ -4,7 +4,8 @@ defmodule Pleroma.Web.Plugs.RemoteIpTest do use ExUnit.Case - use Plug.Test + import Plug.Conn + import Plug.Test alias Pleroma.Web.Plugs.RemoteIp diff --git a/test/pleroma/web/plugs/set_format_plug_test.exs b/test/pleroma/web/plugs/set_format_plug_test.exs index 4d64fdde6..08a2b02a2 100644 --- a/test/pleroma/web/plugs/set_format_plug_test.exs +++ b/test/pleroma/web/plugs/set_format_plug_test.exs @@ -4,7 +4,8 @@ defmodule Pleroma.Web.Plugs.SetFormatPlugTest do use ExUnit.Case, async: true - use Plug.Test + import Plug.Conn + import Plug.Test alias Pleroma.Web.Plugs.SetFormatPlug diff --git a/test/pleroma/web/plugs/set_locale_plug_test.exs b/test/pleroma/web/plugs/set_locale_plug_test.exs index 4f664f84e..a7691ea11 100644 --- a/test/pleroma/web/plugs/set_locale_plug_test.exs +++ b/test/pleroma/web/plugs/set_locale_plug_test.exs @@ -4,7 +4,7 @@ defmodule Pleroma.Web.Plugs.SetLocalePlugTest do use ExUnit.Case, async: true - use Plug.Test + import Plug.Test alias Pleroma.Web.Plugs.SetLocalePlug alias Plug.Conn From d67ab670b0707951f905076a823e3d2a4d31749a Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 1 Aug 2025 13:13:01 -0700 Subject: [PATCH 62/74] Fix Gopher server to use modern :ranch --- lib/pleroma/gopher/server.ex | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/lib/pleroma/gopher/server.ex b/lib/pleroma/gopher/server.ex index 54245c9fa..add3ba925 100644 --- a/lib/pleroma/gopher/server.ex +++ b/lib/pleroma/gopher/server.ex @@ -22,14 +22,18 @@ defmodule Pleroma.Gopher.Server do def init([ip, port]) do Logger.info("Starting gopher server on #{port}") - :ranch.start_listener( - :gopher, - 100, - :ranch_tcp, - [ip: ip, port: port], - __MODULE__.ProtocolHandler, - [] - ) + {:ok, _pid} = + :ranch.start_listener( + :gopher, + :ranch_tcp, + %{ + num_acceptors: 100, + max_connections: 100, + socket_opts: [ip: ip, port: port] + }, + __MODULE__.ProtocolHandler, + [] + ) {:ok, %{ip: ip, port: port}} end @@ -43,13 +47,13 @@ defmodule Pleroma.Gopher.Server.ProtocolHandler do alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Visibility - def start_link(ref, socket, transport, opts) do - pid = spawn_link(__MODULE__, :init, [ref, socket, transport, opts]) + def start_link(ref, transport, opts) do + pid = spawn_link(__MODULE__, :init, [ref, transport, opts]) {:ok, pid} end - def init(ref, socket, transport, [] = _Opts) do - :ok = :ranch.accept_ack(ref) + def init(ref, transport, opts \\ []) do + {:ok, socket} = :ranch.handshake(ref, opts) loop(socket, transport) end From 9195cfb2bc5f3e5db0f2ff98bee0473ee00dbaca Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 1 Aug 2025 16:23:20 -0700 Subject: [PATCH 63/74] Document Gun, Cowboy, and Plug update --- changelog.d/gun.change | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/gun.change diff --git a/changelog.d/gun.change b/changelog.d/gun.change new file mode 100644 index 000000000..3d72b7701 --- /dev/null +++ b/changelog.d/gun.change @@ -0,0 +1 @@ +Update Cowboy, Gun, and Plug family of dependencies From c1836c98214896469381fe8ef11abb33669452b0 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 2 Aug 2025 09:53:56 -0700 Subject: [PATCH 64/74] Fix test that relied on previous fake hashtag behavior This test is normally skipped on MacOS due to weird unicode behavior --- .../mastodon_api/controllers/search_controller_test.exs | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/test/pleroma/web/mastodon_api/controllers/search_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/search_controller_test.exs index 8b4c6add2..f0c9c1901 100644 --- a/test/pleroma/web/mastodon_api/controllers/search_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/search_controller_test.exs @@ -7,7 +7,6 @@ defmodule Pleroma.Web.MastodonAPI.SearchControllerTest do alias Pleroma.Object alias Pleroma.Web.CommonAPI - alias Pleroma.Web.Endpoint import Pleroma.Factory import ExUnit.CaptureLog import Tesla.Mock @@ -66,9 +65,7 @@ defmodule Pleroma.Web.MastodonAPI.SearchControllerTest do [account | _] = results["accounts"] assert account["id"] == to_string(user_three.id) - assert results["hashtags"] == [ - %{"name" => "private", "url" => "#{Endpoint.url()}/tag/private"} - ] + assert results["hashtags"] == [] [status] = results["statuses"] assert status["id"] == to_string(activity.id) @@ -77,9 +74,7 @@ defmodule Pleroma.Web.MastodonAPI.SearchControllerTest do get(conn, "/api/v2/search?q=天子") |> json_response_and_validate_schema(200) - assert results["hashtags"] == [ - %{"name" => "天子", "url" => "#{Endpoint.url()}/tag/天子"} - ] + assert results["hashtags"] == [] [status] = results["statuses"] assert status["id"] == to_string(activity.id) From 321bd75dca71e395544c05de3583261a2793c7af Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Tue, 14 Jan 2025 02:02:46 +0300 Subject: [PATCH 65/74] Add a way to upload emoji pack from zip/url easily Essentially the same as the mix task --- changelog.d/emoji-pack-upload-zip.add | 1 + lib/pleroma/emoji/pack.ex | 61 +++++++++++++++++++ .../pleroma_emoji_pack_operation.ex | 33 ++++++++++ .../controllers/emoji_pack_controller.ex | 30 +++++++++ lib/pleroma/web/router.ex | 1 + 5 files changed, 126 insertions(+) create mode 100644 changelog.d/emoji-pack-upload-zip.add diff --git a/changelog.d/emoji-pack-upload-zip.add b/changelog.d/emoji-pack-upload-zip.add new file mode 100644 index 000000000..3f1973269 --- /dev/null +++ b/changelog.d/emoji-pack-upload-zip.add @@ -0,0 +1 @@ +Added a way to upload new packs from a URL or ZIP file via Admin API \ No newline at end of file diff --git a/lib/pleroma/emoji/pack.ex b/lib/pleroma/emoji/pack.ex index 99fa1994f..3c6603b5f 100644 --- a/lib/pleroma/emoji/pack.ex +++ b/lib/pleroma/emoji/pack.ex @@ -225,6 +225,67 @@ defmodule Pleroma.Emoji.Pack do end end + def download_zip(name, opts \\ %{}) do + pack_path = + Path.join([ + Pleroma.Config.get!([:instance, :static_dir]), + "emoji", + name + ]) + + with {_, false} <- + {"Pack already exists, refusing to import #{name}", File.exists?(pack_path)}, + {_, :ok} <- {"Could not create the pack directory", File.mkdir_p(pack_path)}, + {_, {:ok, %{body: binary_archive}}} <- + (case opts do + %{url: url} -> + {"Could not download pack", Pleroma.HTTP.get(url)} + + %{file: file} -> + case File.read(file.path) do + {:ok, data} -> {nil, {:ok, %{body: data}}} + {:error, _e} -> {"Could not read the uploaded pack file", :error} + end + + _ -> + {"Neither file nor URL was present in the request", :error} + end), + {_, {:ok, _}} <- + {"Could not unzip pack", + :zip.unzip(binary_archive, cwd: String.to_charlist(pack_path))} do + # Get the pack SHA + archive_sha = :crypto.hash(:sha256, binary_archive) |> Base.encode16() + + pack_json_path = Path.join([pack_path, "pack.json"]) + # Make a json if it does not exist + if not File.exists?(pack_json_path) do + # Make a list of the emojis + emoji_map = + Pleroma.Emoji.Loader.make_shortcode_to_file_map( + pack_path, + Map.get(opts, :exts, [".png", ".gif", ".jpg"]) + ) + + pack_json = %{ + pack: %{ + license: Map.get(opts, :license, ""), + homepage: Map.get(opts, :homepage, ""), + description: Map.get(opts, :description, ""), + src: Map.get(opts, :url), + src_sha256: archive_sha + }, + files: emoji_map + } + + File.write!(pack_json_path, Jason.encode!(pack_json, pretty: true)) + end + + :ok + else + {err, _} -> {:error, err} + end + end + @spec download(String.t(), String.t(), String.t()) :: {:ok, t()} | {:error, atom()} def download(name, url, as) do uri = url |> String.trim() |> URI.parse() diff --git a/lib/pleroma/web/api_spec/operations/pleroma_emoji_pack_operation.ex b/lib/pleroma/web/api_spec/operations/pleroma_emoji_pack_operation.ex index efa36ffdc..dd503997a 100644 --- a/lib/pleroma/web/api_spec/operations/pleroma_emoji_pack_operation.ex +++ b/lib/pleroma/web/api_spec/operations/pleroma_emoji_pack_operation.ex @@ -127,6 +127,20 @@ defmodule Pleroma.Web.ApiSpec.PleromaEmojiPackOperation do } end + def download_zip_operation do + %Operation{ + tags: ["Emoji pack administration"], + summary: "Download a pack from a URL or an uploaded file", + operationId: "PleromaAPI.EmojiPackController.download_zip", + security: [%{"oAuth" => ["admin:write"]}], + requestBody: request_body("Parameters", download_zip_request(), required: true), + responses: %{ + 200 => ok_response(), + 400 => Operation.response("Bad Request", "application/json", ApiError) + } + } + end + defp download_request do %Schema{ type: :object, @@ -143,6 +157,25 @@ defmodule Pleroma.Web.ApiSpec.PleromaEmojiPackOperation do } end + defp download_zip_request do + %Schema{ + type: :object, + required: [:name], + properties: %{ + url: %Schema{ + type: :string, + format: :uri, + description: "URL of the file" + }, + file: %Schema{ + description: "The uploaded ZIP file", + type: :object + }, + name: %Schema{type: :string, format: :uri, description: "Pack Name"} + } + } + end + def create_operation do %Operation{ tags: ["Emoji pack administration"], diff --git a/lib/pleroma/web/pleroma_api/controllers/emoji_pack_controller.ex b/lib/pleroma/web/pleroma_api/controllers/emoji_pack_controller.ex index 32360d2a2..cc4493cdf 100644 --- a/lib/pleroma/web/pleroma_api/controllers/emoji_pack_controller.ex +++ b/lib/pleroma/web/pleroma_api/controllers/emoji_pack_controller.ex @@ -16,6 +16,7 @@ defmodule Pleroma.Web.PleromaAPI.EmojiPackController do :import_from_filesystem, :remote, :download, + :download_zip, :create, :update, :delete @@ -113,6 +114,35 @@ defmodule Pleroma.Web.PleromaAPI.EmojiPackController do end end + def download_zip( + %{private: %{open_api_spex: %{body_params: %{url: url, name: name}}}} = conn, + _ + ) do + with :ok <- Pack.download_zip(name, %{url: url}) do + json(conn, "ok") + else + {:error, error} -> + conn + |> put_status(:bad_request) + |> json(%{error: error}) + end + end + + def download_zip( + %{private: %{open_api_spex: %{body_params: %{file: %Plug.Upload{} = file, name: name}}}} = + conn, + _ + ) do + with :ok <- Pack.download_zip(name, %{file: file}) do + json(conn, "ok") + else + {:error, error} -> + conn + |> put_status(:bad_request) + |> json(%{error: error}) + end + end + def download( %{private: %{open_api_spex: %{body_params: %{url: url, name: name} = params}}} = conn, _ diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index dfab1b216..cd9cfd3ed 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -466,6 +466,7 @@ defmodule Pleroma.Web.Router do get("/import", EmojiPackController, :import_from_filesystem) get("/remote", EmojiPackController, :remote) post("/download", EmojiPackController, :download) + post("/download_zip", EmojiPackController, :download_zip) post("/files", EmojiFileController, :create) patch("/files", EmojiFileController, :update) From 26ac875bc8f1853cb2718c57292fbd336584359e Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Wed, 6 Aug 2025 22:50:44 +0300 Subject: [PATCH 66/74] Use path_join_name_safe for pathname joining --- lib/pleroma/emoji/pack.ex | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/emoji/pack.ex b/lib/pleroma/emoji/pack.ex index 3c6603b5f..9b50fb74c 100644 --- a/lib/pleroma/emoji/pack.ex +++ b/lib/pleroma/emoji/pack.ex @@ -227,11 +227,10 @@ defmodule Pleroma.Emoji.Pack do def download_zip(name, opts \\ %{}) do pack_path = - Path.join([ - Pleroma.Config.get!([:instance, :static_dir]), - "emoji", + path_join_name_safe( + Path.join(Pleroma.Config.get!([:instance, :static_dir]), "emoji"), name - ]) + ) with {_, false} <- {"Pack already exists, refusing to import #{name}", File.exists?(pack_path)}, From 8d0b29d7183f11c05f695d0c3cf4b4ec1d2d2d67 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Thu, 7 Aug 2025 11:22:51 +0300 Subject: [PATCH 67/74] Only calculate SHA when there's no pack json --- lib/pleroma/emoji/pack.ex | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/emoji/pack.ex b/lib/pleroma/emoji/pack.ex index 9b50fb74c..561bc69d8 100644 --- a/lib/pleroma/emoji/pack.ex +++ b/lib/pleroma/emoji/pack.ex @@ -252,9 +252,6 @@ defmodule Pleroma.Emoji.Pack do {_, {:ok, _}} <- {"Could not unzip pack", :zip.unzip(binary_archive, cwd: String.to_charlist(pack_path))} do - # Get the pack SHA - archive_sha = :crypto.hash(:sha256, binary_archive) |> Base.encode16() - pack_json_path = Path.join([pack_path, "pack.json"]) # Make a json if it does not exist if not File.exists?(pack_json_path) do @@ -265,6 +262,9 @@ defmodule Pleroma.Emoji.Pack do Map.get(opts, :exts, [".png", ".gif", ".jpg"]) ) + # Calculate the pack SHA. Only needed when there's no pack.json, as it would already include a hash + archive_sha = :crypto.hash(:sha256, binary_archive) |> Base.encode16() + pack_json = %{ pack: %{ license: Map.get(opts, :license, ""), From 897c1ced5f3f5a17ee80f34c0e7d8b378237c3e1 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Thu, 7 Aug 2025 13:47:54 +0400 Subject: [PATCH 68/74] EmojiPackControllerDownloadZipTest: Add test. --- ...moji_pack_controller_download_zip_test.exs | 311 ++++++++++++++++++ 1 file changed, 311 insertions(+) create mode 100644 test/pleroma/web/pleroma_api/controllers/emoji_pack_controller_download_zip_test.exs diff --git a/test/pleroma/web/pleroma_api/controllers/emoji_pack_controller_download_zip_test.exs b/test/pleroma/web/pleroma_api/controllers/emoji_pack_controller_download_zip_test.exs new file mode 100644 index 000000000..ba72c8e27 --- /dev/null +++ b/test/pleroma/web/pleroma_api/controllers/emoji_pack_controller_download_zip_test.exs @@ -0,0 +1,311 @@ +# Pleroma: A lightweight social networking server +# Copyright © Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.PleromaAPI.EmojiPackControllerDownloadZipTest do + use Pleroma.Web.ConnCase, async: false + + import Tesla.Mock + import Pleroma.Factory + + @emoji_path Path.join( + Pleroma.Config.get!([:instance, :static_dir]), + "emoji" + ) + + setup do + admin = insert(:user, is_admin: true) + token = insert(:oauth_admin_token, user: admin) + + admin_conn = + build_conn() + |> assign(:user, admin) + |> assign(:token, token) + + Pleroma.Emoji.reload() + + # Clean up any test packs from previous runs + on_exit(fn -> + test_packs = [ + "test_zip_pack", + "test_zip_pack_url", + "test_zip_pack_malicious", + "test_invalid_pack", + "test_bad_url_pack", + "test_no_source_pack" + ] + + Enum.each(test_packs, fn pack_name -> + pack_path = Path.join(@emoji_path, pack_name) + + if File.exists?(pack_path) do + File.rm_rf!(pack_path) + end + end) + end) + + {:ok, %{admin_conn: admin_conn}} + end + + describe "POST /api/pleroma/emoji/packs/download_zip" do + setup do + clear_config([:instance, :admin_privileges], [:emoji_manage_emoji]) + end + + test "creates pack from uploaded ZIP file", %{admin_conn: admin_conn} do + # Create a test ZIP file with emojis + {:ok, zip_path} = create_test_emoji_zip() + + upload = %Plug.Upload{ + content_type: "application/zip", + path: zip_path, + filename: "test_pack.zip" + } + + assert admin_conn + |> put_req_header("content-type", "multipart/form-data") + |> post("/api/pleroma/emoji/packs/download_zip", %{ + name: "test_zip_pack", + file: upload + }) + |> json_response_and_validate_schema(200) == "ok" + + # Verify pack was created + assert File.exists?("#{@emoji_path}/test_zip_pack/pack.json") + assert File.exists?("#{@emoji_path}/test_zip_pack/test_emoji.png") + + # Verify pack.json contents + {:ok, pack_json} = File.read("#{@emoji_path}/test_zip_pack/pack.json") + pack_data = Jason.decode!(pack_json) + + assert pack_data["files"]["test_emoji"] == "test_emoji.png" + assert pack_data["pack"]["src_sha256"] != nil + + # Clean up + File.rm!(zip_path) + end + + test "creates pack from URL", %{admin_conn: admin_conn} do + # Mock HTTP request to download ZIP + {:ok, zip_path} = create_test_emoji_zip() + {:ok, zip_data} = File.read(zip_path) + + mock(fn + %{method: :get, url: "https://example.com/emoji_pack.zip"} -> + %Tesla.Env{status: 200, body: zip_data} + end) + + assert admin_conn + |> put_req_header("content-type", "multipart/form-data") + |> post("/api/pleroma/emoji/packs/download_zip", %{ + name: "test_zip_pack_url", + url: "https://example.com/emoji_pack.zip" + }) + |> json_response_and_validate_schema(200) == "ok" + + # Verify pack was created + assert File.exists?("#{@emoji_path}/test_zip_pack_url/pack.json") + assert File.exists?("#{@emoji_path}/test_zip_pack_url/test_emoji.png") + + # Verify pack.json has URL as source + {:ok, pack_json} = File.read("#{@emoji_path}/test_zip_pack_url/pack.json") + pack_data = Jason.decode!(pack_json) + + assert pack_data["pack"]["src"] == "https://example.com/emoji_pack.zip" + assert pack_data["pack"]["src_sha256"] != nil + + # Clean up + File.rm!(zip_path) + end + + test "refuses to overwrite existing pack", %{admin_conn: admin_conn} do + # Create existing pack + pack_path = Path.join(@emoji_path, "test_zip_pack") + File.mkdir_p!(pack_path) + File.write!(Path.join(pack_path, "pack.json"), Jason.encode!(%{files: %{}})) + + {:ok, zip_path} = create_test_emoji_zip() + + upload = %Plug.Upload{ + content_type: "application/zip", + path: zip_path, + filename: "test_pack.zip" + } + + assert admin_conn + |> put_req_header("content-type", "multipart/form-data") + |> post("/api/pleroma/emoji/packs/download_zip", %{ + name: "test_zip_pack", + file: upload + }) + |> json_response_and_validate_schema(400) == %{ + "error" => "Pack already exists, refusing to import test_zip_pack" + } + + # Clean up + File.rm!(zip_path) + end + + test "handles invalid ZIP file", %{admin_conn: admin_conn} do + # Create invalid ZIP file + invalid_zip_path = Path.join(System.tmp_dir!(), "invalid.zip") + File.write!(invalid_zip_path, "not a zip file") + + upload = %Plug.Upload{ + content_type: "application/zip", + path: invalid_zip_path, + filename: "invalid.zip" + } + + assert admin_conn + |> put_req_header("content-type", "multipart/form-data") + |> post("/api/pleroma/emoji/packs/download_zip", %{ + name: "test_invalid_pack", + file: upload + }) + |> json_response_and_validate_schema(400) == %{ + "error" => "Could not unzip pack" + } + + # Clean up + File.rm!(invalid_zip_path) + end + + test "handles URL download failure", %{admin_conn: admin_conn} do + mock(fn + %{method: :get, url: "https://example.com/bad_pack.zip"} -> + %Tesla.Env{status: 404, body: "Not found"} + end) + + assert admin_conn + |> put_req_header("content-type", "multipart/form-data") + |> post("/api/pleroma/emoji/packs/download_zip", %{ + name: "test_bad_url_pack", + url: "https://example.com/bad_pack.zip" + }) + |> json_response_and_validate_schema(400) == %{ + "error" => "Could not download pack" + } + end + + test "requires either file or URL parameter", %{admin_conn: admin_conn} do + assert admin_conn + |> put_req_header("content-type", "multipart/form-data") + |> post("/api/pleroma/emoji/packs/download_zip", %{ + name: "test_no_source_pack" + }) + |> json_response_and_validate_schema(400) == %{ + "error" => "Neither file nor URL was present in the request" + } + end + + test "preserves existing pack.json if present in ZIP", %{admin_conn: admin_conn} do + # Create ZIP with pack.json + {:ok, zip_path} = create_test_emoji_zip_with_pack_json() + + upload = %Plug.Upload{ + content_type: "application/zip", + path: zip_path, + filename: "test_pack_with_json.zip" + } + + assert admin_conn + |> put_req_header("content-type", "multipart/form-data") + |> post("/api/pleroma/emoji/packs/download_zip", %{ + name: "test_zip_pack", + file: upload + }) + |> json_response_and_validate_schema(200) == "ok" + + # Verify original pack.json was preserved + {:ok, pack_json} = File.read("#{@emoji_path}/test_zip_pack/pack.json") + pack_data = Jason.decode!(pack_json) + + assert pack_data["pack"]["description"] == "Test pack from ZIP" + assert pack_data["pack"]["license"] == "Test License" + + # Clean up + File.rm!(zip_path) + end + + test "rejects malicious pack names", %{admin_conn: admin_conn} do + {:ok, zip_path} = create_test_emoji_zip() + + upload = %Plug.Upload{ + content_type: "application/zip", + path: zip_path, + filename: "test_pack.zip" + } + + # Test path traversal attempts + malicious_names = ["../evil", "../../evil", ".", "..", "evil/../../../etc"] + + Enum.each(malicious_names, fn name -> + assert_raise RuntimeError, ~r/Invalid or malicious pack name/, fn -> + admin_conn + |> put_req_header("content-type", "multipart/form-data") + |> post("/api/pleroma/emoji/packs/download_zip", %{ + name: name, + file: upload + }) + end + end) + + # Clean up + File.rm!(zip_path) + end + end + + defp create_test_emoji_zip do + tmp_dir = System.tmp_dir!() + zip_path = Path.join(tmp_dir, "test_emoji_pack_#{:rand.uniform(10000)}.zip") + + # 1x1 pixel PNG + png_data = + Base.decode64!( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==" + ) + + files = [ + {~c"test_emoji.png", png_data}, + # Will be treated as GIF based on extension + {~c"another_emoji.gif", png_data} + ] + + {:ok, {_name, zip_binary}} = :zip.zip(~c"test_pack.zip", files, [:memory]) + File.write!(zip_path, zip_binary) + + {:ok, zip_path} + end + + defp create_test_emoji_zip_with_pack_json do + tmp_dir = System.tmp_dir!() + zip_path = Path.join(tmp_dir, "test_emoji_pack_json_#{:rand.uniform(10000)}.zip") + + png_data = + Base.decode64!( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==" + ) + + pack_json = + Jason.encode!(%{ + pack: %{ + description: "Test pack from ZIP", + license: "Test License" + }, + files: %{ + "test_emoji" => "test_emoji.png" + } + }) + + files = [ + {~c"test_emoji.png", png_data}, + {~c"pack.json", pack_json} + ] + + {:ok, {_name, zip_binary}} = :zip.zip(~c"test_pack.zip", files, [:memory]) + File.write!(zip_path, zip_binary) + + {:ok, zip_path} + end +end From b249340fce23b1a4b30aa66688194b1eabfcefc7 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Thu, 7 Aug 2025 13:51:19 +0400 Subject: [PATCH 69/74] Emoji.Pack: Refactor and use safe_unzip. --- lib/pleroma/emoji/pack.ex | 129 ++++++++++++++++++++++---------------- 1 file changed, 76 insertions(+), 53 deletions(-) diff --git a/lib/pleroma/emoji/pack.ex b/lib/pleroma/emoji/pack.ex index 561bc69d8..1a4625db6 100644 --- a/lib/pleroma/emoji/pack.ex +++ b/lib/pleroma/emoji/pack.ex @@ -226,63 +226,86 @@ defmodule Pleroma.Emoji.Pack do end def download_zip(name, opts \\ %{}) do - pack_path = - path_join_name_safe( - Path.join(Pleroma.Config.get!([:instance, :static_dir]), "emoji"), - name + with :ok <- validate_not_empty([name]), + :ok <- validate_new_pack(name), + {:ok, archive_data} <- fetch_archive_data(opts), + pack_path <- path_join_name_safe(emoji_path(), name), + :ok <- File.mkdir_p(pack_path), + :ok <- safe_unzip(archive_data, pack_path) do + ensure_pack_json(pack_path, archive_data, opts) + else + {:error, reason} when is_binary(reason) -> {:error, reason} + _ -> {:error, "Could not process pack"} + end + end + + defp safe_unzip(archive_data, pack_path) do + case SafeZip.unzip_data(archive_data, pack_path) do + {:ok, _} -> :ok + {:error, reason} when is_binary(reason) -> {:error, reason} + _ -> {:error, "Could not unzip pack"} + end + end + + defp validate_new_pack(name) do + pack_path = path_join_name_safe(emoji_path(), name) + + if File.exists?(pack_path) do + {:error, "Pack already exists, refusing to import #{name}"} + else + :ok + end + end + + defp fetch_archive_data(%{url: url}) do + case Pleroma.HTTP.get(url) do + {:ok, %{status: 200, body: data}} -> {:ok, data} + _ -> {:error, "Could not download pack"} + end + end + + defp fetch_archive_data(%{file: %Plug.Upload{path: path}}) do + case File.read(path) do + {:ok, data} -> {:ok, data} + _ -> {:error, "Could not read the uploaded pack file"} + end + end + + defp fetch_archive_data(_) do + {:error, "Neither file nor URL was present in the request"} + end + + defp ensure_pack_json(pack_path, archive_data, opts) do + pack_json_path = Path.join(pack_path, "pack.json") + + if not File.exists?(pack_json_path) do + create_pack_json(pack_path, pack_json_path, archive_data, opts) + end + + :ok + end + + defp create_pack_json(pack_path, pack_json_path, archive_data, opts) do + emoji_map = + Pleroma.Emoji.Loader.make_shortcode_to_file_map( + pack_path, + Map.get(opts, :exts, [".png", ".gif", ".jpg"]) ) - with {_, false} <- - {"Pack already exists, refusing to import #{name}", File.exists?(pack_path)}, - {_, :ok} <- {"Could not create the pack directory", File.mkdir_p(pack_path)}, - {_, {:ok, %{body: binary_archive}}} <- - (case opts do - %{url: url} -> - {"Could not download pack", Pleroma.HTTP.get(url)} + archive_sha = :crypto.hash(:sha256, archive_data) |> Base.encode16() - %{file: file} -> - case File.read(file.path) do - {:ok, data} -> {nil, {:ok, %{body: data}}} - {:error, _e} -> {"Could not read the uploaded pack file", :error} - end + pack_json = %{ + pack: %{ + license: Map.get(opts, :license, ""), + homepage: Map.get(opts, :homepage, ""), + description: Map.get(opts, :description, ""), + src: Map.get(opts, :url), + src_sha256: archive_sha + }, + files: emoji_map + } - _ -> - {"Neither file nor URL was present in the request", :error} - end), - {_, {:ok, _}} <- - {"Could not unzip pack", - :zip.unzip(binary_archive, cwd: String.to_charlist(pack_path))} do - pack_json_path = Path.join([pack_path, "pack.json"]) - # Make a json if it does not exist - if not File.exists?(pack_json_path) do - # Make a list of the emojis - emoji_map = - Pleroma.Emoji.Loader.make_shortcode_to_file_map( - pack_path, - Map.get(opts, :exts, [".png", ".gif", ".jpg"]) - ) - - # Calculate the pack SHA. Only needed when there's no pack.json, as it would already include a hash - archive_sha = :crypto.hash(:sha256, binary_archive) |> Base.encode16() - - pack_json = %{ - pack: %{ - license: Map.get(opts, :license, ""), - homepage: Map.get(opts, :homepage, ""), - description: Map.get(opts, :description, ""), - src: Map.get(opts, :url), - src_sha256: archive_sha - }, - files: emoji_map - } - - File.write!(pack_json_path, Jason.encode!(pack_json, pretty: true)) - end - - :ok - else - {err, _} -> {:error, err} - end + File.write!(pack_json_path, Jason.encode!(pack_json, pretty: true)) end @spec download(String.t(), String.t(), String.t()) :: {:ok, t()} | {:error, atom()} From f203e7bb4275c1ff1ddf844e4a7eb343e4be2947 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Thu, 7 Aug 2025 13:51:33 +0400 Subject: [PATCH 70/74] EmojiPackController: Refactor. --- .../controllers/emoji_pack_controller.ex | 26 +++++++------------ 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/lib/pleroma/web/pleroma_api/controllers/emoji_pack_controller.ex b/lib/pleroma/web/pleroma_api/controllers/emoji_pack_controller.ex index cc4493cdf..8c5e4c06a 100644 --- a/lib/pleroma/web/pleroma_api/controllers/emoji_pack_controller.ex +++ b/lib/pleroma/web/pleroma_api/controllers/emoji_pack_controller.ex @@ -115,31 +115,23 @@ defmodule Pleroma.Web.PleromaAPI.EmojiPackController do end def download_zip( - %{private: %{open_api_spex: %{body_params: %{url: url, name: name}}}} = conn, + %{private: %{open_api_spex: %{body_params: params}}} = conn, _ ) do - with :ok <- Pack.download_zip(name, %{url: url}) do - json(conn, "ok") - else - {:error, error} -> - conn - |> put_status(:bad_request) - |> json(%{error: error}) - end - end + name = Map.get(params, :name) - def download_zip( - %{private: %{open_api_spex: %{body_params: %{file: %Plug.Upload{} = file, name: name}}}} = - conn, - _ - ) do - with :ok <- Pack.download_zip(name, %{file: file}) do + with :ok <- Pack.download_zip(name, params) do json(conn, "ok") else - {:error, error} -> + {:error, error} when is_binary(error) -> conn |> put_status(:bad_request) |> json(%{error: error}) + + {:error, _} -> + conn + |> put_status(:bad_request) + |> json(%{error: "Could not process pack"}) end end From 4eeb9c1f2d2e53228db25c83de2eb1837585c56c Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Fri, 8 Aug 2025 15:43:58 +0400 Subject: [PATCH 71/74] EmojiPackControllerDownloadZipTest: Add tests for empty pack name and failing creation. --- ...moji_pack_controller_download_zip_test.exs | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/test/pleroma/web/pleroma_api/controllers/emoji_pack_controller_download_zip_test.exs b/test/pleroma/web/pleroma_api/controllers/emoji_pack_controller_download_zip_test.exs index ba72c8e27..50f6446dc 100644 --- a/test/pleroma/web/pleroma_api/controllers/emoji_pack_controller_download_zip_test.exs +++ b/test/pleroma/web/pleroma_api/controllers/emoji_pack_controller_download_zip_test.exs @@ -199,6 +199,67 @@ defmodule Pleroma.Web.PleromaAPI.EmojiPackControllerDownloadZipTest do } end + test "returns error when pack name is empty", %{admin_conn: admin_conn} do + {:ok, zip_path} = create_test_emoji_zip() + + upload = %Plug.Upload{ + content_type: "application/zip", + path: zip_path, + filename: "test_pack.zip" + } + + assert admin_conn + |> put_req_header("content-type", "multipart/form-data") + |> post("/api/pleroma/emoji/packs/download_zip", %{ + name: "", + file: upload + }) + |> json_response_and_validate_schema(400) == %{ + "error" => "Pack name cannot be empty" + } + + # Clean up + File.rm!(zip_path) + end + + test "returns error when unable to create pack directory", %{admin_conn: admin_conn} do + # Make the emoji directory read-only to trigger mkdir_p failure + emoji_path = + Path.join( + Pleroma.Config.get!([:instance, :static_dir]), + "emoji" + ) + + # Save original permissions + {:ok, %{mode: original_mode}} = File.stat(emoji_path) + + # Make emoji directory read-only (no write permission) + File.chmod!(emoji_path, 0o555) + + {:ok, zip_path} = create_test_emoji_zip() + + upload = %Plug.Upload{ + content_type: "application/zip", + path: zip_path, + filename: "test_pack.zip" + } + + # Try to create a pack in the read-only emoji directory + assert admin_conn + |> put_req_header("content-type", "multipart/form-data") + |> post("/api/pleroma/emoji/packs/download_zip", %{ + name: "test_readonly_pack", + file: upload + }) + |> json_response_and_validate_schema(400) == %{ + "error" => "Could not create the pack directory" + } + + # Clean up - restore original permissions + File.chmod!(emoji_path, original_mode) + File.rm!(zip_path) + end + test "preserves existing pack.json if present in ZIP", %{admin_conn: admin_conn} do # Create ZIP with pack.json {:ok, zip_path} = create_test_emoji_zip_with_pack_json() From 80e0f072407728d06fd931ebebca8fd91cc80918 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Fri, 8 Aug 2025 15:44:30 +0400 Subject: [PATCH 72/74] Emoji.Pack: Implement empty name and directory creation failure handling --- lib/pleroma/emoji/pack.ex | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/emoji/pack.ex b/lib/pleroma/emoji/pack.ex index 1a4625db6..616af54ba 100644 --- a/lib/pleroma/emoji/pack.ex +++ b/lib/pleroma/emoji/pack.ex @@ -230,15 +230,23 @@ defmodule Pleroma.Emoji.Pack do :ok <- validate_new_pack(name), {:ok, archive_data} <- fetch_archive_data(opts), pack_path <- path_join_name_safe(emoji_path(), name), - :ok <- File.mkdir_p(pack_path), + :ok <- create_pack_dir(pack_path), :ok <- safe_unzip(archive_data, pack_path) do ensure_pack_json(pack_path, archive_data, opts) else + {:error, :empty_values} -> {:error, "Pack name cannot be empty"} {:error, reason} when is_binary(reason) -> {:error, reason} _ -> {:error, "Could not process pack"} end end + defp create_pack_dir(pack_path) do + case File.mkdir_p(pack_path) do + :ok -> :ok + {:error, _} -> {:error, "Could not create the pack directory"} + end + end + defp safe_unzip(archive_data, pack_path) do case SafeZip.unzip_data(archive_data, pack_path) do {:ok, _} -> :ok From 4ab96bbb9f950cfd90818ee7fc7ea863690d6eee Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sat, 9 Aug 2025 11:11:44 +0400 Subject: [PATCH 73/74] EmojiPackControllerDownloadZipTest: Use a unique folder for each test. --- ...moji_pack_controller_download_zip_test.exs | 89 +++++++++---------- 1 file changed, 44 insertions(+), 45 deletions(-) diff --git a/test/pleroma/web/pleroma_api/controllers/emoji_pack_controller_download_zip_test.exs b/test/pleroma/web/pleroma_api/controllers/emoji_pack_controller_download_zip_test.exs index 50f6446dc..5150a75f0 100644 --- a/test/pleroma/web/pleroma_api/controllers/emoji_pack_controller_download_zip_test.exs +++ b/test/pleroma/web/pleroma_api/controllers/emoji_pack_controller_download_zip_test.exs @@ -8,12 +8,30 @@ defmodule Pleroma.Web.PleromaAPI.EmojiPackControllerDownloadZipTest do import Tesla.Mock import Pleroma.Factory - @emoji_path Path.join( - Pleroma.Config.get!([:instance, :static_dir]), - "emoji" - ) + setup_all do + # Create a base temp directory for this test module + base_temp_dir = Path.join(System.tmp_dir!(), "emoji_test_#{Ecto.UUID.generate()}") + + # Clean up when all tests in module are done + on_exit(fn -> + File.rm_rf!(base_temp_dir) + end) + + {:ok, %{base_temp_dir: base_temp_dir}} + end + + setup %{base_temp_dir: base_temp_dir} do + # Create a unique subdirectory for each test + test_id = Ecto.UUID.generate() + temp_dir = Path.join(base_temp_dir, test_id) + emoji_dir = Path.join(temp_dir, "emoji") + + # Create the directory structure + File.mkdir_p!(emoji_dir) + + # Configure this test to use the temp directory + clear_config([:instance, :static_dir], temp_dir) - setup do admin = insert(:user, is_admin: true) token = insert(:oauth_admin_token, user: admin) @@ -24,27 +42,7 @@ defmodule Pleroma.Web.PleromaAPI.EmojiPackControllerDownloadZipTest do Pleroma.Emoji.reload() - # Clean up any test packs from previous runs - on_exit(fn -> - test_packs = [ - "test_zip_pack", - "test_zip_pack_url", - "test_zip_pack_malicious", - "test_invalid_pack", - "test_bad_url_pack", - "test_no_source_pack" - ] - - Enum.each(test_packs, fn pack_name -> - pack_path = Path.join(@emoji_path, pack_name) - - if File.exists?(pack_path) do - File.rm_rf!(pack_path) - end - end) - end) - - {:ok, %{admin_conn: admin_conn}} + {:ok, %{admin_conn: admin_conn, emoji_path: emoji_dir}} end describe "POST /api/pleroma/emoji/packs/download_zip" do @@ -52,7 +50,7 @@ defmodule Pleroma.Web.PleromaAPI.EmojiPackControllerDownloadZipTest do clear_config([:instance, :admin_privileges], [:emoji_manage_emoji]) end - test "creates pack from uploaded ZIP file", %{admin_conn: admin_conn} do + test "creates pack from uploaded ZIP file", %{admin_conn: admin_conn, emoji_path: emoji_path} do # Create a test ZIP file with emojis {:ok, zip_path} = create_test_emoji_zip() @@ -71,11 +69,11 @@ defmodule Pleroma.Web.PleromaAPI.EmojiPackControllerDownloadZipTest do |> json_response_and_validate_schema(200) == "ok" # Verify pack was created - assert File.exists?("#{@emoji_path}/test_zip_pack/pack.json") - assert File.exists?("#{@emoji_path}/test_zip_pack/test_emoji.png") + assert File.exists?("#{emoji_path}/test_zip_pack/pack.json") + assert File.exists?("#{emoji_path}/test_zip_pack/test_emoji.png") # Verify pack.json contents - {:ok, pack_json} = File.read("#{@emoji_path}/test_zip_pack/pack.json") + {:ok, pack_json} = File.read("#{emoji_path}/test_zip_pack/pack.json") pack_data = Jason.decode!(pack_json) assert pack_data["files"]["test_emoji"] == "test_emoji.png" @@ -85,7 +83,7 @@ defmodule Pleroma.Web.PleromaAPI.EmojiPackControllerDownloadZipTest do File.rm!(zip_path) end - test "creates pack from URL", %{admin_conn: admin_conn} do + test "creates pack from URL", %{admin_conn: admin_conn, emoji_path: emoji_path} do # Mock HTTP request to download ZIP {:ok, zip_path} = create_test_emoji_zip() {:ok, zip_data} = File.read(zip_path) @@ -104,11 +102,11 @@ defmodule Pleroma.Web.PleromaAPI.EmojiPackControllerDownloadZipTest do |> json_response_and_validate_schema(200) == "ok" # Verify pack was created - assert File.exists?("#{@emoji_path}/test_zip_pack_url/pack.json") - assert File.exists?("#{@emoji_path}/test_zip_pack_url/test_emoji.png") + assert File.exists?("#{emoji_path}/test_zip_pack_url/pack.json") + assert File.exists?("#{emoji_path}/test_zip_pack_url/test_emoji.png") # Verify pack.json has URL as source - {:ok, pack_json} = File.read("#{@emoji_path}/test_zip_pack_url/pack.json") + {:ok, pack_json} = File.read("#{emoji_path}/test_zip_pack_url/pack.json") pack_data = Jason.decode!(pack_json) assert pack_data["pack"]["src"] == "https://example.com/emoji_pack.zip" @@ -118,9 +116,9 @@ defmodule Pleroma.Web.PleromaAPI.EmojiPackControllerDownloadZipTest do File.rm!(zip_path) end - test "refuses to overwrite existing pack", %{admin_conn: admin_conn} do + test "refuses to overwrite existing pack", %{admin_conn: admin_conn, emoji_path: emoji_path} do # Create existing pack - pack_path = Path.join(@emoji_path, "test_zip_pack") + pack_path = Path.join(emoji_path, "test_zip_pack") File.mkdir_p!(pack_path) File.write!(Path.join(pack_path, "pack.json"), Jason.encode!(%{files: %{}})) @@ -222,13 +220,11 @@ defmodule Pleroma.Web.PleromaAPI.EmojiPackControllerDownloadZipTest do File.rm!(zip_path) end - test "returns error when unable to create pack directory", %{admin_conn: admin_conn} do + test "returns error when unable to create pack directory", %{ + admin_conn: admin_conn, + emoji_path: emoji_path + } do # Make the emoji directory read-only to trigger mkdir_p failure - emoji_path = - Path.join( - Pleroma.Config.get!([:instance, :static_dir]), - "emoji" - ) # Save original permissions {:ok, %{mode: original_mode}} = File.stat(emoji_path) @@ -260,7 +256,10 @@ defmodule Pleroma.Web.PleromaAPI.EmojiPackControllerDownloadZipTest do File.rm!(zip_path) end - test "preserves existing pack.json if present in ZIP", %{admin_conn: admin_conn} do + test "preserves existing pack.json if present in ZIP", %{ + admin_conn: admin_conn, + emoji_path: emoji_path + } do # Create ZIP with pack.json {:ok, zip_path} = create_test_emoji_zip_with_pack_json() @@ -273,13 +272,13 @@ defmodule Pleroma.Web.PleromaAPI.EmojiPackControllerDownloadZipTest do assert admin_conn |> put_req_header("content-type", "multipart/form-data") |> post("/api/pleroma/emoji/packs/download_zip", %{ - name: "test_zip_pack", + name: "test_zip_pack_with_json", file: upload }) |> json_response_and_validate_schema(200) == "ok" # Verify original pack.json was preserved - {:ok, pack_json} = File.read("#{@emoji_path}/test_zip_pack/pack.json") + {:ok, pack_json} = File.read("#{emoji_path}/test_zip_pack_with_json/pack.json") pack_data = Jason.decode!(pack_json) assert pack_data["pack"]["description"] == "Test pack from ZIP" From 20812151a7f4483c9d68bbd458d2bc2ac018cf21 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sun, 10 Aug 2025 17:44:21 +0400 Subject: [PATCH 74/74] Gitlab CI: Don't run as root. --- .gitlab-ci.yml | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bd36387c9..a3733eebe 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -132,10 +132,25 @@ unit-testing-1.14.5-otp-25: - name: postgres:13-alpine alias: postgres command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] + before_script: &testing_before_script + - echo $MIX_ENV + - rm -rf _build/*/lib/pleroma + # Create a non-root user for running tests + - useradd -m -s /bin/bash testuser + # Install dependencies as root first + - mix deps.get + # Set proper ownership for everything + - chown -R testuser:testuser . + - chown -R testuser:testuser /root/.mix || true + - chown -R testuser:testuser /root/.hex || true + # Create user-specific directories + - su testuser -c "HOME=/home/testuser mix local.hex --force" + - su testuser -c "HOME=/home/testuser mix local.rebar --force" script: &testing_script - - mix ecto.create - - mix ecto.migrate - - mix pleroma.test_runner --cover --preload-modules + # Run tests as non-root user + - su testuser -c "HOME=/home/testuser mix ecto.create" + - su testuser -c "HOME=/home/testuser mix ecto.migrate" + - su testuser -c "HOME=/home/testuser mix pleroma.test_runner --cover --preload-modules" coverage: '/^Line total: ([^ ]*%)$/' artifacts: reports: @@ -151,6 +166,7 @@ unit-testing-1.18.3-otp-27: image: git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.18.3-otp-27 cache: *testing_cache_policy services: *testing_services + before_script: *testing_before_script script: *testing_script formatting-1.15: