From 8e3404a239415e95be53f8f38e4e3f331ca56789 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 27 Oct 2026 20:48:02 +0400 Subject: [PATCH 1/5] feat(instances): record the instance of new remote users The instances table only got rows from fetches, reachability changes and favicon lookups, so most hosts we know users from had none. Insert a row when a remote user is created, keeping existing rows. --- lib/pleroma/instances/instance.ex | 7 +++++++ lib/pleroma/web/activity_pub/activity_pub.ex | 10 ++++++---- .../web/activity_pub/activity_pub_test.exs | 19 +++++++++++++++++++ 3 files changed, 32 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex index 3695e0b75..3d3ef5774 100644 --- a/lib/pleroma/instances/instance.ex +++ b/lib/pleroma/instances/instance.ex @@ -91,6 +91,13 @@ defmodule Pleroma.Instances.Instance do def reachable?(url_or_host) when is_binary(url_or_host), do: true + @doc "Creates a record for the instance unless one exists. Existing records are kept." + def ensure_exists(url_or_host) when is_binary(url_or_host) do + %Instance{host: host(url_or_host)} + |> changeset(%{}) + |> Repo.insert(on_conflict: :nothing, conflict_target: :host) + end + def set_reachable(url_or_host) when is_binary(url_or_host) do host = host(url_or_host) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index b997e7831..a5325d8ec 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -11,6 +11,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do alias Pleroma.Conversation.Participation alias Pleroma.Filter alias Pleroma.Hashtag + alias Pleroma.Instances.Instance alias Pleroma.Maps alias Pleroma.Notification alias Pleroma.Object @@ -2068,10 +2069,11 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do else maybe_handle_clashing_nickname(data) - data - |> User.remote_user_changeset() - |> Repo.insert() - |> User.set_cache() + with {:ok, user} <- data |> User.remote_user_changeset() |> Repo.insert() do + # Keeps the instances table complete, so it can list our peers. + Instance.ensure_exists(user.ap_id) + User.set_cache({:ok, user}) + end end end end diff --git a/test/pleroma/web/activity_pub/activity_pub_test.exs b/test/pleroma/web/activity_pub/activity_pub_test.exs index c118cd8f1..fcebdffb1 100644 --- a/test/pleroma/web/activity_pub/activity_pub_test.exs +++ b/test/pleroma/web/activity_pub/activity_pub_test.exs @@ -184,6 +184,25 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do assert user.follower_address == "http://mastodon.example.org/users/admin/followers" end + test "it records the user's instance" do + refute Repo.get_by(Pleroma.Instances.Instance, host: "mastodon.example.org") + + {:ok, _user} = ActivityPub.make_user_from_ap_id("http://mastodon.example.org/users/admin") + + assert %{unreachable_since: nil} = + Repo.get_by(Pleroma.Instances.Instance, host: "mastodon.example.org") + end + + test "it keeps an existing instance record" do + unreachable_since = ~N[2020-01-01 00:00:00.000000] + insert(:instance, host: "mastodon.example.org", unreachable_since: unreachable_since) + + {:ok, _user} = ActivityPub.make_user_from_ap_id("http://mastodon.example.org/users/admin") + + assert %{unreachable_since: ^unreachable_since} = + Repo.get_by(Pleroma.Instances.Instance, host: "mastodon.example.org") + end + test "it returns a user that is invisible" do user_id = "http://mastodon.example.org/users/relay" {:ok, user} = ActivityPub.make_user_from_ap_id(user_id) From db11e5e790eb255742155a806652e2732cf841ca Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 27 Oct 2026 20:49:07 +0400 Subject: [PATCH 2/5] feat(migration): add the hosts of existing remote users to instances --- ...28000000_backfill_instances_from_users.exs | 29 +++++++++++++++++ .../backfill_instances_from_users_test.exs | 31 +++++++++++++++++++ 2 files changed, 60 insertions(+) create mode 100644 priv/repo/migrations/20261028000000_backfill_instances_from_users.exs create mode 100644 test/pleroma/repo/migrations/backfill_instances_from_users_test.exs diff --git a/priv/repo/migrations/20261028000000_backfill_instances_from_users.exs b/priv/repo/migrations/20261028000000_backfill_instances_from_users.exs new file mode 100644 index 000000000..13d805c30 --- /dev/null +++ b/priv/repo/migrations/20261028000000_backfill_instances_from_users.exs @@ -0,0 +1,29 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2026 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Repo.Migrations.BackfillInstancesFromUsers do + use Ecto.Migration + + # Remote users now create an instances row; add the hosts of existing ones, + # so the peer list can come from the instances table. + def up do + Pleroma.Repo.query!( + """ + INSERT INTO instances (host, inserted_at, updated_at) + SELECT DISTINCT host, now(), now() + FROM ( + SELECT substring(ap_id from '^[A-Za-z][A-Za-z0-9+.-]*://([^/:?#]+)') AS host + FROM users + WHERE local = false AND ap_id IS NOT NULL + ) AS hosts + WHERE host IS NOT NULL + ON CONFLICT (host) DO NOTHING + """, + [], + timeout: :infinity + ) + end + + def down, do: :ok +end diff --git a/test/pleroma/repo/migrations/backfill_instances_from_users_test.exs b/test/pleroma/repo/migrations/backfill_instances_from_users_test.exs new file mode 100644 index 000000000..733c6644b --- /dev/null +++ b/test/pleroma/repo/migrations/backfill_instances_from_users_test.exs @@ -0,0 +1,31 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2026 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Repo.Migrations.BackfillInstancesFromUsersTest do + use Pleroma.DataCase + import Pleroma.Factory + import Pleroma.Tests.Helpers + + alias Pleroma.Instances.Instance + + setup_all do: require_migration("20261028000000_backfill_instances_from_users") + + test "up/0 adds the hosts of remote users and keeps existing records", %{migration: migration} do + unreachable_since = ~N[2020-01-01 00:00:00.000000] + insert(:instance, host: "known.example", unreachable_since: unreachable_since) + + insert(:user, local: false, ap_id: "https://known.example/users/a") + insert(:user, local: false, ap_id: "https://new.example/users/a") + insert(:user, local: false, ap_id: "https://new.example/users/b") + insert(:user, local: false, ap_id: "https://ported.example:8443/users/c") + insert(:user, local: true) + + migration.up() + + hosts = Instance |> Repo.all() |> Enum.map(& &1.host) |> Enum.sort() + assert hosts == ["known.example", "new.example", "ported.example"] + + assert %{unreachable_since: ^unreachable_since} = Repo.get_by(Instance, host: "known.example") + end +end From e8f238359f6a35a34094dfd5f02a653048c22c6e Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 27 Oct 2026 20:49:52 +0400 Subject: [PATCH 3/5] perf(stats): read peers from instances, update every 5 minutes The peers query read the nickname of every remote user every minute (2.7 s and 1.2M buffer hits on a large instance). The instances table now has a row for every host we know users from, so read it instead, and recompute the stats every 5 minutes. Based on Akkoma 138b1aea2 and 7b76fdeed. Co-authored-by: Oneric Co-authored-by: FloatingGhost --- changelog.d/stats-peers-from-instances.change | 1 + lib/pleroma/stats.ex | 14 +++++--------- .../controllers/instance_controller_test.exs | 9 +++++---- 3 files changed, 11 insertions(+), 13 deletions(-) create mode 100644 changelog.d/stats-peers-from-instances.change diff --git a/changelog.d/stats-peers-from-instances.change b/changelog.d/stats-peers-from-instances.change new file mode 100644 index 000000000..c493b4550 --- /dev/null +++ b/changelog.d/stats-peers-from-instances.change @@ -0,0 +1 @@ +Read the peer list from the instances table and update instance stats every 5 minutes instead of every minute; the old peers query was one of the most expensive regular queries diff --git a/lib/pleroma/stats.ex b/lib/pleroma/stats.ex index 47b30b951..328aa110a 100644 --- a/lib/pleroma/stats.ex +++ b/lib/pleroma/stats.ex @@ -8,10 +8,11 @@ defmodule Pleroma.Stats do import Ecto.Query alias Pleroma.CounterCache + alias Pleroma.Instances.Instance alias Pleroma.Repo alias Pleroma.User - @interval :timer.seconds(60) + @interval :timer.seconds(300) def start_link(_) do GenServer.start_link( @@ -64,14 +65,9 @@ defmodule Pleroma.Stats do } } def calculate_stat_data do - peers = - from( - u in User, - select: fragment("distinct split_part(?, '@', 2)", u.nickname), - where: u.local != ^true - ) - |> Repo.all() - |> Enum.filter(& &1) + # Hosts are unique in instances, and remote users add theirs. Scanning + # all remote users for their domains was one of the costliest queries. + peers = Repo.all(from(i in Instance, select: i.host)) domain_count = Enum.count(peers) diff --git a/test/pleroma/web/mastodon_api/controllers/instance_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/instance_controller_test.exs index 461b46066..404d276db 100644 --- a/test/pleroma/web/mastodon_api/controllers/instance_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/instance_controller_test.exs @@ -63,8 +63,8 @@ defmodule Pleroma.Web.MastodonAPI.InstanceControllerTest do user2 = insert(:user, %{local: true}) {:ok, _user2} = User.set_activation(user2, false) - insert(:user, %{local: false, nickname: "u@peer1.com"}) - insert(:user, %{local: false, nickname: "u@peer2.com"}) + insert(:instance, host: "peer1.com") + insert(:instance, host: "peer2.com") {:ok, _} = Pleroma.Web.CommonAPI.post(user, %{status: "cofe"}) @@ -83,8 +83,9 @@ defmodule Pleroma.Web.MastodonAPI.InstanceControllerTest do end test "get peers", %{conn: conn} do - insert(:user, %{local: false, nickname: "u@peer1.com"}) - insert(:user, %{local: false, nickname: "u@peer2.com"}) + # Peers come from the instances table, which remote users add to. + insert(:instance, host: "peer1.com") + insert(:instance, host: "peer2.com") Pleroma.Stats.force_update() From 958166589ef5a77f0274e2f5b5cb76273ed940ac Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 27 Oct 2026 20:53:56 +0400 Subject: [PATCH 4/5] fix(stats): leave our own host out of the peers Favicon lookups for local accounts can add our own host to instances. Also skip userinfo and IPv6 literals in the backfill migration. --- changelog.d/stats-peers-from-instances.change | 2 +- lib/pleroma/stats.ex | 8 +++++++- .../20261028000000_backfill_instances_from_users.exs | 3 ++- .../migrations/backfill_instances_from_users_test.exs | 4 +++- .../mastodon_api/controllers/instance_controller_test.exs | 2 ++ 5 files changed, 15 insertions(+), 4 deletions(-) diff --git a/changelog.d/stats-peers-from-instances.change b/changelog.d/stats-peers-from-instances.change index c493b4550..a3dfe2e0c 100644 --- a/changelog.d/stats-peers-from-instances.change +++ b/changelog.d/stats-peers-from-instances.change @@ -1 +1 @@ -Read the peer list from the instances table and update instance stats every 5 minutes instead of every minute; the old peers query was one of the most expensive regular queries +Read the peer list from the instances table (hosts of remote actors, without our own host) and update instance stats every 5 minutes; the old peers query scanned all remote users every minute diff --git a/lib/pleroma/stats.ex b/lib/pleroma/stats.ex index 328aa110a..079f6f7dd 100644 --- a/lib/pleroma/stats.ex +++ b/lib/pleroma/stats.ex @@ -67,7 +67,10 @@ defmodule Pleroma.Stats do def calculate_stat_data do # Hosts are unique in instances, and remote users add theirs. Scanning # all remote users for their domains was one of the costliest queries. - peers = Repo.all(from(i in Instance, select: i.host)) + # Favicon lookups for local accounts can add our own host, so skip it. + peers = + from(i in Instance, where: i.host != ^local_host(), select: i.host) + |> Repo.all() domain_count = Enum.count(peers) @@ -93,6 +96,9 @@ defmodule Pleroma.Stats do } end + # Read from the config: this also runs before the endpoint has started. + defp local_host, do: Pleroma.Config.get([Pleroma.Web.Endpoint, :url, :host]) + @spec get_status_visibility_count(String.t() | nil) :: map() def get_status_visibility_count(instance \\ nil) do if is_nil(instance) do diff --git a/priv/repo/migrations/20261028000000_backfill_instances_from_users.exs b/priv/repo/migrations/20261028000000_backfill_instances_from_users.exs index 13d805c30..88dfe9ec5 100644 --- a/priv/repo/migrations/20261028000000_backfill_instances_from_users.exs +++ b/priv/repo/migrations/20261028000000_backfill_instances_from_users.exs @@ -13,7 +13,8 @@ defmodule Pleroma.Repo.Migrations.BackfillInstancesFromUsers do INSERT INTO instances (host, inserted_at, updated_at) SELECT DISTINCT host, now(), now() FROM ( - SELECT substring(ap_id from '^[A-Za-z][A-Za-z0-9+.-]*://([^/:?#]+)') AS host + -- Skips userinfo; IPv6 literals give NULL and are left out. + SELECT substring(ap_id from '^[A-Za-z][A-Za-z0-9+.-]*://(?:[^/?#@]*@)?([^/:?#\\[]+)') AS host FROM users WHERE local = false AND ap_id IS NOT NULL ) AS hosts diff --git a/test/pleroma/repo/migrations/backfill_instances_from_users_test.exs b/test/pleroma/repo/migrations/backfill_instances_from_users_test.exs index 733c6644b..811e336bd 100644 --- a/test/pleroma/repo/migrations/backfill_instances_from_users_test.exs +++ b/test/pleroma/repo/migrations/backfill_instances_from_users_test.exs @@ -19,12 +19,14 @@ defmodule Pleroma.Repo.Migrations.BackfillInstancesFromUsersTest do insert(:user, local: false, ap_id: "https://new.example/users/a") insert(:user, local: false, ap_id: "https://new.example/users/b") insert(:user, local: false, ap_id: "https://ported.example:8443/users/c") + insert(:user, local: false, ap_id: "https://user@userinfo.example/users/d") + insert(:user, local: false, ap_id: "https://[2001:db8::1]/users/e") insert(:user, local: true) migration.up() hosts = Instance |> Repo.all() |> Enum.map(& &1.host) |> Enum.sort() - assert hosts == ["known.example", "new.example", "ported.example"] + assert hosts == ["known.example", "new.example", "ported.example", "userinfo.example"] assert %{unreachable_since: ^unreachable_since} = Repo.get_by(Instance, host: "known.example") end diff --git a/test/pleroma/web/mastodon_api/controllers/instance_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/instance_controller_test.exs index 404d276db..f9d42d981 100644 --- a/test/pleroma/web/mastodon_api/controllers/instance_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/instance_controller_test.exs @@ -86,6 +86,8 @@ defmodule Pleroma.Web.MastodonAPI.InstanceControllerTest do # Peers come from the instances table, which remote users add to. insert(:instance, host: "peer1.com") insert(:instance, host: "peer2.com") + # Favicon lookups for local accounts can add our own host. + insert(:instance, host: Pleroma.Web.Endpoint.host()) Pleroma.Stats.force_update() From 1849356abe698cee3c9590944d3883e1758aafc1 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Thu, 29 Oct 2026 10:42:46 +0400 Subject: [PATCH 5/5] fix(instances): record the instance when refreshing known users too Hosts without a row (IPv6 literals skipped by the backfill, rows deleted later) were never added again, because known users take the update path. --- lib/pleroma/web/activity_pub/activity_pub.ex | 11 ++++++----- test/pleroma/web/activity_pub/activity_pub_test.exs | 9 +++++++++ 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index a5325d8ec..f6c74bdac 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -2058,6 +2058,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do def make_user_from_ap_id(ap_id, additional \\ []) do with {:ok, data} <- fetch_and_prepare_user_from_ap_id(ap_id, additional) do enqueue_pin_fetches(data) + # Keeps the instances table complete, so it can list our peers. + Instance.ensure_exists(data.ap_id) # the fetched actor's id differs from ap_id when it was fetched from a display URL user = User.get_cached_by_ap_id(data.ap_id) @@ -2069,11 +2071,10 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do else maybe_handle_clashing_nickname(data) - with {:ok, user} <- data |> User.remote_user_changeset() |> Repo.insert() do - # Keeps the instances table complete, so it can list our peers. - Instance.ensure_exists(user.ap_id) - User.set_cache({:ok, user}) - end + data + |> User.remote_user_changeset() + |> Repo.insert() + |> User.set_cache() end end end diff --git a/test/pleroma/web/activity_pub/activity_pub_test.exs b/test/pleroma/web/activity_pub/activity_pub_test.exs index fcebdffb1..bd7c97820 100644 --- a/test/pleroma/web/activity_pub/activity_pub_test.exs +++ b/test/pleroma/web/activity_pub/activity_pub_test.exs @@ -193,6 +193,15 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do Repo.get_by(Pleroma.Instances.Instance, host: "mastodon.example.org") end + test "it records the instance of a known user when refreshing it" do + insert(:user, local: false, ap_id: "http://mastodon.example.org/users/admin") + refute Repo.get_by(Pleroma.Instances.Instance, host: "mastodon.example.org") + + {:ok, _user} = ActivityPub.make_user_from_ap_id("http://mastodon.example.org/users/admin") + + assert Repo.get_by(Pleroma.Instances.Instance, host: "mastodon.example.org") + end + test "it keeps an existing instance record" do unreachable_since = ~N[2020-01-01 00:00:00.000000] insert(:instance, host: "mastodon.example.org", unreachable_since: unreachable_since)