diff --git a/changelog.d/stats-peers-from-instances.change b/changelog.d/stats-peers-from-instances.change new file mode 100644 index 000000000..a3dfe2e0c --- /dev/null +++ b/changelog.d/stats-peers-from-instances.change @@ -0,0 +1 @@ +Read the peer list from the instances table (hosts of remote actors, without our own host) and update instance stats every 5 minutes; the old peers query scanned all remote users every minute diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex index 3695e0b75..3d3ef5774 100644 --- a/lib/pleroma/instances/instance.ex +++ b/lib/pleroma/instances/instance.ex @@ -91,6 +91,13 @@ defmodule Pleroma.Instances.Instance do def reachable?(url_or_host) when is_binary(url_or_host), do: true + @doc "Creates a record for the instance unless one exists. Existing records are kept." + def ensure_exists(url_or_host) when is_binary(url_or_host) do + %Instance{host: host(url_or_host)} + |> changeset(%{}) + |> Repo.insert(on_conflict: :nothing, conflict_target: :host) + end + def set_reachable(url_or_host) when is_binary(url_or_host) do host = host(url_or_host) diff --git a/lib/pleroma/stats.ex b/lib/pleroma/stats.ex index 47b30b951..079f6f7dd 100644 --- a/lib/pleroma/stats.ex +++ b/lib/pleroma/stats.ex @@ -8,10 +8,11 @@ defmodule Pleroma.Stats do import Ecto.Query alias Pleroma.CounterCache + alias Pleroma.Instances.Instance alias Pleroma.Repo alias Pleroma.User - @interval :timer.seconds(60) + @interval :timer.seconds(300) def start_link(_) do GenServer.start_link( @@ -64,14 +65,12 @@ defmodule Pleroma.Stats do } } def calculate_stat_data do + # Hosts are unique in instances, and remote users add theirs. Scanning + # all remote users for their domains was one of the costliest queries. + # Favicon lookups for local accounts can add our own host, so skip it. peers = - from( - u in User, - select: fragment("distinct split_part(?, '@', 2)", u.nickname), - where: u.local != ^true - ) + from(i in Instance, where: i.host != ^local_host(), select: i.host) |> Repo.all() - |> Enum.filter(& &1) domain_count = Enum.count(peers) @@ -97,6 +96,9 @@ defmodule Pleroma.Stats do } end + # Read from the config: this also runs before the endpoint has started. + defp local_host, do: Pleroma.Config.get([Pleroma.Web.Endpoint, :url, :host]) + @spec get_status_visibility_count(String.t() | nil) :: map() def get_status_visibility_count(instance \\ nil) do if is_nil(instance) do diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index b997e7831..f6c74bdac 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -11,6 +11,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do alias Pleroma.Conversation.Participation alias Pleroma.Filter alias Pleroma.Hashtag + alias Pleroma.Instances.Instance alias Pleroma.Maps alias Pleroma.Notification alias Pleroma.Object @@ -2057,6 +2058,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do def make_user_from_ap_id(ap_id, additional \\ []) do with {:ok, data} <- fetch_and_prepare_user_from_ap_id(ap_id, additional) do enqueue_pin_fetches(data) + # Keeps the instances table complete, so it can list our peers. + Instance.ensure_exists(data.ap_id) # the fetched actor's id differs from ap_id when it was fetched from a display URL user = User.get_cached_by_ap_id(data.ap_id) diff --git a/priv/repo/migrations/20261028000000_backfill_instances_from_users.exs b/priv/repo/migrations/20261028000000_backfill_instances_from_users.exs new file mode 100644 index 000000000..88dfe9ec5 --- /dev/null +++ b/priv/repo/migrations/20261028000000_backfill_instances_from_users.exs @@ -0,0 +1,30 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2026 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Repo.Migrations.BackfillInstancesFromUsers do + use Ecto.Migration + + # Remote users now create an instances row; add the hosts of existing ones, + # so the peer list can come from the instances table. + def up do + Pleroma.Repo.query!( + """ + INSERT INTO instances (host, inserted_at, updated_at) + SELECT DISTINCT host, now(), now() + FROM ( + -- Skips userinfo; IPv6 literals give NULL and are left out. + SELECT substring(ap_id from '^[A-Za-z][A-Za-z0-9+.-]*://(?:[^/?#@]*@)?([^/:?#\\[]+)') AS host + FROM users + WHERE local = false AND ap_id IS NOT NULL + ) AS hosts + WHERE host IS NOT NULL + ON CONFLICT (host) DO NOTHING + """, + [], + timeout: :infinity + ) + end + + def down, do: :ok +end diff --git a/test/pleroma/repo/migrations/backfill_instances_from_users_test.exs b/test/pleroma/repo/migrations/backfill_instances_from_users_test.exs new file mode 100644 index 000000000..811e336bd --- /dev/null +++ b/test/pleroma/repo/migrations/backfill_instances_from_users_test.exs @@ -0,0 +1,33 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2026 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Repo.Migrations.BackfillInstancesFromUsersTest do + use Pleroma.DataCase + import Pleroma.Factory + import Pleroma.Tests.Helpers + + alias Pleroma.Instances.Instance + + setup_all do: require_migration("20261028000000_backfill_instances_from_users") + + test "up/0 adds the hosts of remote users and keeps existing records", %{migration: migration} do + unreachable_since = ~N[2020-01-01 00:00:00.000000] + insert(:instance, host: "known.example", unreachable_since: unreachable_since) + + insert(:user, local: false, ap_id: "https://known.example/users/a") + insert(:user, local: false, ap_id: "https://new.example/users/a") + insert(:user, local: false, ap_id: "https://new.example/users/b") + insert(:user, local: false, ap_id: "https://ported.example:8443/users/c") + insert(:user, local: false, ap_id: "https://user@userinfo.example/users/d") + insert(:user, local: false, ap_id: "https://[2001:db8::1]/users/e") + insert(:user, local: true) + + migration.up() + + hosts = Instance |> Repo.all() |> Enum.map(& &1.host) |> Enum.sort() + assert hosts == ["known.example", "new.example", "ported.example", "userinfo.example"] + + assert %{unreachable_since: ^unreachable_since} = Repo.get_by(Instance, host: "known.example") + end +end diff --git a/test/pleroma/web/activity_pub/activity_pub_test.exs b/test/pleroma/web/activity_pub/activity_pub_test.exs index c118cd8f1..bd7c97820 100644 --- a/test/pleroma/web/activity_pub/activity_pub_test.exs +++ b/test/pleroma/web/activity_pub/activity_pub_test.exs @@ -184,6 +184,34 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do assert user.follower_address == "http://mastodon.example.org/users/admin/followers" end + test "it records the user's instance" do + refute Repo.get_by(Pleroma.Instances.Instance, host: "mastodon.example.org") + + {:ok, _user} = ActivityPub.make_user_from_ap_id("http://mastodon.example.org/users/admin") + + assert %{unreachable_since: nil} = + Repo.get_by(Pleroma.Instances.Instance, host: "mastodon.example.org") + end + + test "it records the instance of a known user when refreshing it" do + insert(:user, local: false, ap_id: "http://mastodon.example.org/users/admin") + refute Repo.get_by(Pleroma.Instances.Instance, host: "mastodon.example.org") + + {:ok, _user} = ActivityPub.make_user_from_ap_id("http://mastodon.example.org/users/admin") + + assert Repo.get_by(Pleroma.Instances.Instance, host: "mastodon.example.org") + end + + test "it keeps an existing instance record" do + unreachable_since = ~N[2020-01-01 00:00:00.000000] + insert(:instance, host: "mastodon.example.org", unreachable_since: unreachable_since) + + {:ok, _user} = ActivityPub.make_user_from_ap_id("http://mastodon.example.org/users/admin") + + assert %{unreachable_since: ^unreachable_since} = + Repo.get_by(Pleroma.Instances.Instance, host: "mastodon.example.org") + end + test "it returns a user that is invisible" do user_id = "http://mastodon.example.org/users/relay" {:ok, user} = ActivityPub.make_user_from_ap_id(user_id) diff --git a/test/pleroma/web/mastodon_api/controllers/instance_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/instance_controller_test.exs index 461b46066..f9d42d981 100644 --- a/test/pleroma/web/mastodon_api/controllers/instance_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/instance_controller_test.exs @@ -63,8 +63,8 @@ defmodule Pleroma.Web.MastodonAPI.InstanceControllerTest do user2 = insert(:user, %{local: true}) {:ok, _user2} = User.set_activation(user2, false) - insert(:user, %{local: false, nickname: "u@peer1.com"}) - insert(:user, %{local: false, nickname: "u@peer2.com"}) + insert(:instance, host: "peer1.com") + insert(:instance, host: "peer2.com") {:ok, _} = Pleroma.Web.CommonAPI.post(user, %{status: "cofe"}) @@ -83,8 +83,11 @@ defmodule Pleroma.Web.MastodonAPI.InstanceControllerTest do end test "get peers", %{conn: conn} do - insert(:user, %{local: false, nickname: "u@peer1.com"}) - insert(:user, %{local: false, nickname: "u@peer2.com"}) + # Peers come from the instances table, which remote users add to. + insert(:instance, host: "peer1.com") + insert(:instance, host: "peer2.com") + # Favicon lookups for local accounts can add our own host. + insert(:instance, host: Pleroma.Web.Endpoint.host()) Pleroma.Stats.force_update()