Merge pull request 'Stats: read peers from the instances table' (#8030) from lambadalambda/pleroma:perf/stats-peers-from-instances into develop

Reviewed-on: https://git.pleroma.social/pleroma/pleroma/pulls/8030

changelog.d/stats-peers-from-instances.change

@@ -0,0 +1 @@
+Read the peer list from the instances table (hosts of remote actors, without our own host) and update instance stats every 5 minutes; the old peers query scanned all remote users every minute

lib/pleroma/instances/instance.ex

@@ -91,6 +91,13 @@ defmodule Pleroma.Instances.Instance do
 
   def reachable?(url_or_host) when is_binary(url_or_host), do: true
 
+  @doc "Creates a record for the instance unless one exists. Existing records are kept."
+  def ensure_exists(url_or_host) when is_binary(url_or_host) do
+    %Instance{host: host(url_or_host)}
+    |> changeset(%{})
+    |> Repo.insert(on_conflict: :nothing, conflict_target: :host)
+  end
+
   def set_reachable(url_or_host) when is_binary(url_or_host) do
     host = host(url_or_host)
 

lib/pleroma/stats.ex

@@ -8,10 +8,11 @@ defmodule Pleroma.Stats do
   import Ecto.Query
 
   alias Pleroma.CounterCache
+  alias Pleroma.Instances.Instance
   alias Pleroma.Repo
   alias Pleroma.User
 
-  @interval :timer.seconds(60)
+  @interval :timer.seconds(300)
 
   def start_link(_) do
     GenServer.start_link(
@@ -64,14 +65,12 @@ defmodule Pleroma.Stats do
           }
         }
   def calculate_stat_data do
+    # Hosts are unique in instances, and remote users add theirs. Scanning
+    # all remote users for their domains was one of the costliest queries.
+    # Favicon lookups for local accounts can add our own host, so skip it.
     peers =
-      from(
-        u in User,
-        select: fragment("distinct split_part(?, '@', 2)", u.nickname),
-        where: u.local != ^true
-      )
+      from(i in Instance, where: i.host != ^local_host(), select: i.host)
       |> Repo.all()
-      |> Enum.filter(& &1)
 
     domain_count = Enum.count(peers)
 
@@ -97,6 +96,9 @@ defmodule Pleroma.Stats do
     }
   end
 
+  # Read from the config: this also runs before the endpoint has started.
+  defp local_host, do: Pleroma.Config.get([Pleroma.Web.Endpoint, :url, :host])
+
   @spec get_status_visibility_count(String.t() | nil) :: map()
   def get_status_visibility_count(instance \\ nil) do
     if is_nil(instance) do

lib/pleroma/web/activity_pub/activity_pub.ex

@@ -11,6 +11,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   alias Pleroma.Conversation.Participation
   alias Pleroma.Filter
   alias Pleroma.Hashtag
+  alias Pleroma.Instances.Instance
   alias Pleroma.Maps
   alias Pleroma.Notification
   alias Pleroma.Object
@@ -2057,6 +2058,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   def make_user_from_ap_id(ap_id, additional \\ []) do
     with {:ok, data} <- fetch_and_prepare_user_from_ap_id(ap_id, additional) do
       enqueue_pin_fetches(data)
+      # Keeps the instances table complete, so it can list our peers.
+      Instance.ensure_exists(data.ap_id)
 
       # the fetched actor's id differs from ap_id when it was fetched from a display URL
       user = User.get_cached_by_ap_id(data.ap_id)

priv/repo/migrations/20261028000000_backfill_instances_from_users.exs

@@ -0,0 +1,30 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2026 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Repo.Migrations.BackfillInstancesFromUsers do
+  use Ecto.Migration
+
+  # Remote users now create an instances row; add the hosts of existing ones,
+  # so the peer list can come from the instances table.
+  def up do
+    Pleroma.Repo.query!(
+      """
+      INSERT INTO instances (host, inserted_at, updated_at)
+      SELECT DISTINCT host, now(), now()
+      FROM (
+        -- Skips userinfo; IPv6 literals give NULL and are left out.
+        SELECT substring(ap_id from '^[A-Za-z][A-Za-z0-9+.-]*://(?:[^/?#@]*@)?([^/:?#\\[]+)') AS host
+        FROM users
+        WHERE local = false AND ap_id IS NOT NULL
+      ) AS hosts
+      WHERE host IS NOT NULL
+      ON CONFLICT (host) DO NOTHING
+      """,
+      [],
+      timeout: :infinity
+    )
+  end
+
+  def down, do: :ok
+end

test/pleroma/repo/migrations/backfill_instances_from_users_test.exs

@@ -0,0 +1,33 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2026 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Repo.Migrations.BackfillInstancesFromUsersTest do
+  use Pleroma.DataCase
+  import Pleroma.Factory
+  import Pleroma.Tests.Helpers
+
+  alias Pleroma.Instances.Instance
+
+  setup_all do: require_migration("20261028000000_backfill_instances_from_users")
+
+  test "up/0 adds the hosts of remote users and keeps existing records", %{migration: migration} do
+    unreachable_since = ~N[2020-01-01 00:00:00.000000]
+    insert(:instance, host: "known.example", unreachable_since: unreachable_since)
+
+    insert(:user, local: false, ap_id: "https://known.example/users/a")
+    insert(:user, local: false, ap_id: "https://new.example/users/a")
+    insert(:user, local: false, ap_id: "https://new.example/users/b")
+    insert(:user, local: false, ap_id: "https://ported.example:8443/users/c")
+    insert(:user, local: false, ap_id: "https://user@userinfo.example/users/d")
+    insert(:user, local: false, ap_id: "https://[2001:db8::1]/users/e")
+    insert(:user, local: true)
+
+    migration.up()
+
+    hosts = Instance |> Repo.all() |> Enum.map(& &1.host) |> Enum.sort()
+    assert hosts == ["known.example", "new.example", "ported.example", "userinfo.example"]
+
+    assert %{unreachable_since: ^unreachable_since} = Repo.get_by(Instance, host: "known.example")
+  end
+end

test/pleroma/web/activity_pub/activity_pub_test.exs

@@ -184,6 +184,34 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do
       assert user.follower_address == "http://mastodon.example.org/users/admin/followers"
     end
 
+    test "it records the user's instance" do
+      refute Repo.get_by(Pleroma.Instances.Instance, host: "mastodon.example.org")
+
+      {:ok, _user} = ActivityPub.make_user_from_ap_id("http://mastodon.example.org/users/admin")
+
+      assert %{unreachable_since: nil} =
+               Repo.get_by(Pleroma.Instances.Instance, host: "mastodon.example.org")
+    end
+
+    test "it records the instance of a known user when refreshing it" do
+      insert(:user, local: false, ap_id: "http://mastodon.example.org/users/admin")
+      refute Repo.get_by(Pleroma.Instances.Instance, host: "mastodon.example.org")
+
+      {:ok, _user} = ActivityPub.make_user_from_ap_id("http://mastodon.example.org/users/admin")
+
+      assert Repo.get_by(Pleroma.Instances.Instance, host: "mastodon.example.org")
+    end
+
+    test "it keeps an existing instance record" do
+      unreachable_since = ~N[2020-01-01 00:00:00.000000]
+      insert(:instance, host: "mastodon.example.org", unreachable_since: unreachable_since)
+
+      {:ok, _user} = ActivityPub.make_user_from_ap_id("http://mastodon.example.org/users/admin")
+
+      assert %{unreachable_since: ^unreachable_since} =
+               Repo.get_by(Pleroma.Instances.Instance, host: "mastodon.example.org")
+    end
+
     test "it returns a user that is invisible" do
       user_id = "http://mastodon.example.org/users/relay"
       {:ok, user} = ActivityPub.make_user_from_ap_id(user_id)

test/pleroma/web/mastodon_api/controllers/instance_controller_test.exs

@@ -63,8 +63,8 @@ defmodule Pleroma.Web.MastodonAPI.InstanceControllerTest do
     user2 = insert(:user, %{local: true})
     {:ok, _user2} = User.set_activation(user2, false)
 
-    insert(:user, %{local: false, nickname: "u@peer1.com"})
-    insert(:user, %{local: false, nickname: "u@peer2.com"})
+    insert(:instance, host: "peer1.com")
+    insert(:instance, host: "peer2.com")
 
     {:ok, _} = Pleroma.Web.CommonAPI.post(user, %{status: "cofe"})
 
@@ -83,8 +83,11 @@ defmodule Pleroma.Web.MastodonAPI.InstanceControllerTest do
   end
 
   test "get peers", %{conn: conn} do
-    insert(:user, %{local: false, nickname: "u@peer1.com"})
-    insert(:user, %{local: false, nickname: "u@peer2.com"})
+    # Peers come from the instances table, which remote users add to.
+    insert(:instance, host: "peer1.com")
+    insert(:instance, host: "peer2.com")
+    # Favicon lookups for local accounts can add our own host.
+    insert(:instance, host: Pleroma.Web.Endpoint.host())
 
     Pleroma.Stats.force_update()