Merge pull request 'Stats: read peers from the instances table' (#8030) from lambadalambda/pleroma:perf/stats-peers-from-instances into develop

Reviewed-on: https://git.pleroma.social/pleroma/pleroma/pulls/8030

lib/pleroma/instances/instance.ex

@@ -91,6 +91,13 @@ defmodule Pleroma.Instances.Instance do
 
   def reachable?(url_or_host) when is_binary(url_or_host), do: true
 
+  @doc "Creates a record for the instance unless one exists. Existing records are kept."
+  def ensure_exists(url_or_host) when is_binary(url_or_host) do
+    %Instance{host: host(url_or_host)}
+    |> changeset(%{})
+    |> Repo.insert(on_conflict: :nothing, conflict_target: :host)
+  end
+
   def set_reachable(url_or_host) when is_binary(url_or_host) do
     host = host(url_or_host)
 

lib/pleroma/stats.ex

@@ -8,10 +8,11 @@ defmodule Pleroma.Stats do
   import Ecto.Query
 
   alias Pleroma.CounterCache
+  alias Pleroma.Instances.Instance
   alias Pleroma.Repo
   alias Pleroma.User
 
-  @interval :timer.seconds(60)
+  @interval :timer.seconds(300)
 
   def start_link(_) do
     GenServer.start_link(
@@ -64,14 +65,12 @@ defmodule Pleroma.Stats do
           }
         }
   def calculate_stat_data do
+    # Hosts are unique in instances, and remote users add theirs. Scanning
+    # all remote users for their domains was one of the costliest queries.
+    # Favicon lookups for local accounts can add our own host, so skip it.
     peers =
-      from(
-        u in User,
-        select: fragment("distinct split_part(?, '@', 2)", u.nickname),
-        where: u.local != ^true
-      )
+      from(i in Instance, where: i.host != ^local_host(), select: i.host)
       |> Repo.all()
-      |> Enum.filter(& &1)
 
     domain_count = Enum.count(peers)
 
@@ -97,6 +96,9 @@ defmodule Pleroma.Stats do
     }
   end
 
+  # Read from the config: this also runs before the endpoint has started.
+  defp local_host, do: Pleroma.Config.get([Pleroma.Web.Endpoint, :url, :host])
+
   @spec get_status_visibility_count(String.t() | nil) :: map()
   def get_status_visibility_count(instance \\ nil) do
     if is_nil(instance) do

lib/pleroma/web/activity_pub/activity_pub.ex

@@ -11,6 +11,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   alias Pleroma.Conversation.Participation
   alias Pleroma.Filter
   alias Pleroma.Hashtag
+  alias Pleroma.Instances.Instance
   alias Pleroma.Maps
   alias Pleroma.Notification
   alias Pleroma.Object
@@ -2057,6 +2058,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   def make_user_from_ap_id(ap_id, additional \\ []) do
     with {:ok, data} <- fetch_and_prepare_user_from_ap_id(ap_id, additional) do
       enqueue_pin_fetches(data)
+      # Keeps the instances table complete, so it can list our peers.
+      Instance.ensure_exists(data.ap_id)
 
       # the fetched actor's id differs from ap_id when it was fetched from a display URL
       user = User.get_cached_by_ap_id(data.ap_id)