Merge branch 'develop' into gun

lib/pleroma/web/activity_pub/activity_pub.ex

@@ -770,13 +770,18 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     |> Enum.reverse()
   end
 
-  def fetch_instance_activities(params) do
+  def fetch_statuses(reading_user, params) do
     params =
       params
       |> Map.put("type", ["Create", "Announce"])
-      |> Map.put("instance", params["instance"])
 
-    fetch_activities([Pleroma.Constants.as_public()], params, :offset)
+    recipients =
+      user_activities_recipients(%{
+        "godmode" => params["godmode"],
+        "reading_user" => reading_user
+      })
+
+    fetch_activities(recipients, params, :offset)
     |> Enum.reverse()
   end
 

lib/pleroma/web/activity_pub/transmogrifier.ex

@@ -156,10 +156,11 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
       when not is_nil(in_reply_to) do
     in_reply_to_id = prepare_in_reply_to(in_reply_to)
     object = Map.put(object, "inReplyToAtomUri", in_reply_to_id)
+    depth = (options[:depth] || 0) + 1
 
-    if Federator.allowed_incoming_reply_depth?(options[:depth]) do
+    if Federator.allowed_thread_distance?(depth) do
       with {:ok, replied_object} <- get_obj_helper(in_reply_to_id, options),
-           %Activity{} = _ <- Activity.get_create_by_object_ap_id(replied_object.data["id"]) do
+           %Activity{} <- Activity.get_create_by_object_ap_id(replied_object.data["id"]) do
         object
         |> Map.put("inReplyTo", replied_object.data["id"])
         |> Map.put("inReplyToAtomUri", object["inReplyToAtomUri"] || in_reply_to_id)
@@ -312,7 +313,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
 
   def fix_type(%{"inReplyTo" => reply_id, "name" => _} = object, options)
       when is_binary(reply_id) do
-    with true <- Federator.allowed_incoming_reply_depth?(options[:depth]),
+    with true <- Federator.allowed_thread_distance?(options[:depth]),
          {:ok, %{data: %{"type" => "Question"} = _} = _} <- get_obj_helper(reply_id, options) do
       Map.put(object, "type", "Answer")
     else
@@ -406,8 +407,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
 
     with nil <- Activity.get_create_by_object_ap_id(object["id"]),
          {:ok, %User{} = user} <- User.get_or_fetch_by_ap_id(data["actor"]) do
-      options = Keyword.put(options, :depth, (options[:depth] || 0) + 1)
-      object = fix_object(data["object"], options)
+      object = fix_object(object, options)
 
       params = %{
         to: data["to"],
@@ -424,7 +424,20 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
           ])
       }
 
-      ActivityPub.create(params)
+      with {:ok, created_activity} <- ActivityPub.create(params) do
+        reply_depth = (options[:depth] || 0) + 1
+
+        if Federator.allowed_thread_distance?(reply_depth) do
+          for reply_id <- replies(object) do
+            Pleroma.Workers.RemoteFetcherWorker.enqueue("fetch_remote", %{
+              "id" => reply_id,
+              "depth" => reply_depth
+            })
+          end
+        end
+
+        {:ok, created_activity}
+      end
     else
       %Activity{} = activity -> {:ok, activity}
       _e -> :error
@@ -442,7 +455,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
       |> fix_addressing
 
     with {:ok, %User{} = user} <- User.get_or_fetch_by_ap_id(data["actor"]) do
-      options = Keyword.put(options, :depth, (options[:depth] || 0) + 1)
+      reply_depth = (options[:depth] || 0) + 1
+      options = Keyword.put(options, :depth, reply_depth)
       object = fix_object(object, options)
 
       params = %{
@@ -903,6 +917,50 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
 
   def set_reply_to_uri(obj), do: obj
 
+  @doc """
+  Serialized Mastodon-compatible `replies` collection containing _self-replies_.
+  Based on Mastodon's ActivityPub::NoteSerializer#replies.
+  """
+  def set_replies(obj_data) do
+    replies_uris =
+      with limit when limit > 0 <-
+             Pleroma.Config.get([:activitypub, :note_replies_output_limit], 0),
+           %Object{} = object <- Object.get_cached_by_ap_id(obj_data["id"]) do
+        object
+        |> Object.self_replies()
+        |> select([o], fragment("?->>'id'", o.data))
+        |> limit(^limit)
+        |> Repo.all()
+      else
+        _ -> []
+      end
+
+    set_replies(obj_data, replies_uris)
+  end
+
+  defp set_replies(obj, []) do
+    obj
+  end
+
+  defp set_replies(obj, replies_uris) do
+    replies_collection = %{
+      "type" => "Collection",
+      "items" => replies_uris
+    }
+
+    Map.merge(obj, %{"replies" => replies_collection})
+  end
+
+  def replies(%{"replies" => %{"first" => %{"items" => items}}}) when not is_nil(items) do
+    items
+  end
+
+  def replies(%{"replies" => %{"items" => items}}) when not is_nil(items) do
+    items
+  end
+
+  def replies(_), do: []
+
   # Prepares the object of an outgoing create activity.
   def prepare_object(object) do
     object
@@ -914,6 +972,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
     |> prepare_attachments
     |> set_conversation
     |> set_reply_to_uri
+    |> set_replies
     |> strip_internal_fields
     |> strip_internal_tags
     |> set_type

lib/pleroma/web/activity_pub/utils.ex

@@ -45,8 +45,8 @@ defmodule Pleroma.Web.ActivityPub.Utils do
     Map.put(params, "actor", get_ap_id(params["actor"]))
   end
 
-  @spec determine_explicit_mentions(map()) :: map()
-  def determine_explicit_mentions(%{"tag" => tag} = _) when is_list(tag) do
+  @spec determine_explicit_mentions(map()) :: [any]
+  def determine_explicit_mentions(%{"tag" => tag}) when is_list(tag) do
     Enum.flat_map(tag, fn
       %{"type" => "Mention", "href" => href} -> [href]
       _ -> []
@@ -427,7 +427,7 @@ defmodule Pleroma.Web.ActivityPub.Utils do
   @doc """
   Updates a follow activity's state (for locked accounts).
   """
-  @spec update_follow_state_for_all(Activity.t(), String.t()) :: {:ok, Activity} | {:error, any()}
+  @spec update_follow_state_for_all(Activity.t(), String.t()) :: {:ok, Activity | nil}
   def update_follow_state_for_all(
         %Activity{data: %{"actor" => actor, "object" => object}} = activity,
         state

lib/pleroma/web/admin_api/admin_api_controller.ex

@@ -13,6 +13,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
   alias Pleroma.ModerationLog
   alias Pleroma.Plugs.OAuthScopesPlug
   alias Pleroma.ReportNote
+  alias Pleroma.Stats
   alias Pleroma.User
   alias Pleroma.UserInviteToken
   alias Pleroma.Web.ActivityPub.ActivityPub
@@ -98,7 +99,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
   plug(
     OAuthScopesPlug,
     %{scopes: ["read"], admin: true}
-    when action in [:config_show, :list_log]
+    when action in [:config_show, :list_log, :stats]
   )
 
   plug(
@@ -243,13 +244,15 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
   end
 
   def list_instance_statuses(conn, %{"instance" => instance} = params) do
+    with_reblogs = params["with_reblogs"] == "true" || params["with_reblogs"] == true
     {page, page_size} = page_params(params)
 
     activities =
-      ActivityPub.fetch_instance_activities(%{
+      ActivityPub.fetch_statuses(nil, %{
         "instance" => instance,
         "limit" => page_size,
-        "offset" => (page - 1) * page_size
+        "offset" => (page - 1) * page_size,
+        "exclude_reblogs" => !with_reblogs && "true"
       })
 
     conn
@@ -258,6 +261,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
   end
 
   def list_user_statuses(conn, %{"nickname" => nickname} = params) do
+    with_reblogs = params["with_reblogs"] == "true" || params["with_reblogs"] == true
     godmode = params["godmode"] == "true" || params["godmode"] == true
 
     with %User{} = user <- User.get_cached_by_nickname_or_id(nickname) do
@@ -266,7 +270,8 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
       activities =
         ActivityPub.fetch_user_activities(user, nil, %{
           "limit" => page_size,
-          "godmode" => godmode
+          "godmode" => godmode,
+          "exclude_reblogs" => !with_reblogs && "true"
         })
 
       conn
@@ -740,6 +745,24 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
     end
   end
 
+  def list_statuses(%{assigns: %{user: admin}} = conn, params) do
+    godmode = params["godmode"] == "true" || params["godmode"] == true
+    local_only = params["local_only"] == "true" || params["local_only"] == true
+    {page, page_size} = page_params(params)
+
+    activities =
+      ActivityPub.fetch_statuses(admin, %{
+        "godmode" => godmode,
+        "local_only" => local_only,
+        "limit" => page_size,
+        "offset" => (page - 1) * page_size
+      })
+
+    conn
+    |> put_view(Pleroma.Web.AdminAPI.StatusView)
+    |> render("index.json", %{activities: activities, as: :activity})
+  end
+
   def status_update(%{assigns: %{user: admin}} = conn, %{"id" => id} = params) do
     with {:ok, activity} <- CommonAPI.update_activity_scope(id, params) do
       {:ok, sensitive} = Ecto.Type.cast(:boolean, params["sensitive"])
@@ -953,6 +976,13 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
     conn |> json("")
   end
 
+  def stats(conn, _) do
+    count = Stats.get_status_visibility_count()
+
+    conn
+    |> json(%{"status_visibility" => count})
+  end
+
   def errors(conn, {:error, :not_found}) do
     conn
     |> put_status(:not_found)

lib/pleroma/web/admin_api/search.ex

@@ -18,7 +18,11 @@ defmodule Pleroma.Web.AdminAPI.Search do
 
   @spec user(map()) :: {:ok, [User.t()], pos_integer()}
   def user(params \\ %{}) do
-    query = User.Query.build(params) |> order_by([u], u.nickname)
+    query =
+      params
+      |> Map.drop([:page, :page_size])
+      |> User.Query.build()
+      |> order_by([u], u.nickname)
 
     paginated_query =
       User.Query.paginate(query, params[:page] || 1, params[:page_size] || @page_size)

lib/pleroma/web/admin_api/views/status_view.ex

@@ -10,7 +10,7 @@ defmodule Pleroma.Web.AdminAPI.StatusView do
   alias Pleroma.User
 
   def render("index.json", opts) do
-    render_many(opts.activities, __MODULE__, "show.json", opts)
+    safe_render_many(opts.activities, __MODULE__, "show.json", opts)
   end
 
   def render("show.json", %{activity: %{data: %{"object" => _object}} = activity} = opts) do

lib/pleroma/web/federator/federator.ex

@@ -15,13 +15,19 @@ defmodule Pleroma.Web.Federator do
 
   require Logger
 
-  @doc "Addresses [memory leaks on recursive replies fetching](https://git.pleroma.social/pleroma/pleroma/issues/161)"
+  @doc """
+  Returns `true` if the distance to target object does not exceed max configured value.
+  Serves to prevent fetching of very long threads, especially useful on smaller instances.
+  Addresses [memory leaks on recursive replies fetching](https://git.pleroma.social/pleroma/pleroma/issues/161).
+  Applies to fetching of both ancestor (reply-to) and child (reply) objects.
+  """
   # credo:disable-for-previous-line Credo.Check.Readability.MaxLineLength
-  def allowed_incoming_reply_depth?(depth) do
-    max_replies_depth = Pleroma.Config.get([:instance, :federation_incoming_replies_max_depth])
+  def allowed_thread_distance?(distance) do
+    max_distance = Pleroma.Config.get([:instance, :federation_incoming_replies_max_depth])
 
-    if max_replies_depth do
-      (depth || 1) <= max_replies_depth
+    if max_distance && max_distance >= 0 do
+      # Default depth is 0 (an object has zero distance from itself in its thread)
+      (distance || 0) <= max_distance
     else
       true
     end

lib/pleroma/web/nodeinfo/nodeinfo_controller.ex

@@ -92,9 +92,9 @@ defmodule Pleroma.Web.Nodeinfo.NodeinfoController do
       openRegistrations: Config.get([:instance, :registrations_open]),
       usage: %{
         users: %{
-          total: stats.user_count || 0
+          total: Map.get(stats, :user_count, 0)
         },
-        localPosts: stats.status_count || 0
+        localPosts: Map.get(stats, :status_count, 0)
       },
       metadata: %{
         nodeName: Config.get([:instance, :name]),

lib/pleroma/web/pleroma_api/controllers/emoji_api_controller.ex

@@ -323,7 +323,7 @@ keeping it in cache for #{div(cache_ms, 1000)}s")
       {:ok, _} ->
         conn |> json("ok")
 
-      {:error, _} ->
+      {:error, _, _} ->
         conn
         |> put_status(:internal_server_error)
         |> json(%{error: "Couldn't delete the pack #{name}"})

lib/pleroma/web/router.ex

@@ -192,6 +192,7 @@ defmodule Pleroma.Web.Router do
 
     put("/statuses/:id", AdminAPIController, :status_update)
     delete("/statuses/:id", AdminAPIController, :status_delete)
+    get("/statuses", AdminAPIController, :list_statuses)
 
     get("/config", AdminAPIController, :config_show)
     post("/config", AdminAPIController, :config_update)
@@ -201,6 +202,7 @@ defmodule Pleroma.Web.Router do
     get("/moderation_log", AdminAPIController, :list_log)
 
     post("/reload_emoji", AdminAPIController, :reload_emoji)
+    get("/stats", AdminAPIController, :stats)
   end
 
   scope "/api/pleroma/emoji", Pleroma.Web.PleromaAPI do