Merge branch 'develop' of git.pleroma.social:pleroma/pleroma into websearch

lib/mix/pleroma.ex

@@ -19,6 +19,7 @@ defmodule Mix.Pleroma do
   def start_pleroma do
     Pleroma.Config.Holder.save_default()
     Pleroma.Config.Oban.warn()
+    Pleroma.Application.limiters_setup()
     Application.put_env(:phoenix, :serve_endpoints, false, persistent: true)
 
     if Pleroma.Config.get(:env) != :test do

lib/pleroma/activity.ex

@@ -356,4 +356,15 @@ defmodule Pleroma.Activity do
     actor = user_actor(activity)
     activity.id in actor.pinned_activities
   end
+
+  @spec get_by_object_ap_id_with_object(String.t()) :: t() | nil
+  def get_by_object_ap_id_with_object(ap_id) when is_binary(ap_id) do
+    ap_id
+    |> Queries.by_object_id()
+    |> with_preloaded_object()
+    |> first()
+    |> Repo.one()
+  end
+
+  def get_by_object_ap_id_with_object(_), do: nil
 end

lib/pleroma/application.ex

@@ -57,6 +57,7 @@ defmodule Pleroma.Application do
     setup_instrumenters()
     load_custom_modules()
     Pleroma.Docs.JSON.compile()
+    limiters_setup()
 
     adapter = Application.get_env(:tesla, :adapter)
 
@@ -293,4 +294,10 @@ defmodule Pleroma.Application do
   end
 
   defp http_children(_, _), do: []
+
+  @spec limiters_setup() :: :ok
+  def limiters_setup do
+    [Pleroma.Web.RichMedia.Helpers, Pleroma.Web.MediaProxy]
+    |> Enum.each(&ConcurrentLimiter.new(&1, 1, 0))
+  end
 end

lib/pleroma/emails/admin_email.ex

@@ -48,6 +48,9 @@ defmodule Pleroma.Emails.AdminEmail do
               status_url = Helpers.o_status_url(Pleroma.Web.Endpoint, :notice, id)
               "<li><a href=\"#{status_url}\">#{status_url}</li>"
 
+            %{"id" => id} when is_binary(id) ->
+              "<li><a href=\"#{id}\">#{id}</li>"
+
             id when is_binary(id) ->
               "<li><a href=\"#{id}\">#{id}</li>"
           end)

lib/pleroma/user/search.ex

@@ -85,7 +85,6 @@ defmodule Pleroma.User.Search do
     |> base_query(following)
     |> filter_blocked_user(for_user)
     |> filter_invisible_users()
-    |> filter_discoverable_users()
     |> filter_internal_users()
     |> filter_blocked_domains(for_user)
     |> fts_search(query_string)
@@ -163,10 +162,6 @@ defmodule Pleroma.User.Search do
     from(q in query, where: q.invisible == false)
   end
 
-  defp filter_discoverable_users(query) do
-    from(q in query, where: q.is_discoverable == true)
-  end
-
   defp filter_internal_users(query) do
     from(q in query, where: q.actor_type != "Application")
   end

lib/pleroma/web/activity_pub/activity_pub.ex

@@ -123,7 +123,9 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
       # Splice in the child object if we have one.
       activity = Maps.put_if_present(activity, :object, object)
 
-      BackgroundWorker.enqueue("fetch_data_for_activity", %{"activity_id" => activity.id})
+      ConcurrentLimiter.limit(Pleroma.Web.RichMedia.Helpers, fn ->
+        Task.start(fn -> Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) end)
+      end)
 
       {:ok, activity}
     else
@@ -332,15 +334,21 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   end
 
   @spec flag(map()) :: {:ok, Activity.t()} | {:error, any()}
-  def flag(
-        %{
-          actor: actor,
-          context: _context,
-          account: account,
-          statuses: statuses,
-          content: content
-        } = params
-      ) do
+  def flag(params) do
+    with {:ok, result} <- Repo.transaction(fn -> do_flag(params) end) do
+      result
+    end
+  end
+
+  defp do_flag(
+         %{
+           actor: actor,
+           context: _context,
+           account: account,
+           statuses: statuses,
+           content: content
+         } = params
+       ) do
     # only accept false as false value
     local = !(params[:local] == false)
     forward = !(params[:forward] == false)
@@ -358,7 +366,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
          {:ok, activity} <- insert(flag_data, local),
          {:ok, stripped_activity} <- strip_report_status_data(activity),
          _ <- notify_and_stream(activity),
-         :ok <- maybe_federate(stripped_activity) do
+         :ok <-
+           maybe_federate(stripped_activity) do
       User.all_superusers()
       |> Enum.filter(fn user -> not is_nil(user.email) end)
       |> Enum.each(fn superuser ->
@@ -368,6 +377,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
       end)
 
       {:ok, activity}
+    else
+      {:error, error} -> Repo.rollback(error)
     end
   end
 
@@ -791,10 +802,10 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
       where:
         fragment(
           """
-          ?->>'type' != 'Create'     -- This isn't a Create      
+          ?->>'type' != 'Create'     -- This isn't a Create
           OR ?->>'inReplyTo' is null -- this isn't a reply
-          OR ? && array_remove(?, ?) -- The recipient is us or one of our friends, 
-                                     -- unless they are the author (because authors 
+          OR ? && array_remove(?, ?) -- The recipient is us or one of our friends,
+                                     -- unless they are the author (because authors
                                      -- are also part of the recipients). This leads
                                      -- to a bug that self-replies by friends won't
                                      -- show up.

lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex

@@ -8,7 +8,6 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do
 
   alias Pleroma.HTTP
   alias Pleroma.Web.MediaProxy
-  alias Pleroma.Workers.BackgroundWorker
 
   require Logger
 
@@ -17,7 +16,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do
     recv_timeout: 10_000
   ]
 
-  def perform(:prefetch, url) do
+  defp prefetch(url) do
     # Fetching only proxiable resources
     if MediaProxy.enabled?() and MediaProxy.url_proxiable?(url) do
       # If preview proxy is enabled, it'll also hit media proxy (so we're caching both requests)
@@ -25,17 +24,25 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do
 
       Logger.debug("Prefetching #{inspect(url)} as #{inspect(prefetch_url)}")
 
-      HTTP.get(prefetch_url, [], @adapter_options)
+      if Pleroma.Config.get(:env) == :test do
+        fetch(prefetch_url)
+      else
+        ConcurrentLimiter.limit(MediaProxy, fn ->
+          Task.start(fn -> fetch(prefetch_url) end)
+        end)
+      end
     end
   end
 
-  def perform(:preload, %{"object" => %{"attachment" => attachments}} = _message) do
+  defp fetch(url), do: HTTP.get(url, [], @adapter_options)
+
+  defp preload(%{"object" => %{"attachment" => attachments}} = _message) do
     Enum.each(attachments, fn
       %{"url" => url} when is_list(url) ->
         url
         |> Enum.each(fn
           %{"href" => href} ->
-            BackgroundWorker.enqueue("media_proxy_prefetch", %{"url" => href})
+            prefetch(href)
 
           x ->
             Logger.debug("Unhandled attachment URL object #{inspect(x)}")
@@ -51,7 +58,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do
         %{"type" => "Create", "object" => %{"attachment" => attachments} = _object} = message
       )
       when is_list(attachments) and length(attachments) > 0 do
-    BackgroundWorker.enqueue("media_proxy_preload", %{"message" => message})
+    preload(message)
 
     {:ok, message}
   end

lib/pleroma/web/activity_pub/side_effects.ex

@@ -24,7 +24,6 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
   alias Pleroma.Web.ActivityPub.Utils
   alias Pleroma.Web.Push
   alias Pleroma.Web.Streamer
-  alias Pleroma.Workers.BackgroundWorker
 
   require Logger
 
@@ -191,7 +190,9 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
         Object.increase_replies_count(in_reply_to)
       end
 
-      BackgroundWorker.enqueue("fetch_data_for_activity", %{"activity_id" => activity.id})
+      ConcurrentLimiter.limit(Pleroma.Web.RichMedia.Helpers, fn ->
+        Task.start(fn -> Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) end)
+      end)
 
       meta =
         meta

lib/pleroma/web/activity_pub/utils.ex

@@ -702,14 +702,30 @@ defmodule Pleroma.Web.ActivityPub.Utils do
 
   def make_flag_data(_, _), do: %{}
 
-  defp build_flag_object(%{account: account, statuses: statuses} = _) do
-    [account.ap_id] ++ build_flag_object(%{statuses: statuses})
+  defp build_flag_object(%{account: account, statuses: statuses}) do
+    [account.ap_id | build_flag_object(%{statuses: statuses})]
   end
 
   defp build_flag_object(%{statuses: statuses}) do
     Enum.map(statuses || [], &build_flag_object/1)
   end
 
+  defp build_flag_object(%Activity{data: %{"id" => id}, object: %{data: data}}) do
+    activity_actor = User.get_by_ap_id(data["actor"])
+
+    %{
+      "type" => "Note",
+      "id" => id,
+      "content" => data["content"],
+      "published" => data["published"],
+      "actor" =>
+        AccountView.render(
+          "show.json",
+          %{user: activity_actor, skip_visibility_check: true}
+        )
+    }
+  end
+
   defp build_flag_object(act) when is_map(act) or is_binary(act) do
     id =
       case act do
@@ -720,22 +736,14 @@ defmodule Pleroma.Web.ActivityPub.Utils do
 
     case Activity.get_by_ap_id_with_object(id) do
       %Activity{} = activity ->
-        activity_actor = User.get_by_ap_id(activity.object.data["actor"])
+        build_flag_object(activity)
 
-        %{
-          "type" => "Note",
-          "id" => activity.data["id"],
-          "content" => activity.object.data["content"],
-          "published" => activity.object.data["published"],
-          "actor" =>
-            AccountView.render(
-              "show.json",
-              %{user: activity_actor, skip_visibility_check: true}
-            )
-        }
-
-      _ ->
-        %{"id" => id, "deleted" => true}
+      nil ->
+        if activity = Activity.get_by_object_ap_id_with_object(id) do
+          build_flag_object(activity)
+        else
+          %{"id" => id, "deleted" => true}
+        end
     end
   end
 

lib/pleroma/web/activity_pub/views/user_view.ex

@@ -110,6 +110,7 @@ defmodule Pleroma.Web.ActivityPub.UserView do
       "endpoints" => endpoints,
       "attachment" => fields,
       "tag" => emoji_tags,
+      # Note: key name is indeed "discoverable" (not an error)
       "discoverable" => user.is_discoverable,
       "capabilities" => capabilities
     }

lib/pleroma/web/api_spec/operations/account_operation.ex

@@ -624,7 +624,7 @@ defmodule Pleroma.Web.ApiSpec.AccountOperation do
           allOf: [BooleanLike],
           nullable: true,
           description:
-            "Discovery of this account in search results and other services is allowed."
+            "Discovery (listing, indexing) of this account by external services (search bots etc.) is allowed."
         },
         actor_type: ActorType
       },

lib/pleroma/web/api_spec/schemas/account.ex

@@ -127,7 +127,7 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Account do
               discoverable: %Schema{
                 type: :boolean,
                 description:
-                  "whether the user allows discovery of the account in search results and other services."
+                  "whether the user allows indexing / listing of the account by external services (search engines etc.)."
               },
               no_rich_text: %Schema{
                 type: :boolean,

lib/pleroma/web/mastodon_api/controllers/account_controller.ex

@@ -208,7 +208,9 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do
         if bot, do: {:ok, "Service"}, else: {:ok, "Person"}
       end)
       |> Maps.put_if_present(:actor_type, params[:actor_type])
+      # Note: param name is indeed :locked (not an error)
       |> Maps.put_if_present(:is_locked, params[:locked])
+      # Note: param name is indeed :discoverable (not an error)
       |> Maps.put_if_present(:is_discoverable, params[:discoverable])
 
     # What happens here:

lib/pleroma/web/metadata/providers/restrict_indexing.ex

@@ -6,7 +6,7 @@ defmodule Pleroma.Web.Metadata.Providers.RestrictIndexing do
   @behaviour Pleroma.Web.Metadata.Providers.Provider
 
   @moduledoc """
-  Restricts indexing of remote users.
+  Restricts indexing of remote and/or non-discoverable users.
   """
 
   @impl true

lib/pleroma/web/rich_media/helpers.ex

@@ -78,11 +78,6 @@ defmodule Pleroma.Web.RichMedia.Helpers do
 
   def fetch_data_for_activity(_), do: %{}
 
-  def perform(:fetch, %Activity{} = activity) do
-    fetch_data_for_activity(activity)
-    :ok
-  end
-
   def rich_media_get(url) do
     headers = [{"user-agent", Pleroma.Application.user_agent() <> "; Bot"}]
 

lib/pleroma/workers/background_worker.ex

@@ -3,9 +3,7 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Workers.BackgroundWorker do
-  alias Pleroma.Activity
   alias Pleroma.User
-  alias Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy
 
   use Pleroma.Workers.WorkerHelper, queue: "background"
 
@@ -32,19 +30,6 @@ defmodule Pleroma.Workers.BackgroundWorker do
     {:ok, User.Import.perform(String.to_atom(op), user, identifiers)}
   end
 
-  def perform(%Job{args: %{"op" => "media_proxy_preload", "message" => message}}) do
-    MediaProxyWarmingPolicy.perform(:preload, message)
-  end
-
-  def perform(%Job{args: %{"op" => "media_proxy_prefetch", "url" => url}}) do
-    MediaProxyWarmingPolicy.perform(:prefetch, url)
-  end
-
-  def perform(%Job{args: %{"op" => "fetch_data_for_activity", "activity_id" => activity_id}}) do
-    activity = Activity.get_by_id(activity_id)
-    Pleroma.Web.RichMedia.Helpers.perform(:fetch, activity)
-  end
-
   def perform(%Job{
         args: %{"op" => "move_following", "origin_id" => origin_id, "target_id" => target_id}
       }) do