Merge branch 'develop' into 'bookmark-folders'

# Conflicts:
#   docs/development/API/differences_in_mastoapi_responses.md

lib/pleroma/following_relationship.ex

@@ -241,13 +241,13 @@ defmodule Pleroma.FollowingRelationship do
   end
 
   @doc """
-  For a query with joined activity,
-  keeps rows where activity's actor is followed by user -or- is NOT domain-blocked by user.
+  For a query with joined activity's actor,
+  keeps rows where actor is followed by user -or- is NOT domain-blocked by user.
   """
   def keep_following_or_not_domain_blocked(query, user) do
     where(
       query,
-      [_, activity],
+      [_, user_actor: user_actor],
       fragment(
         # "(actor's domain NOT in domain_blocks) OR (actor IS in followed AP IDs)"
         """
@@ -255,9 +255,9 @@ defmodule Pleroma.FollowingRelationship do
           ? = ANY(SELECT ap_id FROM users AS u INNER JOIN following_relationships AS fr
             ON u.id = fr.following_id WHERE fr.follower_id = ? AND fr.state = ?)
         """,
-        activity.actor,
+        user_actor.ap_id,
         ^user.domain_blocks,
-        activity.actor,
+        user_actor.ap_id,
         ^User.binary_id(user.id),
         ^accept_state_code()
       )

lib/pleroma/gun/connection_pool/worker_supervisor.ex

@@ -18,10 +18,10 @@ defmodule Pleroma.Gun.ConnectionPool.WorkerSupervisor do
     )
   end
 
-  def start_worker(opts, retry \\ false) do
+  def start_worker(opts, last_attempt \\ false) do
     case DynamicSupervisor.start_child(__MODULE__, {Pleroma.Gun.ConnectionPool.Worker, opts}) do
       {:error, :max_children} ->
-        funs = [fn -> !retry end, fn -> match?(:error, free_pool()) end]
+        funs = [fn -> last_attempt end, fn -> match?(:error, free_pool()) end]
 
         if Enum.any?(funs, fn fun -> fun.() end) do
           :telemetry.execute([:pleroma, :connection_pool, :provision_failure], %{opts: opts})

lib/pleroma/helpers/media_helper.ex

@@ -40,28 +40,32 @@ defmodule Pleroma.Helpers.MediaHelper do
   end
 
   # Note: video thumbnail is intentionally not resized (always has original dimensions)
+  @spec video_framegrab(String.t()) :: {:ok, binary()} | {:error, any()}
   def video_framegrab(url) do
     with executable when is_binary(executable) <- System.find_executable("ffmpeg"),
          {:ok, env} <- HTTP.get(url, [], pool: :media),
          {:ok, pid} <- StringIO.open(env.body) do
       body_stream = IO.binstream(pid, 1)
 
-      Exile.stream!(
-        [
-          executable,
-          "-i",
-          "pipe:0",
-          "-vframes",
-          "1",
-          "-f",
-          "mjpeg",
-          "pipe:1"
-        ],
-        input: body_stream,
-        ignore_epipe: true,
-        stderr: :disable
-      )
-      |> Enum.into(<<>>)
+      result =
+        Exile.stream!(
+          [
+            executable,
+            "-i",
+            "pipe:0",
+            "-vframes",
+            "1",
+            "-f",
+            "mjpeg",
+            "pipe:1"
+          ],
+          input: body_stream,
+          ignore_epipe: true,
+          stderr: :disable
+        )
+        |> Enum.into(<<>>)
+
+      {:ok, result}
     else
       nil -> {:error, {:ffmpeg, :command_not_found}}
       {:error, _} = error -> error

lib/pleroma/notification.ex

@@ -137,7 +137,7 @@ defmodule Pleroma.Notification do
     blocked_ap_ids = opts[:blocked_users_ap_ids] || User.blocked_users_ap_ids(user)
 
     query
-    |> where([n, a], a.actor not in ^blocked_ap_ids)
+    |> where([..., user_actor: user_actor], user_actor.ap_id not in ^blocked_ap_ids)
     |> FollowingRelationship.keep_following_or_not_domain_blocked(user)
   end
 
@@ -148,7 +148,7 @@ defmodule Pleroma.Notification do
       blocker_ap_ids = User.incoming_relationships_ungrouped_ap_ids(user, [:block])
 
       query
-      |> where([n, a], a.actor not in ^blocker_ap_ids)
+      |> where([..., user_actor: user_actor], user_actor.ap_id not in ^blocker_ap_ids)
     end
   end
 
@@ -161,7 +161,7 @@ defmodule Pleroma.Notification do
       opts[:notification_muted_users_ap_ids] || User.notification_muted_users_ap_ids(user)
 
     query
-    |> where([n, a], a.actor not in ^notification_muted_ap_ids)
+    |> where([..., user_actor: user_actor], user_actor.ap_id not in ^notification_muted_ap_ids)
     |> join(:left, [n, a], tm in ThreadMute,
       on: tm.user_id == ^user.id and tm.context == fragment("?->>'context'", a.data),
       as: :thread_mute

lib/pleroma/user.ex

@@ -8,6 +8,7 @@ defmodule Pleroma.User do
   import Ecto.Changeset
   import Ecto.Query
   import Ecto, only: [assoc: 2]
+  import Pleroma.Web.Utils.Guards, only: [not_empty_string: 1]
 
   alias Ecto.Multi
   alias Pleroma.Activity
@@ -596,9 +597,23 @@ defmodule Pleroma.User do
 
   defp put_fields(changeset) do
     if raw_fields = get_change(changeset, :raw_fields) do
+      old_fields = changeset.data.raw_fields
+
       raw_fields =
         raw_fields
         |> Enum.filter(fn %{"name" => n} -> n != "" end)
+        |> Enum.map(fn field ->
+          previous =
+            old_fields
+            |> Enum.find(fn %{"value" => value} -> field["value"] == value end)
+
+          if previous && Map.has_key?(previous, "verified_at") do
+            field
+            |> Map.put("verified_at", previous["verified_at"])
+          else
+            field
+          end
+        end)
 
       fields =
         raw_fields
@@ -1200,6 +1215,10 @@ defmodule Pleroma.User do
 
   def update_and_set_cache(changeset) do
     with {:ok, user} <- Repo.update(changeset, stale_error_field: :id) do
+      if get_change(changeset, :raw_fields) do
+        BackgroundWorker.enqueue("verify_fields_links", %{"user_id" => user.id})
+      end
+
       set_cache(user)
     end
   end
@@ -1975,8 +1994,45 @@ defmodule Pleroma.User do
     maybe_delete_from_db(user)
   end
 
+  def perform(:verify_fields_links, user) do
+    profile_urls = [user.ap_id]
+
+    fields =
+      user.raw_fields
+      |> Enum.map(&verify_field_link(&1, profile_urls))
+
+    changeset =
+      user
+      |> update_changeset(%{raw_fields: fields})
+
+    with {:ok, user} <- Repo.update(changeset, stale_error_field: :id) do
+      set_cache(user)
+    end
+  end
+
   def perform(:set_activation_async, user, status), do: set_activation(user, status)
 
+  defp verify_field_link(field, profile_urls) do
+    verified_at =
+      with %{"value" => value} <- field,
+           {:verified_at, nil} <- {:verified_at, Map.get(field, "verified_at")},
+           %{scheme: scheme, userinfo: nil, host: host}
+           when not_empty_string(host) and scheme in ["http", "https"] <-
+             URI.parse(value),
+           {:not_idn, true} <- {:not_idn, to_string(:idna.encode(host)) == host},
+           "me" <- Pleroma.Web.RelMe.maybe_put_rel_me(value, profile_urls) do
+        CommonUtils.to_masto_date(NaiveDateTime.utc_now())
+      else
+        {:verified_at, value} when not_empty_string(value) ->
+          value
+
+        _ ->
+          nil
+      end
+
+    Map.put(field, "verified_at", verified_at)
+  end
+
   @spec external_users_query() :: Ecto.Query.t()
   def external_users_query do
     User.Query.build(%{
@@ -2664,10 +2720,11 @@ defmodule Pleroma.User do
   # - display name
   def sanitize_html(%User{} = user, filter) do
     fields =
-      Enum.map(user.fields, fn %{"name" => name, "value" => value} ->
+      Enum.map(user.fields, fn %{"name" => name, "value" => value} = fields ->
         %{
           "name" => name,
-          "value" => HTML.filter_tags(value, Pleroma.HTML.Scrubber.LinksOnly)
+          "value" => HTML.filter_tags(value, Pleroma.HTML.Scrubber.LinksOnly),
+          "verified_at" => Map.get(fields, "verified_at")
         }
       end)
 

lib/pleroma/user/backup.ex

@@ -196,7 +196,14 @@ defmodule Pleroma.User.Backup do
     end
   end
 
-  @files ['actor.json', 'outbox.json', 'likes.json', 'bookmarks.json']
+  @files [
+    'actor.json',
+    'outbox.json',
+    'likes.json',
+    'bookmarks.json',
+    'followers.json',
+    'following.json'
+  ]
   @spec export(Pleroma.User.Backup.t(), pid()) :: {:ok, String.t()} | :error
   def export(%__MODULE__{} = backup, caller_pid) do
     backup = Repo.preload(backup, :user)
@@ -207,6 +214,8 @@ defmodule Pleroma.User.Backup do
          :ok <- statuses(dir, backup.user, caller_pid),
          :ok <- likes(dir, backup.user, caller_pid),
          :ok <- bookmarks(dir, backup.user, caller_pid),
+         :ok <- followers(dir, backup.user, caller_pid),
+         :ok <- following(dir, backup.user, caller_pid),
          {:ok, zip_path} <- :zip.create(backup.file_name, @files, cwd: dir),
          {:ok, _} <- File.rm_rf(dir) do
       {:ok, zip_path}
@@ -357,6 +366,16 @@ defmodule Pleroma.User.Backup do
       caller_pid
     )
   end
+
+  defp followers(dir, user, caller_pid) do
+    User.get_followers_query(user)
+    |> write(dir, "followers", fn a -> {:ok, a.ap_id} end, caller_pid)
+  end
+
+  defp following(dir, user, caller_pid) do
+    User.get_friends_query(user)
+    |> write(dir, "following", fn a -> {:ok, a.ap_id} end, caller_pid)
+  end
 end
 
 defmodule Pleroma.User.Backup.ProcessorAPI do

lib/pleroma/web/activity_pub/mrf/force_mention.ex

@@ -0,0 +1,59 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2024 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.MRF.ForceMention do
+  require Pleroma.Constants
+
+  alias Pleroma.Config
+  alias Pleroma.Object
+  alias Pleroma.User
+
+  @behaviour Pleroma.Web.ActivityPub.MRF.Policy
+
+  defp get_author(url) do
+    with %Object{data: %{"actor" => actor}} <- Object.normalize(url, fetch: false),
+         %User{ap_id: ap_id, nickname: nickname} <- User.get_cached_by_ap_id(actor) do
+      %{"type" => "Mention", "href" => ap_id, "name" => "@#{nickname}"}
+    else
+      _ -> nil
+    end
+  end
+
+  defp prepend_author(tags, _, false), do: tags
+
+  defp prepend_author(tags, nil, _), do: tags
+
+  defp prepend_author(tags, url, _) do
+    actor = get_author(url)
+
+    if not is_nil(actor) do
+      [actor | tags]
+    else
+      tags
+    end
+  end
+
+  @impl true
+  def filter(%{"type" => "Create", "object" => %{"tag" => tag} = object} = activity) do
+    tag =
+      tag
+      |> prepend_author(
+        object["inReplyTo"],
+        Config.get([:mrf_force_mention, :mention_parent, true])
+      )
+      |> prepend_author(
+        object["quoteUrl"],
+        Config.get([:mrf_force_mention, :mention_quoted, true])
+      )
+      |> Enum.uniq()
+
+    {:ok, put_in(activity["object"]["tag"], tag)}
+  end
+
+  @impl true
+  def filter(object), do: {:ok, object}
+
+  @impl true
+  def describe, do: {:ok, %{}}
+end

lib/pleroma/web/activity_pub/object_validators/question_validator.ex

@@ -29,6 +29,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do
 
     field(:closed, ObjectValidators.DateTime)
     field(:voters, {:array, ObjectValidators.ObjectID}, default: [])
+    field(:nonAnonymous, :boolean)
     embeds_many(:anyOf, QuestionOptionsValidator)
     embeds_many(:oneOf, QuestionOptionsValidator)
   end

lib/pleroma/web/activity_pub/publisher.ex

@@ -129,6 +129,10 @@ defmodule Pleroma.Web.ActivityPub.Publisher do
           _ -> {:error, e}
         end
 
+      {:error, :pool_full} ->
+        Logger.debug("Publisher snoozing worker job due to full connection pool")
+        {:snooze, 30}
+
       e ->
         unless params[:unreachable_since], do: Instances.set_unreachable(inbox)
         Logger.metadata(activity: id, inbox: inbox)
@@ -154,19 +158,18 @@ defmodule Pleroma.Web.ActivityPub.Publisher do
     end
   end
 
-  defp should_federate?(inbox, public) do
-    if public do
-      true
-    else
-      %{host: host} = URI.parse(inbox)
+  def should_federate?(nil, _), do: false
+  def should_federate?(_, true), do: true
 
-      quarantined_instances =
-        Config.get([:instance, :quarantined_instances], [])
-        |> Pleroma.Web.ActivityPub.MRF.instance_list_from_tuples()
-        |> Pleroma.Web.ActivityPub.MRF.subdomains_regex()
+  def should_federate?(inbox, _) do
+    %{host: host} = URI.parse(inbox)
 
-      !Pleroma.Web.ActivityPub.MRF.subdomain_match?(quarantined_instances, host)
-    end
+    quarantined_instances =
+      Config.get([:instance, :quarantined_instances], [])
+      |> Pleroma.Web.ActivityPub.MRF.instance_list_from_tuples()
+      |> Pleroma.Web.ActivityPub.MRF.subdomains_regex()
+
+    !Pleroma.Web.ActivityPub.MRF.subdomain_match?(quarantined_instances, host)
   end
 
   @spec recipients(User.t(), Activity.t()) :: [[User.t()]]

lib/pleroma/web/api_spec/schemas/poll.ex

@@ -56,6 +56,15 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Poll do
           }
         },
         description: "Possible answers for the poll."
+      },
+      pleroma: %Schema{
+        type: :object,
+        properties: %{
+          non_anonymous: %Schema{
+            type: :boolean,
+            description: "Can voters be publicly identified?"
+          }
+        }
       }
     },
     example: %{
@@ -79,7 +88,10 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Poll do
           votes_count: 4
         }
       ],
-      emojis: []
+      emojis: [],
+      pleroma: %{
+        non_anonymous: false
+      }
     }
   })
 end

lib/pleroma/web/mastodon_api/views/instance_view.ex

@@ -28,6 +28,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do
         |> to_string,
       registrations: Keyword.get(instance, :registrations_open),
       approval_required: Keyword.get(instance, :account_approval_required),
+      contact_account: contact_account(Keyword.get(instance, :contact_username)),
       configuration: configuration(),
       # Extra (not present in Mastodon):
       max_toot_chars: Keyword.get(instance, :limit),
@@ -63,11 +64,12 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do
       registrations: %{
         enabled: Keyword.get(instance, :registrations_open),
         approval_required: Keyword.get(instance, :account_approval_required),
-        message: nil
+        message: nil,
+        url: nil
       },
       contact: %{
         email: Keyword.get(instance, :email),
-        account: nil
+        account: contact_account(Keyword.get(instance, :contact_username))
       },
       # Extra (not present in Mastodon):
       pleroma: pleroma_configuration2(instance)
@@ -78,7 +80,8 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do
     %{
       title: Keyword.get(instance, :name),
       version: "#{@mastodon_api_level} (compatible; #{Pleroma.Application.named_version()})",
-      languages: Keyword.get(instance, :languages, ["en"])
+      languages: Keyword.get(instance, :languages, ["en"]),
+      rules: []
     }
   end
 
@@ -168,15 +171,35 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do
     }
   end
 
+  defp contact_account(nil), do: nil
+
+  defp contact_account("@" <> username) do
+    contact_account(username)
+  end
+
+  defp contact_account(username) do
+    user = Pleroma.User.get_cached_by_nickname(username)
+
+    if user do
+      Pleroma.Web.MastodonAPI.AccountView.render("show.json", %{user: user, for: nil})
+    else
+      nil
+    end
+  end
+
   defp configuration do
     %{
+      accounts: %{
+        max_featured_tags: 0
+      },
       statuses: %{
         max_characters: Config.get([:instance, :limit]),
         max_media_attachments: Config.get([:instance, :max_media_attachments])
       },
       media_attachments: %{
         image_size_limit: Config.get([:instance, :upload_limit]),
-        video_size_limit: Config.get([:instance, :upload_limit])
+        video_size_limit: Config.get([:instance, :upload_limit]),
+        supported_mime_types: ["application/octet-stream"]
       },
       polls: %{
         max_options: Config.get([:instance, :poll_limits, :max_options]),
@@ -190,7 +213,13 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do
   defp configuration2 do
     configuration()
     |> Map.merge(%{
-      urls: %{streaming: Pleroma.Web.Endpoint.websocket_url()}
+      urls: %{
+        streaming: Pleroma.Web.Endpoint.websocket_url(),
+        status: Config.get([:instance, :status_page])
+      },
+      vapid: %{
+        public_key: Keyword.get(Pleroma.Web.Push.vapid_config(), :public_key)
+      }
     })
   end
 

lib/pleroma/web/mastodon_api/views/poll_view.ex

@@ -21,7 +21,10 @@ defmodule Pleroma.Web.MastodonAPI.PollView do
       votes_count: votes_count,
       voters_count: voters_count(object),
       options: options,
-      emojis: Pleroma.Web.MastodonAPI.StatusView.build_emojis(object.data["emoji"])
+      emojis: Pleroma.Web.MastodonAPI.StatusView.build_emojis(object.data["emoji"]),
+      pleroma: %{
+        non_anonymous: object.data["nonAnonymous"] || false
+      }
     }
 
     if params[:for] do

lib/pleroma/workers/background_worker.ex

@@ -40,6 +40,11 @@ defmodule Pleroma.Workers.BackgroundWorker do
     Pleroma.FollowingRelationship.move_following(origin, target)
   end
 
+  def perform(%Job{args: %{"op" => "verify_fields_links", "user_id" => user_id}}) do
+    user = User.get_by_id(user_id)
+    User.perform(:verify_fields_links, user)
+  end
+
   def perform(%Job{args: %{"op" => "delete_instance", "host" => host}}) do
     Instance.perform(:delete_instance, host)
   end

lib/pleroma/workers/remote_fetcher_worker.ex

@@ -22,8 +22,11 @@ defmodule Pleroma.Workers.RemoteFetcherWorker do
       {:error, :allowed_depth} ->
         {:discard, :allowed_depth}
 
-      _ ->
-        :error
+      {:error, _} = e ->
+        e
+
+      e ->
+        {:error, e}
     end
   end