Merge branch 'develop' into mastodon-admin-api

lib/mix/tasks/pleroma/search/meilisearch.ex

@@ -72,7 +72,7 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do
           query,
           timeout: :infinity
         )
-        |> Stream.map(&Pleroma.Search.Meilisearch.object_to_search_data/1)
+        |> Stream.map(&Pleroma.Search.object_to_search_data/1)
         |> Stream.filter(fn o -> not is_nil(o) end)
         |> Stream.chunk_every(chunk_size)
         |> Stream.transform(0, fn objects, acc ->

lib/pleroma/bookmark.ex

@@ -19,7 +19,7 @@ defmodule Pleroma.Bookmark do
   schema "bookmarks" do
     belongs_to(:user, User, type: FlakeId.Ecto.CompatType)
     belongs_to(:activity, Activity, type: FlakeId.Ecto.CompatType)
-    belongs_to(:folder, BookmarkFolder, type: FlakeId.Ecto.CompatType)
+    belongs_to(:folder, BookmarkFolder, type: FlakeId.Ecto.Type)
 
     timestamps()
   end
@@ -38,7 +38,7 @@ defmodule Pleroma.Bookmark do
     |> validate_required([:user_id, :activity_id])
     |> unique_constraint(:activity_id, name: :bookmarks_user_id_activity_id_index)
     |> Repo.insert(
-      on_conflict: [set: [folder_id: folder_id]],
+      on_conflict: [set: [folder_id: folder_id, updated_at: NaiveDateTime.utc_now()]],
       conflict_target: [:user_id, :activity_id]
     )
   end
@@ -76,11 +76,4 @@ defmodule Pleroma.Bookmark do
     |> Repo.one()
     |> Repo.delete()
   end
-
-  def set_folder(bookmark, folder_id) do
-    bookmark
-    |> cast(%{folder_id: folder_id}, [:folder_id])
-    |> validate_required([:folder_id])
-    |> Repo.update()
-  end
 end

lib/pleroma/bookmark_folder.ex

@@ -14,7 +14,7 @@ defmodule Pleroma.BookmarkFolder do
   alias Pleroma.User
 
   @type t :: %__MODULE__{}
-  @primary_key {:id, FlakeId.Ecto.CompatType, autogenerate: true}
+  @primary_key {:id, FlakeId.Ecto.Type, autogenerate: true}
 
   schema "bookmark_folders" do
     field(:name, :string)

lib/pleroma/config_db.ex

@@ -10,6 +10,7 @@ defmodule Pleroma.ConfigDB do
   import Pleroma.Web.Gettext
 
   alias __MODULE__
+  alias Pleroma.EctoType.Config.RateLimit
   alias Pleroma.Repo
 
   @type t :: %__MODULE__{}
@@ -60,8 +61,59 @@ defmodule Pleroma.ConfigDB do
     |> cast(params, [:key, :group, :value])
     |> validate_required([:key, :group, :value])
     |> unique_constraint(:key, name: :config_group_key_index)
+    |> validate_rate_limit()
   end
 
+  defp validate_rate_limit(changeset) do
+    group = get_field(changeset, :group)
+    key = get_field(changeset, :key)
+
+    if group == :pleroma and key == :rate_limit do
+      value = get_field(changeset, :value)
+
+      case normalize_rate_limit(value) do
+        {:ok, normalized_value} ->
+          put_change(changeset, :value, normalized_value)
+
+        {:error, {limiter_name, reason}} ->
+          add_error(
+            changeset,
+            :value,
+            "invalid :rate_limit value for #{inspect(limiter_name)}: #{reason}"
+          )
+      end
+    else
+      changeset
+    end
+  end
+
+  defp normalize_rate_limit(nil), do: {:ok, nil}
+
+  defp normalize_rate_limit(%{} = value), do: normalize_rate_limit(Map.to_list(value))
+
+  defp normalize_rate_limit(value) when is_list(value) do
+    if Keyword.keyword?(value) do
+      value
+      |> Enum.reduce_while({:ok, []}, fn {limiter_name, limiter_value}, {:ok, acc} ->
+        case RateLimit.cast_with_error(limiter_value) do
+          {:ok, normalized_limiter_value} ->
+            {:cont, {:ok, [{limiter_name, normalized_limiter_value} | acc]}}
+
+          {:error, reason} ->
+            {:halt, {:error, {limiter_name, reason}}}
+        end
+      end)
+      |> case do
+        {:ok, acc} -> {:ok, Enum.reverse(acc)}
+        {:error, _} = error -> error
+      end
+    else
+      {:error, {:rate_limit, "must be a keyword list"}}
+    end
+  end
+
+  defp normalize_rate_limit(_), do: {:error, {:rate_limit, "must be a keyword list"}}
+
   defp create(params) do
     %ConfigDB{}
     |> changeset(params)

lib/pleroma/constants.ex

@@ -29,7 +29,7 @@ defmodule Pleroma.Constants do
 
   const(static_only_files,
     do:
-      ~w(index.html robots.txt static static-fe finmoji emoji packs sounds images instance sw.js sw-pleroma.js favicon.png schemas doc embed.js embed.css)
+      ~w(index.html robots.txt static static-fe finmoji emoji packs sounds images instance sw.js sw-pleroma.js schemas doc embed.js embed.css)
   )
 
   const(status_updatable_fields,

lib/pleroma/ecto_type/config/rate_limit.ex

@@ -0,0 +1,71 @@
+# Pleroma: A lightweight social networking server
+# Copyright © Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.EctoType.Config.RateLimit do
+  @moduledoc false
+
+  use Ecto.Type
+
+  @type t ::
+          nil
+          | {non_neg_integer(), non_neg_integer()}
+          | [{non_neg_integer(), non_neg_integer()}]
+
+  @impl true
+  def type, do: :term
+
+  @impl true
+  def cast(value) do
+    case cast_with_error(value) do
+      {:ok, normalized} -> {:ok, normalized}
+      {:error, _reason} -> :error
+    end
+  end
+
+  @impl true
+  def load(value), do: cast(value)
+
+  @impl true
+  def dump(value), do: cast(value)
+
+  @spec cast_with_error(term()) :: {:ok, t()} | {:error, String.t()}
+  def cast_with_error(nil), do: {:ok, nil}
+
+  def cast_with_error({scale, limit}) do
+    with {:ok, scale} <- parse_integer(scale, "scale"),
+         {:ok, limit} <- parse_integer(limit, "limit"),
+         true <- scale >= 1 and limit >= 1 do
+      {:ok, {scale, limit}}
+    else
+      false -> {:error, "scale and limit must be >= 1"}
+      {:error, reason} -> {:error, reason}
+    end
+  end
+
+  def cast_with_error([{_, _} = unauth, {_, _} = auth]) do
+    with {:ok, unauth} <- cast_with_error(unauth),
+         {:ok, auth} <- cast_with_error(auth) do
+      {:ok, [unauth, auth]}
+    else
+      {:error, reason} -> {:error, reason}
+    end
+  end
+
+  def cast_with_error(_),
+    do:
+      {:error, "must be a {scale, limit} tuple, a [{scale, limit}, {scale, limit}] list, or nil"}
+
+  defp parse_integer(value, _label) when is_integer(value), do: {:ok, value}
+
+  defp parse_integer(value, label) when is_binary(value) do
+    value = String.trim(value)
+
+    case Integer.parse(value) do
+      {number, ""} -> {:ok, number}
+      _ -> {:error, "#{label} must be an integer"}
+    end
+  end
+
+  defp parse_integer(_value, label), do: {:error, "#{label} must be an integer"}
+end

lib/pleroma/formatter.ex

@@ -127,6 +127,13 @@ defmodule Pleroma.Formatter do
     Earmark.as_html!(text, %Earmark.Options{compact_output: true, smartypants: false})
   end
 
+  def markdown_to_html(text, opts) do
+    Earmark.as_html!(
+      text,
+      %Earmark.Options{compact_output: true, smartypants: false} |> Map.merge(opts)
+    )
+  end
+
   def html_escape({text, mentions, hashtags}, type) do
     {html_escape(text, type), mentions, hashtags}
   end
@@ -135,6 +142,10 @@ defmodule Pleroma.Formatter do
     HTML.filter_tags(text)
   end
 
+  def html_escape(text, "text/x.misskeymarkdown") do
+    HTML.filter_tags(text)
+  end
+
   def html_escape(text, "text/plain") do
     Regex.split(@link_regex, text, include_captures: true)
     |> Enum.map_every(2, fn chunk ->

lib/pleroma/frontend.ex

@@ -75,8 +75,8 @@ defmodule Pleroma.Frontend do
   end
 
   defp download_build(frontend_info, dest) do
-    Logger.info("Downloading pre-built bundle for #{frontend_info["name"]}")
     url = String.replace(frontend_info["build_url"], "${ref}", frontend_info["ref"])
+    Logger.info("Downloading pre-built bundle for #{frontend_info["name"]} from #{url}")
 
     with {:ok, %{status: 200, body: zip_body}} <-
            Pleroma.HTTP.get(url, [], pool: :media, recv_timeout: 120_000) do

lib/pleroma/gopher/server.ex

@@ -21,10 +21,13 @@ defmodule Pleroma.Gopher.Server do
 
   def init([ip, port]) do
     Logger.info("Starting gopher server on #{port}")
+    Process.flag(:trap_exit, true)
+
+    listener = :gopher
 
     {:ok, _pid} =
       :ranch.start_listener(
-        :gopher,
+        listener,
         :ranch_tcp,
         %{
           num_acceptors: 100,
@@ -35,7 +38,11 @@ defmodule Pleroma.Gopher.Server do
         []
       )
 
-    {:ok, %{ip: ip, port: port}}
+    {:ok, %{ip: ip, port: port, listener: listener}}
+  end
+
+  def terminate(_reason, state) do
+    :ranch.stop_listener(state.listener)
   end
 end
 

lib/pleroma/list.ex

@@ -17,13 +17,14 @@ defmodule Pleroma.List do
     field(:title, :string)
     field(:following, {:array, :string}, default: [])
     field(:ap_id, :string)
+    field(:exclusive, :boolean, default: false)
 
     timestamps()
   end
 
-  def title_changeset(list, attrs \\ %{}) do
+  def update_changeset(list, attrs \\ %{}) do
     list
-    |> cast(attrs, [:title])
+    |> cast(attrs, [:title, :exclusive])
     |> validate_required([:title])
   end
 
@@ -91,14 +92,14 @@ defmodule Pleroma.List do
     |> Repo.all()
   end
 
-  def rename(%Pleroma.List{} = list, title) do
+  def update(%Pleroma.List{} = list, params) do
     list
-    |> title_changeset(%{title: title})
+    |> update_changeset(params)
     |> Repo.update()
   end
 
-  def create(title, %User{} = creator) do
-    changeset = title_changeset(%Pleroma.List{user_id: creator.id}, %{title: title})
+  def create(params, %User{} = creator) do
+    changeset = update_changeset(%Pleroma.List{user_id: creator.id}, params)
 
     if changeset.valid? do
       Repo.transaction(fn ->
@@ -149,4 +150,14 @@ defmodule Pleroma.List do
   end
 
   def member?(_, _), do: false
+
+  def get_exclusive_list_members(%User{id: user_id}) do
+    Pleroma.List
+    |> where([l], l.user_id == ^user_id)
+    |> where([l], l.exclusive == true)
+    |> select([l], l.following)
+    |> Repo.all()
+    |> List.flatten()
+    |> Enum.uniq()
+  end
 end

lib/pleroma/marker.ex

@@ -78,7 +78,7 @@ defmodule Pleroma.Marker do
 
   defp get_marker(user, timeline) do
     case Repo.find_resource(get_query(user, timeline)) do
-      {:ok, marker} -> %__MODULE__{marker | user: user}
+      {:ok, %__MODULE__{} = marker} -> %{marker | user: user}
       _ -> %__MODULE__{timeline: timeline, user_id: user.id}
     end
   end

lib/pleroma/mfa/changeset.ex

@@ -8,7 +8,8 @@ defmodule Pleroma.MFA.Changeset do
   alias Pleroma.User
 
   def disable(%Ecto.Changeset{} = changeset, force \\ false) do
-    settings =
+    %Settings{} =
+      settings =
       changeset
       |> Ecto.Changeset.apply_changes()
       |> MFA.fetch_settings()
@@ -20,20 +21,20 @@ defmodule Pleroma.MFA.Changeset do
     end
   end
 
-  def disable_totp(%User{multi_factor_authentication_settings: settings} = user) do
+  def disable_totp(%User{multi_factor_authentication_settings: %Settings{} = settings} = user) do
     user
     |> put_change(%Settings{settings | totp: %Settings.TOTP{}})
   end
 
-  def confirm_totp(%User{multi_factor_authentication_settings: settings} = user) do
-    totp_settings = %Settings.TOTP{settings.totp | confirmed: true}
+  def confirm_totp(%User{multi_factor_authentication_settings: %Settings{} = settings} = user) do
+    totp_settings = %Settings.TOTP{(%Settings.TOTP{} = settings.totp) | confirmed: true}
 
     user
     |> put_change(%Settings{settings | totp: totp_settings, enabled: true})
   end
 
   def setup_totp(%User{} = user, attrs) do
-    mfa_settings = MFA.fetch_settings(user)
+    %Settings{} = mfa_settings = MFA.fetch_settings(user)
 
     totp_settings =
       %Settings.TOTP{}
@@ -46,7 +47,7 @@ defmodule Pleroma.MFA.Changeset do
   def cast_backup_codes(%User{} = user, codes) do
     user
     |> put_change(%Settings{
-      user.multi_factor_authentication_settings
+      (%Settings{} = user.multi_factor_authentication_settings)
       | backup_codes: codes
     })
   end

lib/pleroma/object.ex

@@ -372,12 +372,28 @@ defmodule Pleroma.Object do
             option
         end)
 
-      voters = [actor | object.data["voters"] || []] |> Enum.uniq()
+      existing_voters = object.data["voters"] || []
+      voters = [actor | existing_voters] |> Enum.uniq()
+      new_voter? = actor not in existing_voters
+      existing_voters_count = object.data["votersCount"]
+
+      voters_count =
+        cond do
+          is_integer(existing_voters_count) and new_voter? ->
+            existing_voters_count + 1
+
+          is_integer(existing_voters_count) ->
+            existing_voters_count
+
+          true ->
+            length(voters)
+        end
 
       data =
         object.data
         |> Map.put(key, options)
         |> Map.put("voters", voters)
+        |> Map.put("votersCount", voters_count)
 
       object
       |> Object.change(%{data: data})

lib/pleroma/release_tasks.ex

@@ -5,7 +5,10 @@
 defmodule Pleroma.ReleaseTasks do
   @repo Pleroma.Repo
 
-  def run(args) do
+  # TODO: Kept for some backwards compatibility with buggy pleroma_ctl,
+  # if a mismatch between pleroma_ctl and Pleroma accidentaly happens.
+  # Remove in the future.
+  def run(args) when is_binary(args) do
     [task | args] = String.split(args)
 
     case task do
@@ -16,6 +19,20 @@ defmodule Pleroma.ReleaseTasks do
     end
   end
 
+  # HACK: Script arguments need to be received as a list, otherwise (quoted) arguments with
+  # whitespace will be broken. Previously the broken string form above was used,
+  # escaping in the shell does not help.
+  def run(args) when is_list(args) do
+    [task | args] = args
+
+    case task do
+      "migrate" -> migrate(args)
+      "create" -> create()
+      "rollback" -> rollback(args)
+      task -> mix_task(task, args)
+    end
+  end
+
   def find_module(task) do
     module_name =
       task

lib/pleroma/reverse_proxy.ex

@@ -12,7 +12,7 @@ defmodule Pleroma.ReverseProxy do
   @keep_resp_headers @resp_cache_headers ++
                        ~w(content-length content-type content-disposition content-encoding) ++
                        ~w(content-range accept-ranges vary)
-  @default_cache_control_header "public, max-age=1209600"
+  @default_cache_control_header "public, max-age=1209600, immutable"
   @valid_resp_codes [200, 206, 304]
   @max_read_duration :timer.seconds(30)
   @max_body_length :infinity

lib/pleroma/reverse_proxy/client/hackney.ex

@@ -4,6 +4,9 @@
 
 defmodule Pleroma.ReverseProxy.Client.Hackney do
   @behaviour Pleroma.ReverseProxy.Client
+  @redirect_limit 5
+
+  require Logger
 
   # In-app redirect handler to avoid Hackney redirect bugs:
   # - https://github.com/benoitc/hackney/issues/527 (relative/protocol-less redirects can crash Hackney)
@@ -31,26 +34,15 @@ defmodule Pleroma.ReverseProxy.Client.Hackney do
 
     if opts[:follow_redirect] != false do
       {_state, req_opts} = Access.get_and_update(opts, :follow_redirect, fn a -> {a, false} end)
+      env = %{method: method, headers: headers, body: body, req_opts: req_opts}
       res = :hackney.request(method, url, headers, body, req_opts)
 
       case res do
         {:ok, code, resp_headers, _client} when code in @redirect_statuses ->
-          :hackney.request(
-            method,
-            absolute_redirect_url(url, resp_headers),
-            headers,
-            body,
-            req_opts
-          )
+          redirect(url, resp_headers, env, @redirect_limit)
 
         {:ok, code, resp_headers} when code in @redirect_statuses ->
-          :hackney.request(
-            method,
-            absolute_redirect_url(url, resp_headers),
-            headers,
-            body,
-            req_opts
-          )
+          redirect(url, resp_headers, env, @redirect_limit)
 
         _ ->
           res
@@ -71,4 +63,32 @@ defmodule Pleroma.ReverseProxy.Client.Hackney do
 
   @impl true
   def close(ref), do: :hackney.close(ref)
+
+  defp redirect(url, resp_headers, env, limit) when limit == 0 do
+    new_url = absolute_redirect_url(url, resp_headers)
+
+    Logger.debug(
+      "#{__MODULE__}: Handling redirect #{url} -> #{new_url}; redirect limit was reached - returning response after final redirect"
+    )
+
+    :hackney.request(env.method, new_url, env.headers, env.body, env.req_opts)
+  end
+
+  defp redirect(url, resp_headers, env, limit) do
+    new_url = absolute_redirect_url(url, resp_headers)
+    Logger.debug("#{__MODULE__}: handling redirect #{url} -> #{new_url}; limit = #{limit}")
+
+    res = :hackney.request(env.method, new_url, env.headers, env.body, env.req_opts)
+
+    case res do
+      {:ok, code, new_resp_headers, _client} when code in @redirect_statuses ->
+        redirect(new_url, new_resp_headers, env, limit - 1)
+
+      {:ok, code, new_resp_headers} when code in @redirect_statuses ->
+        redirect(new_url, new_resp_headers, env, limit - 1)
+
+      _ ->
+        res
+    end
+  end
 end

lib/pleroma/search.ex

@@ -1,11 +1,28 @@
 defmodule Pleroma.Search do
+  alias Pleroma.Activity
+  alias Pleroma.Object
+  alias Pleroma.Web.ActivityPub.Visibility
   alias Pleroma.Workers.SearchIndexingWorker
 
-  def add_to_index(%Pleroma.Activity{id: activity_id}) do
-    SearchIndexingWorker.new(%{"op" => "add_to_index", "activity" => activity_id})
-    |> Oban.insert()
+  @spec add_to_index(Activity.t()) :: {:ok, Oban.Job.t() | :noop} | {:error, Oban.Job.changeset()}
+  def add_to_index(%Activity{id: activity_id, object: %Object{} = object} = activity) do
+    with {_, true} <- {:indexable, indexable?(activity)},
+         {_, "public"} <- {:visibility, Visibility.get_visibility(object)} do
+      SearchIndexingWorker.new(%{"op" => "add_to_index", "activity" => activity_id})
+      |> Oban.insert()
+    else
+      _ -> {:ok, :noop}
+    end
   end
 
+  def add_to_index(%Activity{id: activity_id}) do
+    case Activity.get_by_id_with_object(activity_id) do
+      %Activity{} = preloaded -> add_to_index(preloaded)
+      _ -> {:ok, :noop}
+    end
+  end
+
+  @spec remove_from_index(Object.t()) :: {:ok, Oban.Job.t()} | {:error, Oban.Job.changeset()}
   def remove_from_index(%Pleroma.Object{id: object_id}) do
     SearchIndexingWorker.new(%{"op" => "remove_from_index", "object" => object_id})
     |> Oban.insert()
@@ -20,4 +37,44 @@ defmodule Pleroma.Search do
     search_module = Pleroma.Config.get([Pleroma.Search, :module])
     search_module.healthcheck_endpoints()
   end
+
+  def object_to_search_data(%Object{} = object) do
+    data = object.data
+
+    content_str =
+      case data["content"] do
+        [nil | rest] -> to_string(rest)
+        str -> str
+      end
+
+    content =
+      with {:ok, scrubbed} <-
+             FastSanitize.Sanitizer.scrub(content_str, Pleroma.HTML.Scrubber.SearchIndexing),
+           trimmed <- String.trim(scrubbed) do
+        trimmed
+      end
+
+    # Make sure we have a non-empty string
+    if content != "" do
+      {:ok, published, _} = DateTime.from_iso8601(data["published"])
+
+      %{
+        id: object.id,
+        content: content,
+        ap: data["id"],
+        published: published |> DateTime.to_unix()
+      }
+    end
+  end
+
+  defp indexable?(%Activity{
+         data: %{"type" => "Create"},
+         object: %Object{
+           data: %{"content" => content, "published" => published, "type" => "Note"}
+         }
+       })
+       when not is_nil(content) and content not in ["", "."] and not is_nil(published),
+       do: true
+
+  defp indexable?(_), do: false
 end

lib/pleroma/search/meilisearch.ex

@@ -4,6 +4,8 @@ defmodule Pleroma.Search.Meilisearch do
 
   alias Pleroma.Activity
   alias Pleroma.Config.Getting, as: Config
+  alias Pleroma.Object
+  alias Pleroma.Search
 
   import Pleroma.Search.DatabaseSearch
   import Ecto.Query
@@ -118,66 +120,24 @@ defmodule Pleroma.Search.Meilisearch do
     end
   end
 
-  def object_to_search_data(object) do
-    # Only index public or unlisted Notes
-    if not is_nil(object) and object.data["type"] == "Note" and
-         not is_nil(object.data["content"]) and
-         not is_nil(object.data["published"]) and
-         (Pleroma.Constants.as_public() in object.data["to"] or
-            Pleroma.Constants.as_public() in object.data["cc"]) and
-         object.data["content"] not in ["", "."] do
-      data = object.data
-
-      content_str =
-        case data["content"] do
-          [nil | rest] -> to_string(rest)
-          str -> str
-        end
-
-      content =
-        with {:ok, scrubbed} <-
-               FastSanitize.Sanitizer.scrub(content_str, Pleroma.HTML.Scrubber.SearchIndexing),
-             trimmed <- String.trim(scrubbed) do
-          trimmed
-        end
-
-      # Make sure we have a non-empty string
-      if content != "" do
-        {:ok, published, _} = DateTime.from_iso8601(data["published"])
-
-        %{
-          id: object.id,
-          content: content,
-          ap: data["id"],
-          published: published |> DateTime.to_unix()
-        }
-      end
-    end
-  end
-
   @impl true
-  def add_to_index(activity) do
-    maybe_search_data = object_to_search_data(activity.object)
+  def add_to_index(%Activity{object: %Object{} = object} = activity) do
+    search_data = Search.object_to_search_data(object)
 
-    if activity.data["type"] == "Create" and maybe_search_data do
-      result =
-        meili_put(
-          "/indexes/objects/documents",
-          [maybe_search_data]
-        )
+    result =
+      meili_put(
+        "/indexes/objects/documents",
+        [search_data]
+      )
 
-      with {:ok, %{"status" => "enqueued"}} <- result do
-        # Added successfully
-        :ok
-      else
-        _ ->
-          # There was an error, report it
-          Logger.error("Failed to add activity #{activity.id} to index: #{inspect(result)}")
-          {:error, result}
-      end
-    else
-      # The post isn't something we can search, that's ok
+    with {:ok, %{"status" => "enqueued"}} <- result do
+      # Added successfully
       :ok
+    else
+      _ ->
+        # There was an error, report it
+        Logger.error("Failed to add activity #{activity.id} to index: #{inspect(result)}")
+        {:error, result}
     end
   end
 

lib/pleroma/search/qdrant_search.ex

@@ -4,11 +4,12 @@ defmodule Pleroma.Search.QdrantSearch do
 
   alias Pleroma.Activity
   alias Pleroma.Config.Getting, as: Config
+  alias Pleroma.Object
+  alias Pleroma.Search
 
   alias __MODULE__.OpenAIClient
   alias __MODULE__.QdrantClient
 
-  import Pleroma.Search.Meilisearch, only: [object_to_search_data: 1]
   import Pleroma.Search.DatabaseSearch, only: [maybe_fetch: 3]
 
   @impl true
@@ -82,23 +83,18 @@ defmodule Pleroma.Search.QdrantSearch do
   end
 
   @impl true
-  def add_to_index(activity) do
-    # This will only index public or unlisted notes
-    maybe_search_data = object_to_search_data(activity.object)
+  def add_to_index(%Activity{object: %Object{} = object} = activity) do
+    search_data = Search.object_to_search_data(object)
 
-    if activity.data["type"] == "Create" and maybe_search_data do
-      with {:ok, embedding} <- get_embedding(maybe_search_data.content),
-           {:ok, %{status: 200}} <-
-             QdrantClient.put(
-               "/collections/posts/points",
-               build_index_payload(activity, embedding)
-             ) do
-        :ok
-      else
-        e -> {:error, e}
-      end
-    else
+    with {:ok, embedding} <- get_embedding(search_data.content),
+         {:ok, %{status: 200}} <-
+           QdrantClient.put(
+             "/collections/posts/points",
+             build_index_payload(activity, embedding)
+           ) do
       :ok
+    else
+      e -> {:error, e}
     end
   end
 

lib/pleroma/upload.ex

@@ -93,7 +93,7 @@ defmodule Pleroma.Upload do
   def store(upload, opts \\ []) do
     opts = get_opts(opts)
 
-    with {:ok, upload} <- prepare_upload(upload, opts),
+    with {:ok, %__MODULE__{} = upload} <- prepare_upload(upload, opts),
          upload = %__MODULE__{upload | path: upload.path || "#{upload.id}/#{upload.name}"},
          {:ok, upload} <- Pleroma.Upload.Filter.filter(opts.filters, upload),
          description = get_description(upload),

lib/pleroma/utils.ex

@@ -17,7 +17,7 @@ defmodule Pleroma.Utils do
     dir
     |> File.ls!()
     |> Enum.map(&Path.join(dir, &1))
-    |> Kernel.ParallelCompiler.compile()
+    |> Kernel.ParallelCompiler.compile(return_diagnostics: true)
   end
 
   @doc """

lib/pleroma/web/activity_pub/activity_pub.ex

@@ -1618,6 +1618,10 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   defp normalize_image(urls) when is_list(urls), do: urls |> List.first() |> normalize_image()
   defp normalize_image(_), do: nil
 
+  defp normalize_also_known_as(urls) when is_list(urls), do: urls
+  defp normalize_also_known_as(url) when is_binary(url), do: [url]
+  defp normalize_also_known_as(nil), do: []
+
   defp maybe_put_description(map, %{"name" => description}) when is_binary(description) do
     Map.put(map, "name", description)
   end
@@ -1673,44 +1677,80 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
     show_birthday = !!birthday
 
-    # if WebFinger request was already done, we probably have acct, otherwise
-    # we request WebFinger here
-    nickname = additional[:nickname_from_acct] || generate_nickname(data)
+    with {:ok, nickname} <- nickname_from_actor(data, additional) do
+      {:ok,
+       %{
+         ap_id: data["id"],
+         uri: get_actor_url(data["url"]),
+         banner: normalize_image(data["image"]),
+         fields: fields,
+         emoji: emojis,
+         is_locked: is_locked,
+         is_discoverable: is_discoverable,
+         invisible: invisible,
+         avatar: normalize_image(data["icon"]),
+         name: data["name"],
+         follower_address: data["followers"],
+         following_address: data["following"],
+         featured_address: featured_address,
+         bio: data["summary"] || "",
+         actor_type: actor_type,
+         also_known_as: normalize_also_known_as(data["alsoKnownAs"]),
+         public_key: public_key,
+         inbox: data["inbox"],
+         shared_inbox: shared_inbox,
+         accepts_chat_messages: accepts_chat_messages,
+         birthday: birthday,
+         show_birthday: show_birthday,
+         pinned_objects: pinned_objects,
+         nickname: nickname
+       }}
+    end
+  end
 
-    %{
-      ap_id: data["id"],
-      uri: get_actor_url(data["url"]),
-      banner: normalize_image(data["image"]),
-      fields: fields,
-      emoji: emojis,
-      is_locked: is_locked,
-      is_discoverable: is_discoverable,
-      invisible: invisible,
-      avatar: normalize_image(data["icon"]),
-      name: data["name"],
-      follower_address: data["followers"],
-      following_address: data["following"],
-      featured_address: featured_address,
-      bio: data["summary"] || "",
-      actor_type: actor_type,
-      also_known_as: Map.get(data, "alsoKnownAs", []),
-      public_key: public_key,
-      inbox: data["inbox"],
-      shared_inbox: shared_inbox,
-      accepts_chat_messages: accepts_chat_messages,
-      birthday: birthday,
-      show_birthday: show_birthday,
-      pinned_objects: pinned_objects,
-      nickname: nickname
-    }
+  defp nickname_from_actor(data, additional) do
+    generated = generated_nickname(data)
+
+    case additional[:nickname_from_acct] do
+      ^generated when is_binary(generated) ->
+        {:ok, generated}
+
+      acct when is_binary(acct) ->
+        with ^acct <- webfinger_nickname(data) do
+          {:ok, acct}
+        else
+          _ -> {:error, {:webfinger_actor_mismatch, acct, data["id"]}}
+        end
+
+      _ ->
+        {:ok, generate_nickname(data)}
+    end
+  end
+
+  defp generated_nickname(%{"preferredUsername" => username, "id" => ap_id})
+       when is_binary(username) and is_binary(ap_id) do
+    case URI.parse(ap_id) do
+      %URI{host: host} when is_binary(host) -> "#{username}@#{host}"
+      _ -> nil
+    end
+  end
+
+  defp generated_nickname(_), do: nil
+
+  defp webfinger_nickname(data) do
+    with generated when is_binary(generated) <- generated_nickname(data),
+         {:ok, %{"subject" => "acct:" <> acct, "ap_id" => ap_id}} <- WebFinger.finger(generated),
+         true <- ap_id == data["id"] do
+      acct
+    end
   end
 
   defp generate_nickname(%{"preferredUsername" => username} = data) when is_binary(username) do
-    generated = "#{username}@#{URI.parse(data["id"]).host}"
+    generated = generated_nickname(data)
 
     if Config.get([WebFinger, :update_nickname_on_user_fetch]) do
-      case WebFinger.finger(generated) do
-        {:ok, %{"subject" => "acct:" <> acct}} -> acct
+      case webfinger_nickname(data) do
+        acct when is_binary(acct) -> acct
         _ -> generated
       end
     else
@@ -1790,9 +1830,11 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   defp collection_private(_data), do: {:ok, true}
 
   def user_data_from_user_object(data, additional \\ []) do
-    with {:ok, data} <- MRF.filter(data) do
-      {:ok, object_to_user_data(data, additional)}
+    with {:ok, data} <- MRF.filter(data),
+         {:ok, data} <- object_to_user_data(data, additional) do
+      {:ok, data}
     else
+      {:error, _} = e -> e
       e -> {:error, e}
     end
   end

lib/pleroma/web/activity_pub/activity_pub_controller.ex

@@ -348,7 +348,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
   end
 
   def inbox(%{assigns: %{valid_signature: false}} = conn, params) do
-    Federator.incoming_ap_doc(%{
+    Federator.incoming_failed_signature_ap_doc(%{
       method: conn.method,
       req_headers: conn.req_headers,
       request_path: conn.request_path,

lib/pleroma/web/activity_pub/builder.ex

@@ -332,21 +332,18 @@ defmodule Pleroma.Web.ActivityPub.Builder do
 
   @spec announce(User.t(), Object.t(), keyword()) :: {:ok, map(), keyword()}
   def announce(actor, object, options \\ []) do
-    public? = Keyword.get(options, :public, false)
+    visibility = Keyword.get(options, :visibility, "public")
 
-    to =
-      cond do
-        actor.ap_id == Relay.ap_id() ->
-          [actor.follower_address]
-
-        public? and Visibility.local_public?(object) ->
-          [actor.follower_address, object.data["actor"], Utils.as_local_public()]
-
-        public? ->
-          [actor.follower_address, object.data["actor"], Pleroma.Constants.as_public()]
-
-        true ->
-          [actor.follower_address, object.data["actor"]]
+    {to, cc} =
+      if actor.ap_id == Relay.ap_id() do
+        {[actor.follower_address], []}
+      else
+        Pleroma.Web.CommonAPI.Utils.get_to_and_cc_for_visibility(
+          visibility,
+          actor.follower_address,
+          nil,
+          [object.data["actor"]]
+        )
       end
 
     {:ok,
@@ -355,6 +352,7 @@ defmodule Pleroma.Web.ActivityPub.Builder do
        "actor" => actor.ap_id,
        "object" => object.data["id"],
        "to" => to,
+       "cc" => cc,
        "context" => object.data["context"],
        "type" => "Announce",
        "published" => Utils.make_date()

lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex

@@ -6,9 +6,12 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidator do
   use Ecto.Schema
 
   alias Pleroma.EctoType.ActivityPub.ObjectValidators
+  alias Pleroma.HTML
+  alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes
   alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations
   alias Pleroma.Web.ActivityPub.Transmogrifier
+  alias Pleroma.Web.CommonAPI.Utils
 
   import Ecto.Changeset
 
@@ -26,6 +29,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidator do
     end
 
     field(:replies, {:array, ObjectValidators.ObjectID}, default: [])
+    field(:source, :map)
   end
 
   def cast_and_apply(data) do
@@ -80,6 +84,113 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidator do
 
   def fix_attachments(data), do: data
 
+  defp remote_mention_resolver(
+         %{"id" => ap_id, "tag" => tags},
+         "@" <> nickname = mention,
+         buffer,
+         opts,
+         acc
+       )
+       when is_binary(ap_id) and is_list(tags) do
+    initial_host =
+      ap_id
+      |> URI.parse()
+      |> Map.get(:host)
+
+    with mention_tag when not is_nil(mention_tag) <-
+           Enum.find(tags, &mention_tag?(&1, mention, initial_host)),
+         href when is_binary(href) <- mention_tag["href"],
+         %User{} = user <- User.get_cached_by_ap_id(href) do
+      link = Pleroma.Formatter.mention_from_user(user, opts)
+      {link, %{acc | mentions: MapSet.put(acc.mentions, {"@" <> nickname, user})}}
+    else
+      _ -> {buffer, acc}
+    end
+  end
+
+  defp remote_mention_resolver(_object, _mention, buffer, _opts, acc), do: {buffer, acc}
+
+  defp mention_tag?(%{"type" => "Mention", "name" => name}, mention, initial_host)
+       when is_binary(name) do
+    name == mention || mention == "#{name}@#{initial_host}"
+  end
+
+  defp mention_tag?(_tag, _mention, _initial_host), do: false
+
+  defp scrub_content(%{"content" => content} = object) when is_binary(content) do
+    Map.put(object, "content", HTML.filter_tags(content))
+  end
+
+  defp scrub_content(object), do: object
+
+  defp mfm_parse_limit do
+    min(Pleroma.Config.get([:instance, :limit]), Pleroma.Config.get([:instance, :remote_limit]))
+  end
+
+  defp normalize_source(%{"source" => source} = object) when is_binary(source) do
+    object
+    |> Map.put("source", %{"content" => source})
+    |> normalize_source()
+  end
+
+  defp normalize_source(%{"source" => source} = object) when is_map(source) do
+    source =
+      case source["content"] do
+        content when is_binary(content) ->
+          if String.length(content) <= mfm_parse_limit() do
+            source
+          else
+            Map.delete(source, "content")
+          end
+
+        nil ->
+          source
+
+        _ ->
+          Map.delete(source, "content")
+      end
+
+    Map.put(object, "source", source)
+  end
+
+  defp normalize_source(object), do: object
+
+  defp fix_misskey_content(%{"htmlMfm" => true, "content" => content} = object)
+       when is_binary(content) do
+    Map.put(object, "content", HTML.filter_tags(content))
+  end
+
+  defp fix_misskey_content(%{"htmlMfm" => true} = object), do: object
+
+  defp fix_misskey_content(
+         %{"source" => %{"mediaType" => "text/x.misskeymarkdown", "content" => content}} = object
+       )
+       when is_binary(content) do
+    mention_handler = fn nick, buffer, opts, acc ->
+      remote_mention_resolver(object, nick, buffer, opts, acc)
+    end
+
+    {linked, _mentions, _tags} =
+      Utils.format_input(content, "text/x.misskeymarkdown", mention_handler: mention_handler)
+
+    Map.put(object, "content", linked)
+  end
+
+  defp fix_misskey_content(%{"source" => %{"mediaType" => "text/x.misskeymarkdown"}} = object),
+    do: scrub_content(object)
+
+  defp fix_misskey_content(%{"_misskey_content" => content} = object) when is_binary(content) do
+    object
+    |> Map.put("source", %{
+      "content" => content,
+      "mediaType" => "text/x.misskeymarkdown"
+    })
+    |> Map.delete("_misskey_content")
+    |> fix_misskey_content()
+  end
+
+  defp fix_misskey_content(object), do: object
+
   defp fix(data) do
     data
     |> CommonFixes.fix_actor()
@@ -88,6 +199,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidator do
     |> fix_tag()
     |> fix_replies()
     |> fix_attachments()
+    |> normalize_source()
+    |> fix_misskey_content()
     |> CommonFixes.fix_quote_url()
     |> CommonFixes.fix_likes()
     |> Transmogrifier.fix_emoji()

lib/pleroma/web/activity_pub/object_validators/common_fields.ex

@@ -32,6 +32,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFields do
     quote bind_quoted: binding() do
       field(:content, :string)
       field(:contentMap, ObjectValidators.ContentLanguageMap)
+      field(:htmlMfm, :boolean)
 
       field(:published, ObjectValidators.DateTime)
       field(:updated, ObjectValidators.DateTime)

lib/pleroma/web/activity_pub/object_validators/question_validator.ex

@@ -28,6 +28,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do
     end
 
     field(:closed, ObjectValidators.DateTime)
+    field(:votersCount, :integer)
     field(:voters, {:array, ObjectValidators.ObjectID}, default: [])
     field(:nonAnonymous, :boolean)
     embeds_many(:anyOf, QuestionOptionsValidator)

lib/pleroma/web/activity_pub/object_validators/update_validator.ex

@@ -75,15 +75,40 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.UpdateValidator do
     end
   end
 
-  # For remote Updates, verify the host is the same.
+  # For remote Updates, verify the Actor is the same
   def validate_updating_rights_remote(cng) do
     with actor = get_field(cng, :actor),
          object = get_field(cng, :object),
          {:ok, object_id} <- ObjectValidators.ObjectID.cast(object),
-         actor_uri <- URI.parse(actor),
-         object_uri <- URI.parse(object_id),
-         true <- actor_uri.host == object_uri.host do
-      cng
+         entity <-
+           Object.normalize(object_id, fetch: false) || User.get_cached_by_ap_id(object_id) do
+      case entity do
+        # Actor must own Object to update it
+        %Object{} ->
+          if actor == entity.data["actor"] do
+            cng
+          else
+            cng
+            |> add_error(:object, "Can't be updated by this actor")
+          end
+
+        # Actor must only be allowed to update itself
+        %User{} ->
+          if actor == entity.ap_id do
+            cng
+          else
+            cng
+            |> add_error(:object, "Can't be updated by this actor")
+          end
+
+        nil ->
+          cng
+          |> add_error(:object, "Can't be updated by this actor")
+
+        _ ->
+          cng
+          |> add_error(:object, "Update is neither for Object or Actor")
+      end
     else
       _e ->
         cng

lib/pleroma/web/activity_pub/transmogrifier.ex

@@ -783,6 +783,12 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
 
   def set_replies(obj_data), do: obj_data
 
+  defp set_voters_count(%{"voters" => [_ | _] = voters} = obj) do
+    Map.merge(obj, %{"votersCount" => length(voters)})
+  end
+
+  defp set_voters_count(obj), do: obj
+
   # Prepares and sanitizes the object for federation.
   def prepare_object(object) do
     object
@@ -795,6 +801,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
     |> set_reply_to_uri
     |> set_quote_url
     |> set_replies
+    |> set_voters_count
     |> CommonFixes.maybe_add_content_map()
     |> strip_internal_fields
     |> strip_internal_tags

lib/pleroma/web/activity_pub/utils.ex

@@ -120,7 +120,8 @@ defmodule Pleroma.Web.ActivityPub.Utils do
         "https://www.w3.org/ns/activitystreams",
         "#{Endpoint.url()}/schemas/litepub-0.1.jsonld",
         %{
-          "@language" => get_language(data)
+          "@language" => get_language(data),
+          "htmlMfm" => "https://w3id.org/fep/c16b#htmlMfm"
         }
       ]
     }

lib/pleroma/web/activity_pub/views/user_view.ex

@@ -35,32 +35,14 @@ defmodule Pleroma.Web.ActivityPub.UserView do
   def render("endpoints.json", _), do: %{}
 
   def render("service.json", %{user: user}) do
-    {:ok, _, public_key} = Keys.keys_from_pem(user.keys)
-    public_key = :public_key.pem_entry_encode(:SubjectPublicKeyInfo, public_key)
-    public_key = :public_key.pem_encode([public_key])
-
-    endpoints = render("endpoints.json", %{user: user})
-
-    %{
-      "id" => user.ap_id,
+    Map.merge(common_actor_fields(user), %{
       "type" => "Application",
-      "following" => "#{user.ap_id}/following",
-      "followers" => "#{user.ap_id}/followers",
-      "inbox" => "#{user.ap_id}/inbox",
-      "outbox" => "#{user.ap_id}/outbox",
       "name" => "Pleroma",
       "summary" =>
         "An internal service actor for this Pleroma instance.  No user-serviceable parts inside.",
-      "url" => user.ap_id,
       "manuallyApprovesFollowers" => false,
-      "publicKey" => %{
-        "id" => "#{user.ap_id}#main-key",
-        "owner" => user.ap_id,
-        "publicKeyPem" => public_key
-      },
-      "endpoints" => endpoints,
       "invisible" => User.invisible?(user)
-    }
+    })
     |> Map.merge(Utils.make_json_ld_header())
   end
 
@@ -77,13 +59,8 @@ defmodule Pleroma.Web.ActivityPub.UserView do
   end
 
   def render("user.json", %{user: user}) do
-    {:ok, _, public_key} = Keys.keys_from_pem(user.keys)
-    public_key = :public_key.pem_entry_encode(:SubjectPublicKeyInfo, public_key)
-    public_key = :public_key.pem_encode([public_key])
     user = User.sanitize_html(user)
 
-    endpoints = render("endpoints.json", %{user: user})
-
     emoji_tags = Transmogrifier.take_emoji_tags(user)
 
     fields = Enum.map(user.fields, &Map.put(&1, "type", "PropertyValue"))
@@ -102,25 +79,9 @@ defmodule Pleroma.Web.ActivityPub.UserView do
         do: Date.to_iso8601(user.birthday),
         else: nil
 
-    %{
-      "id" => user.ap_id,
-      "type" => user.actor_type,
-      "following" => "#{user.ap_id}/following",
-      "followers" => "#{user.ap_id}/followers",
-      "inbox" => "#{user.ap_id}/inbox",
-      "outbox" => "#{user.ap_id}/outbox",
+    Map.merge(common_actor_fields(user), %{
       "featured" => "#{user.ap_id}/collections/featured",
       "preferredUsername" => user.nickname,
-      "name" => user.name,
-      "summary" => user.bio,
-      "url" => user.ap_id,
-      "manuallyApprovesFollowers" => user.is_locked,
-      "publicKey" => %{
-        "id" => "#{user.ap_id}#main-key",
-        "owner" => user.ap_id,
-        "publicKeyPem" => public_key
-      },
-      "endpoints" => endpoints,
       "attachment" => fields,
       "tag" => emoji_tags,
       # Note: key name is indeed "discoverable" (not an error)
@@ -130,7 +91,7 @@ defmodule Pleroma.Web.ActivityPub.UserView do
       "vcard:bday" => birthday,
       "webfinger" => "acct:#{User.full_nickname(user)}",
       "published" => Pleroma.Web.CommonAPI.Utils.to_masto_date(user.inserted_at)
-    }
+    })
     |> Map.merge(
       maybe_make_image(
         &User.avatar_url/2,
@@ -309,6 +270,33 @@ defmodule Pleroma.Web.ActivityPub.UserView do
     |> Map.merge(Utils.make_json_ld_header())
   end
 
+  defp common_actor_fields(%User{} = user) do
+    endpoints = render("endpoints.json", %{user: user})
+
+    {:ok, _, public_key} = Keys.keys_from_pem(user.keys)
+    public_key = :public_key.pem_entry_encode(:SubjectPublicKeyInfo, public_key)
+    public_key = :public_key.pem_encode([public_key])
+
+    %{
+      "id" => user.ap_id,
+      "type" => user.actor_type,
+      "following" => "#{user.ap_id}/following",
+      "followers" => "#{user.ap_id}/followers",
+      "inbox" => "#{user.ap_id}/inbox",
+      "outbox" => "#{user.ap_id}/outbox",
+      "name" => user.name,
+      "summary" => user.bio,
+      "url" => user.ap_id,
+      "manuallyApprovesFollowers" => user.is_locked,
+      "endpoints" => endpoints,
+      "publicKey" => %{
+        "id" => "#{user.ap_id}#main-key",
+        "owner" => user.ap_id,
+        "publicKeyPem" => public_key
+      }
+    }
+  end
+
   defp maybe_put_total_items(map, false, _total), do: map
 
   defp maybe_put_total_items(map, true, total) do

lib/pleroma/web/api_spec/cast_and_validate.ex

@@ -106,7 +106,14 @@ defmodule Pleroma.Web.ApiSpec.CastAndValidate do
     OpenApiSpex.cast_and_validate(spec, operation, conn, content_type, cast_opts)
   end
 
-  defp cast_and_validate(spec, operation, conn, content_type, false = _strict, cast_opts) do
+  defp cast_and_validate(
+         spec,
+         operation,
+         %Conn{} = conn,
+         content_type,
+         false = _strict,
+         cast_opts
+       ) do
     case OpenApiSpex.cast_and_validate(spec, operation, conn, content_type) do
       {:ok, conn} ->
         {:ok, conn}

lib/pleroma/web/api_spec/operations/list_operation.ex

@@ -36,7 +36,7 @@ defmodule Pleroma.Web.ApiSpec.ListOperation do
       summary: "Create a list",
       description: "Fetch the list with the given ID. Used for verifying the title of a list.",
       operationId: "ListController.create",
-      requestBody: create_update_request(),
+      requestBody: create_request(),
       security: [%{"oAuth" => ["write:lists"]}],
       responses: %{
         200 => Operation.response("List", "application/json", List),
@@ -68,7 +68,7 @@ defmodule Pleroma.Web.ApiSpec.ListOperation do
       description: "Change the title of a list",
       operationId: "ListController.update",
       parameters: [id_param()],
-      requestBody: create_update_request(),
+      requestBody: update_request(),
       security: [%{"oAuth" => ["write:lists"]}],
       responses: %{
         200 => Operation.response("List", "application/json", List),
@@ -164,14 +164,18 @@ defmodule Pleroma.Web.ApiSpec.ListOperation do
     )
   end
 
-  defp create_update_request do
+  defp create_request do
     request_body(
       "Parameters",
       %Schema{
-        description: "POST body for creating or updating a List",
+        description: "POST body for creating a List",
         type: :object,
         properties: %{
-          title: %Schema{type: :string, description: "List title"}
+          title: %Schema{type: :string, description: "List title"},
+          exclusive: %Schema{
+            type: :boolean,
+            description: "Whether members of the list should be removed from the “Home” feed"
+          }
         },
         required: [:title]
       },
@@ -179,6 +183,24 @@ defmodule Pleroma.Web.ApiSpec.ListOperation do
     )
   end
 
+  defp update_request do
+    request_body(
+      "Parameters",
+      %Schema{
+        description: "PUT body for updating a List",
+        type: :object,
+        properties: %{
+          title: %Schema{type: :string, description: "List title"},
+          exclusive: %Schema{
+            type: :boolean,
+            description: "Whether members of the list should be removed from the “Home” feed"
+          }
+        }
+      },
+      required: true
+    )
+  end
+
   defp add_remove_accounts_request(required) when is_boolean(required) do
     request_body(
       "Parameters",

lib/pleroma/web/api_spec/operations/pleroma_util_operation.ex

@@ -2,7 +2,7 @@
 # Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
-defmodule Pleroma.Web.ApiSpec.TwitterUtilOperation do
+defmodule Pleroma.Web.ApiSpec.PleromaUtilOperation do
   alias OpenApiSpex.Operation
   alias OpenApiSpex.Schema
   alias Pleroma.Web.ApiSpec.Schemas.ApiError
@@ -19,7 +19,7 @@ defmodule Pleroma.Web.ApiSpec.TwitterUtilOperation do
     %Operation{
       tags: ["Custom emojis"],
       summary: "List all custom emojis",
-      operationId: "UtilController.emoji",
+      operationId: "PleromaAPI.UtilController.emoji",
       parameters: [],
       responses: %{
         200 =>
@@ -48,7 +48,7 @@ defmodule Pleroma.Web.ApiSpec.TwitterUtilOperation do
     %Operation{
       tags: ["Others"],
       summary: "Dump frontend configurations",
-      operationId: "UtilController.frontend_configurations",
+      operationId: "PleromaAPI.UtilController.frontend_configurations",
       parameters: [],
       responses: %{
         200 =>
@@ -70,7 +70,7 @@ defmodule Pleroma.Web.ApiSpec.TwitterUtilOperation do
       tags: ["Account credentials"],
       summary: "Change account password",
       security: [%{"oAuth" => ["write:accounts"]}],
-      operationId: "UtilController.change_password",
+      operationId: "PleromaAPI.UtilController.change_password",
       requestBody: request_body("Parameters", change_password_request(), required: true),
       responses: %{
         200 =>
@@ -106,7 +106,7 @@ defmodule Pleroma.Web.ApiSpec.TwitterUtilOperation do
       tags: ["Account credentials"],
       summary: "Change account email",
       security: [%{"oAuth" => ["write:accounts"]}],
-      operationId: "UtilController.change_email",
+      operationId: "PleromaAPI.UtilController.change_email",
       requestBody: request_body("Parameters", change_email_request(), required: true),
       responses: %{
         200 =>
@@ -141,7 +141,7 @@ defmodule Pleroma.Web.ApiSpec.TwitterUtilOperation do
       tags: ["Settings"],
       summary: "Update Notification Settings",
       security: [%{"oAuth" => ["write:accounts"]}],
-      operationId: "UtilController.update_notification_settings",
+      operationId: "PleromaAPI.UtilController.update_notification_settings",
       parameters: [
         Operation.parameter(
           :block_from_strangers,
@@ -173,7 +173,7 @@ defmodule Pleroma.Web.ApiSpec.TwitterUtilOperation do
       tags: ["Account credentials"],
       summary: "Disable Account",
       security: [%{"oAuth" => ["write:accounts"]}],
-      operationId: "UtilController.disable_account",
+      operationId: "PleromaAPI.UtilController.disable_account",
       parameters: [
         Operation.parameter(:password, :query, :string, "Password")
       ],
@@ -193,7 +193,7 @@ defmodule Pleroma.Web.ApiSpec.TwitterUtilOperation do
       tags: ["Account credentials"],
       summary: "Delete Account",
       security: [%{"oAuth" => ["write:accounts"]}],
-      operationId: "UtilController.delete_account",
+      operationId: "PleromaAPI.UtilController.delete_account",
       parameters: [
         Operation.parameter(:password, :query, :string, "Password")
       ],
@@ -212,7 +212,7 @@ defmodule Pleroma.Web.ApiSpec.TwitterUtilOperation do
   def captcha_operation do
     %Operation{
       summary: "Get a captcha",
-      operationId: "UtilController.captcha",
+      operationId: "PleromaAPI.UtilController.captcha",
       tags: ["Others"],
       parameters: [],
       responses: %{
@@ -226,7 +226,7 @@ defmodule Pleroma.Web.ApiSpec.TwitterUtilOperation do
       tags: ["Account credentials"],
       summary: "Move account",
       security: [%{"oAuth" => ["write:accounts"]}],
-      operationId: "UtilController.move_account",
+      operationId: "PleromaAPI.UtilController.move_account",
       requestBody: request_body("Parameters", move_account_request(), required: true),
       responses: %{
         200 =>
@@ -262,7 +262,7 @@ defmodule Pleroma.Web.ApiSpec.TwitterUtilOperation do
       tags: ["Account credentials"],
       summary: "List account aliases",
       security: [%{"oAuth" => ["read:accounts"]}],
-      operationId: "UtilController.list_aliases",
+      operationId: "PleromaAPI.UtilController.list_aliases",
       responses: %{
         200 =>
           Operation.response("Success", "application/json", %Schema{
@@ -286,7 +286,7 @@ defmodule Pleroma.Web.ApiSpec.TwitterUtilOperation do
       tags: ["Account credentials"],
       summary: "Add an alias to this account",
       security: [%{"oAuth" => ["write:accounts"]}],
-      operationId: "UtilController.add_alias",
+      operationId: "PleromaAPI.UtilController.add_alias",
       requestBody: request_body("Parameters", add_alias_request(), required: true),
       responses: %{
         200 =>
@@ -326,7 +326,7 @@ defmodule Pleroma.Web.ApiSpec.TwitterUtilOperation do
       tags: ["Account credentials"],
       summary: "Delete an alias from this account",
       security: [%{"oAuth" => ["write:accounts"]}],
-      operationId: "UtilController.delete_alias",
+      operationId: "PleromaAPI.UtilController.delete_alias",
       requestBody: request_body("Parameters", delete_alias_request(), required: true),
       responses: %{
         200 =>
@@ -366,7 +366,7 @@ defmodule Pleroma.Web.ApiSpec.TwitterUtilOperation do
       tags: ["Others"],
       summary: "Quick status check on the instance",
       security: [%{"oAuth" => ["write:accounts"]}],
-      operationId: "UtilController.healthcheck",
+      operationId: "PleromaAPI.UtilController.healthcheck",
       parameters: [],
       responses: %{
         200 => Operation.response("Healthy", "application/json", %Schema{type: :object}),
@@ -376,52 +376,6 @@ defmodule Pleroma.Web.ApiSpec.TwitterUtilOperation do
     }
   end
 
-  def remote_subscribe_operation do
-    %Operation{
-      tags: ["Remote interaction"],
-      summary: "Remote Subscribe",
-      operationId: "UtilController.remote_subscribe",
-      parameters: [],
-      responses: %{200 => Operation.response("Web Page", "test/html", %Schema{type: :string})}
-    }
-  end
-
-  def remote_interaction_operation do
-    %Operation{
-      tags: ["Remote interaction"],
-      summary: "Remote interaction",
-      operationId: "UtilController.remote_interaction",
-      requestBody: request_body("Parameters", remote_interaction_request(), required: true),
-      responses: %{
-        200 =>
-          Operation.response("Remote interaction URL", "application/json", %Schema{type: :object})
-      }
-    }
-  end
-
-  defp remote_interaction_request do
-    %Schema{
-      title: "RemoteInteractionRequest",
-      description: "POST body for remote interaction",
-      type: :object,
-      required: [:ap_id, :profile],
-      properties: %{
-        ap_id: %Schema{type: :string, description: "Profile or status ActivityPub ID"},
-        profile: %Schema{type: :string, description: "Remote profile webfinger"}
-      }
-    }
-  end
-
-  def show_subscribe_form_operation do
-    %Operation{
-      tags: ["Remote interaction"],
-      summary: "Show remote subscribe form",
-      operationId: "UtilController.show_subscribe_form",
-      parameters: [],
-      responses: %{200 => Operation.response("Web Page", "test/html", %Schema{type: :string})}
-    }
-  end
-
   defp delete_account_request do
     %Schema{
       title: "AccountDeleteRequest",

lib/pleroma/web/api_spec/operations/remote_interaction_operation.ex

@@ -0,0 +1,99 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ApiSpec.RemoteInteractionOperation do
+  alias OpenApiSpex.Operation
+  alias OpenApiSpex.Schema
+
+  import Pleroma.Web.ApiSpec.Helpers
+
+  def open_api_operation(action) do
+    operation = String.to_existing_atom("#{action}_operation")
+    apply(__MODULE__, operation, [])
+  end
+
+  def remote_interaction_operation do
+    %Operation{
+      tags: ["Remote interaction"],
+      summary: "Remote interaction",
+      operationId: "RemoteInteractionController.remote_interaction",
+      requestBody: request_body("Parameters", remote_interaction_request(), required: true),
+      responses: %{
+        200 =>
+          Operation.response("Remote interaction URL", "application/json", %Schema{type: :object})
+      }
+    }
+  end
+
+  defp remote_interaction_request do
+    %Schema{
+      title: "RemoteInteractionRequest",
+      description: "POST body for remote interaction",
+      type: :object,
+      required: [:ap_id, :profile],
+      properties: %{
+        ap_id: %Schema{type: :string, description: "Profile or status ActivityPub ID"},
+        profile: %Schema{type: :string, description: "Remote profile webfinger"}
+      }
+    }
+  end
+
+  def follow_operation do
+    %Operation{
+      tags: ["Remote interaction"],
+      summary: "Display follow form",
+      operationId: "RemoteInteractionController.follow",
+      parameters: [],
+      responses: %{
+        200 => Operation.response("Web Page", "text/html", %Schema{type: :string}),
+        302 => Operation.response("Redirect to the status page", nil, nil)
+      }
+    }
+  end
+
+  def do_follow_operation do
+    %Operation{
+      tags: ["Remote interaction"],
+      summary: "Perform follow activity",
+      operationId: "RemoteInteractionController.do_follow",
+      parameters: [],
+      responses: %{
+        200 => Operation.response("Web page", "text/html", %Schema{type: :string}),
+        302 => Operation.response("Redirect to the account page", nil, nil)
+      }
+    }
+  end
+
+  def authorize_interaction_operation do
+    %Operation{
+      tags: ["Remote interaction"],
+      summary: "Authorize remote interaction",
+      operationId: "RemoteInteractionController.authorize_interaction",
+      parameters: [],
+      responses: %{
+        302 => Operation.response("Redirect to remote_interaction path", nil, nil)
+      }
+    }
+  end
+
+  def show_subscribe_form_operation do
+    %Operation{
+      tags: ["Remote interaction"],
+      summary: "Show remote subscribe form",
+      operationId: "RemoteInteractionController.show_subscribe_form",
+      parameters: [],
+      responses: %{200 => Operation.response("Web Page", "text/html", %Schema{type: :string})}
+    }
+  end
+
+  def remote_subscribe_operation do
+    %Operation{
+      tags: ["Remote interaction"],
+      summary: "Remote Subscribe",
+      operationId: "RemoteInteractionController.remote_subscribe",
+      parameters: [],
+      responses: %{200 => Operation.response("Web Page", "text/html", %Schema{type: :string})}
+    }
+  end
+end

lib/pleroma/web/api_spec/render_error.ex

@@ -17,11 +17,11 @@ defmodule Pleroma.Web.ApiSpec.RenderError do
   def call(conn, errors) do
     errors =
       Enum.map(errors, fn
-        %{name: nil, reason: :invalid_enum} = err ->
-          %OpenApiSpex.Cast.Error{err | name: err.value}
+        %OpenApiSpex.Cast.Error{name: nil, reason: :invalid_enum} = err ->
+          %{err | name: err.value}
 
-        %{name: nil} = err ->
-          %OpenApiSpex.Cast.Error{err | name: List.last(err.path)}
+        %OpenApiSpex.Cast.Error{name: nil} = err ->
+          %{err | name: List.last(err.path)}
 
         err ->
           err

lib/pleroma/web/api_spec/schemas/bookmark_folder.ex

@@ -15,12 +15,18 @@ defmodule Pleroma.Web.ApiSpec.Schemas.BookmarkFolder do
     properties: %{
       id: FlakeID,
       name: %Schema{type: :string, description: "Folder name"},
-      emoji: %Schema{type: :string, description: "Folder emoji", nullable: true}
+      emoji: %Schema{type: :string, description: "Folder emoji", nullable: true},
+      emoji_url: %Schema{
+        type: :string,
+        description: "URL of the folder emoji if it's a custom emoji, null otherwise",
+        nullable: true
+      }
     },
     example: %{
       "id" => "9toJCu5YZW7O7gfvH6",
       "name" => "Read later",
-      "emoji" => nil
+      "emoji" => nil,
+      "emoji_url" => nil
     }
   })
 end

lib/pleroma/web/api_spec/schemas/list.ex

@@ -13,7 +13,11 @@ defmodule Pleroma.Web.ApiSpec.Schemas.List do
     type: :object,
     properties: %{
       id: %Schema{type: :string, description: "The internal database ID of the list"},
-      title: %Schema{type: :string, description: "The user-defined title of the list"}
+      title: %Schema{type: :string, description: "The user-defined title of the list"},
+      exclusive: %Schema{
+        type: :boolean,
+        description: "Whether members of the list should be removed from the “Home” feed"
+      }
     },
     example: %{
       "id" => "12249",

lib/pleroma/web/common_api.ex

@@ -222,8 +222,8 @@ defmodule Pleroma.Web.CommonAPI do
     with %Activity{data: %{"type" => "Create"}} = activity <- Activity.get_by_id(id),
          object = %Object{} <- Object.normalize(activity, fetch: false),
          {_, nil} <- {:existing_announce, Utils.get_existing_announce(user.ap_id, object)},
-         public = public_announce?(object, params),
-         {:ok, announce, _} <- Builder.announce(user, object, public: public),
+         visibility = announce_visibility(object, params),
+         {:ok, announce, _} <- Builder.announce(user, object, visibility: visibility),
          {:ok, activity, _} <- Pipeline.common_pipeline(announce, local: true) do
       {:ok, activity}
     else
@@ -407,13 +407,11 @@ defmodule Pleroma.Web.CommonAPI do
     end
   end
 
-  defp public_announce?(_, %{visibility: visibility})
-       when visibility in ~w{public unlisted private direct},
-       do: visibility in ~w(public unlisted)
+  def announce_visibility(_, %{visibility: visibility})
+      when visibility in ~w{public unlisted private direct local},
+      do: visibility
 
-  defp public_announce?(object, _) do
-    Visibility.public?(object)
-  end
+  def announce_visibility(object, _), do: Visibility.get_visibility(object)
 
   @spec get_visibility(map(), map() | nil, Participation.t() | nil) ::
           {String.t() | nil, String.t() | nil}

lib/pleroma/web/common_api/activity_draft.ex

@@ -88,7 +88,7 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do
     |> validate()
   end
 
-  defp listen_object(draft) do
+  defp listen_object(%__MODULE__{} = draft) do
     object =
       draft.params
       |> Map.take([:album, :artist, :title, :length])
@@ -99,34 +99,34 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do
       |> Map.put("cc", draft.cc)
       |> Map.put("actor", draft.user.ap_id)
 
-    %__MODULE__{draft | object: object}
+    %{draft | object: object}
   end
 
-  defp put_params(draft, params) do
+  defp put_params(%__MODULE__{} = draft, params) do
     params = Map.put_new(params, :in_reply_to_status_id, params[:in_reply_to_id])
-    %__MODULE__{draft | params: params}
+    %{draft | params: params}
   end
 
-  defp status(%{params: %{status: status}} = draft) do
-    %__MODULE__{draft | status: String.trim(status)}
+  defp status(%__MODULE__{params: %{status: status}} = draft) do
+    %{draft | status: String.trim(status)}
   end
 
-  defp summary(%{params: params} = draft) do
-    %__MODULE__{draft | summary: Map.get(params, :spoiler_text, "")}
+  defp summary(%__MODULE__{params: params} = draft) do
+    %{draft | summary: Map.get(params, :spoiler_text, "")}
   end
 
-  defp full_payload(%{status: status, summary: summary} = draft) do
+  defp full_payload(%__MODULE__{status: status, summary: summary} = draft) do
     full_payload = String.trim(status <> summary)
 
     case Utils.validate_character_limit(full_payload, draft.attachments) do
-      :ok -> %__MODULE__{draft | full_payload: full_payload}
+      :ok -> %{draft | full_payload: full_payload}
       {:error, message} -> add_error(draft, message)
     end
   end
 
-  defp attachments(%{params: params} = draft) do
+  defp attachments(%__MODULE__{params: params} = draft) do
     attachments = Utils.attachments_from_ids(params, draft.user)
-    draft = %__MODULE__{draft | attachments: attachments}
+    draft = %{draft | attachments: attachments}
 
     case Utils.validate_attachments_count(attachments) do
       :ok -> draft
@@ -134,9 +134,10 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do
     end
   end
 
-  defp in_reply_to(%{params: %{in_reply_to_status_id: ""}} = draft), do: draft
+  defp in_reply_to(%__MODULE__{params: %{in_reply_to_status_id: ""}} = draft), do: draft
 
-  defp in_reply_to(%{params: %{in_reply_to_status_id: id}} = draft) when is_binary(id) do
+  defp in_reply_to(%__MODULE__{params: %{in_reply_to_status_id: id}} = draft)
+       when is_binary(id) do
     # If a post was deleted all its activities (except the newly added Delete) are purged too,
     # thus lookup by Create db ID will yield nil just as if it never existed in the first place.
     #
@@ -148,7 +149,7 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do
     with %Activity{} = activity <- Activity.get_by_id(id),
          true <- Visibility.visible_for_user?(activity, draft.user),
          {_, type} when type in ["Create", "Announce"] <- {:type, activity.data["type"]} do
-      %__MODULE__{draft | in_reply_to: activity}
+      %{draft | in_reply_to: activity}
     else
       nil ->
         add_error(draft, dgettext("errors", "Cannot reply to a deleted status"))
@@ -166,40 +167,43 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do
     end
   end
 
-  defp in_reply_to(%{params: %{in_reply_to_status_id: %Activity{} = in_reply_to}} = draft) do
-    %__MODULE__{draft | in_reply_to: in_reply_to}
+  defp in_reply_to(
+         %__MODULE__{params: %{in_reply_to_status_id: %Activity{} = in_reply_to}} = draft
+       ) do
+    %{draft | in_reply_to: in_reply_to}
   end
 
   defp in_reply_to(draft), do: draft
 
-  defp quote_post(%{params: %{quoted_status_id: id}} = draft) when not_empty_string(id) do
+  defp quote_post(%__MODULE__{params: %{quoted_status_id: id}} = draft)
+       when not_empty_string(id) do
     case Activity.get_by_id_with_object(id) do
       %Activity{} = activity ->
-        %__MODULE__{draft | quote_post: activity}
+        %{draft | quote_post: activity}
 
       _ ->
         draft
     end
   end
 
-  defp quote_post(%{params: %{quote_id: id}} = draft) when not_empty_string(id) do
+  defp quote_post(%__MODULE__{params: %{quote_id: id}} = draft) when not_empty_string(id) do
     quote_post(%{draft | params: Map.put(draft.params, :quoted_status_id, id)})
   end
 
   defp quote_post(draft), do: draft
 
-  defp in_reply_to_conversation(draft) do
+  defp in_reply_to_conversation(%__MODULE__{} = draft) do
     in_reply_to_conversation = Participation.get(draft.params[:in_reply_to_conversation_id])
-    %__MODULE__{draft | in_reply_to_conversation: in_reply_to_conversation}
+    %{draft | in_reply_to_conversation: in_reply_to_conversation}
   end
 
-  defp visibility(%{params: params} = draft) do
+  defp visibility(%__MODULE__{params: params} = draft) do
     case CommonAPI.get_visibility(params, draft.in_reply_to, draft.in_reply_to_conversation) do
       {visibility, "direct"} when visibility != "direct" ->
         add_error(draft, dgettext("errors", "The message visibility must be direct"))
 
       {visibility, _} ->
-        %__MODULE__{draft | visibility: visibility}
+        %{draft | visibility: visibility}
     end
   end
 
@@ -215,7 +219,7 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do
     false
   end
 
-  defp quoting_visibility(%{quote_post: %Activity{}} = draft) do
+  defp quoting_visibility(%__MODULE__{quote_post: %Activity{}} = draft) do
     with %Object{} = object <- Object.normalize(draft.quote_post, fetch: false),
          true <- can_quote?(draft, object, Visibility.get_visibility(object)) do
       draft
@@ -226,24 +230,24 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do
 
   defp quoting_visibility(draft), do: draft
 
-  defp expires_at(draft) do
+  defp expires_at(%__MODULE__{} = draft) do
     case CommonAPI.check_expiry_date(draft.params[:expires_in]) do
-      {:ok, expires_at} -> %__MODULE__{draft | expires_at: expires_at}
+      {:ok, expires_at} -> %{draft | expires_at: expires_at}
       {:error, message} -> add_error(draft, message)
     end
   end
 
-  defp poll(draft) do
+  defp poll(%__MODULE__{} = draft) do
     case Utils.make_poll_data(draft.params) do
       {:ok, {poll, poll_emoji}} ->
-        %__MODULE__{draft | extra: poll, emoji: Map.merge(draft.emoji, poll_emoji)}
+        %{draft | extra: poll, emoji: Map.merge(draft.emoji, poll_emoji)}
 
       {:error, message} ->
         add_error(draft, message)
     end
   end
 
-  defp content(%{mentions: mentions} = draft) do
+  defp content(%__MODULE__{mentions: mentions} = draft) do
     {content_html, mentioned_users, tags} = Utils.make_content_html(draft)
 
     mentioned_ap_ids =
@@ -254,25 +258,25 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do
       |> Kernel.++(mentioned_ap_ids)
       |> Utils.get_addressed_users(draft.params[:to])
 
-    %__MODULE__{draft | content_html: content_html, mentions: mentions, tags: tags}
+    %{draft | content_html: content_html, mentions: mentions, tags: tags}
   end
 
-  defp to_and_cc(draft) do
+  defp to_and_cc(%__MODULE__{} = draft) do
     {to, cc} = Utils.get_to_and_cc(draft)
-    %__MODULE__{draft | to: to, cc: cc}
+    %{draft | to: to, cc: cc}
   end
 
-  defp context(draft) do
+  defp context(%__MODULE__{} = draft) do
     context = Utils.make_context(draft.in_reply_to, draft.in_reply_to_conversation)
-    %__MODULE__{draft | context: context}
+    %{draft | context: context}
   end
 
-  defp sensitive(draft) do
+  defp sensitive(%__MODULE__{} = draft) do
     sensitive = draft.params[:sensitive]
-    %__MODULE__{draft | sensitive: sensitive}
+    %{draft | sensitive: sensitive}
   end
 
-  defp language(draft) do
+  defp language(%__MODULE__{} = draft) do
     language =
       with language <- draft.params[:language],
            true <- good_locale_code?(language) do
@@ -281,10 +285,10 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do
         _ -> LanguageDetector.detect(draft.content_html <> " " <> draft.summary)
       end
 
-    %__MODULE__{draft | language: language}
+    %{draft | language: language}
   end
 
-  defp object(draft) do
+  defp object(%__MODULE__{} = draft) do
     emoji = Map.merge(Pleroma.Emoji.Formatter.get_emoji_map(draft.full_payload), draft.emoji)
 
     # Sometimes people create posts with subject containing emoji,
@@ -313,6 +317,7 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do
 
     emoji = Map.merge(emoji, summary_emoji)
 
+    media_type = Utils.get_content_type(draft.params[:content_type])
     {:ok, note_data, _meta} = Builder.note(draft)
 
     object =
@@ -320,20 +325,24 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do
       |> Map.put("emoji", emoji)
       |> Map.put("source", %{
         "content" => draft.status,
-        "mediaType" => Utils.get_content_type(draft.params[:content_type])
+        "mediaType" => media_type
       })
+      |> maybe_put("htmlMfm", true, media_type == "text/x.misskeymarkdown")
       |> Map.put("generator", draft.params[:generator])
       |> Map.put("language", draft.language)
 
-    %__MODULE__{draft | object: object}
+    %{draft | object: object}
   end
 
-  defp preview?(draft) do
+  defp maybe_put(map, key, value, true), do: Map.put(map, key, value)
+  defp maybe_put(map, _key, _value, _condition), do: map
+
+  defp preview?(%__MODULE__{} = draft) do
     preview? = Pleroma.Web.Utils.Params.truthy_param?(draft.params[:preview])
-    %__MODULE__{draft | preview?: preview?}
+    %{draft | preview?: preview?}
   end
 
-  defp changes(draft) do
+  defp changes(%__MODULE__{} = draft) do
     direct? = draft.visibility == "direct"
     additional = %{"cc" => draft.cc, "directMessage" => direct?}
 
@@ -353,14 +362,14 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do
       }
       |> Utils.maybe_add_list_data(draft.user, draft.visibility)
 
-    %__MODULE__{draft | changes: changes}
+    %{draft | changes: changes}
   end
 
   defp with_valid(%{valid?: true} = draft, func), do: func.(draft)
   defp with_valid(draft, _func), do: draft
 
-  defp add_error(draft, message) do
-    %__MODULE__{draft | valid?: false, errors: [message | draft.errors]}
+  defp add_error(%__MODULE__{} = draft, message) do
+    %{draft | valid?: false, errors: [message | draft.errors]}
   end
 
   defp validate(%{valid?: true} = draft), do: {:ok, draft}

lib/pleroma/web/common_api/utils.ex

@@ -75,48 +75,70 @@ defmodule Pleroma.Web.CommonAPI.Utils do
     {Enum.map(participation.recipients, & &1.ap_id), []}
   end
 
-  def get_to_and_cc(%{visibility: visibility} = draft) when visibility in ["public", "local"] do
-    to =
-      case visibility do
-        "public" -> [Pleroma.Constants.as_public() | draft.mentions]
-        "local" -> [Utils.as_local_public() | draft.mentions]
+  def get_to_and_cc(%{visibility: visibility} = draft) do
+    # If the OP is a DM already, add the implicit actor
+    mentions =
+      if visibility == "direct" && draft.in_reply_to && Visibility.direct?(draft.in_reply_to) do
+        Enum.uniq([draft.in_reply_to.data["actor"] | draft.mentions])
+      else
+        draft.mentions
       end
 
-    cc = [draft.user.follower_address]
-
-    if draft.in_reply_to do
-      {Enum.uniq([draft.in_reply_to.data["actor"] | to]), cc}
-    else
-      {to, cc}
-    end
+    get_to_and_cc_for_visibility(
+      visibility,
+      draft.user.follower_address,
+      draft.in_reply_to && draft.in_reply_to.data["actor"],
+      mentions
+    )
   end
 
-  def get_to_and_cc(%{visibility: "unlisted"} = draft) do
-    to = [draft.user.follower_address | draft.mentions]
-    cc = [Pleroma.Constants.as_public()]
+  def get_to_and_cc_for_visibility("public", follower_collection, parent_actor, mentions) do
+    scope_addr = Pleroma.Constants.as_public()
 
-    if draft.in_reply_to do
-      {Enum.uniq([draft.in_reply_to.data["actor"] | to]), cc}
-    else
-      {to, cc}
-    end
+    to =
+      if parent_actor,
+        do: Enum.uniq([parent_actor, scope_addr | mentions]),
+        else: [scope_addr | mentions]
+
+    {to, [follower_collection]}
   end
 
-  def get_to_and_cc(%{visibility: "private"} = draft) do
-    {to, cc} = get_to_and_cc(struct(draft, visibility: "direct"))
-    {[draft.user.follower_address | to], cc}
+  def get_to_and_cc_for_visibility("local", follower_collection, parent_actor, mentions) do
+    recipients =
+      if parent_actor,
+        do: Enum.uniq([parent_actor | mentions]),
+        else: mentions
+
+    to = [
+      Utils.as_local_public()
+      | Enum.filter(recipients, fn addr ->
+          String.starts_with?(addr, Pleroma.Web.Endpoint.url() <> "/")
+        end)
+    ]
+
+    {to, [follower_collection]}
   end
 
-  def get_to_and_cc(%{visibility: "direct"} = draft) do
-    # If the OP is a DM already, add the implicit actor.
-    if draft.in_reply_to && Visibility.direct?(draft.in_reply_to) do
-      {Enum.uniq([draft.in_reply_to.data["actor"] | draft.mentions]), []}
-    else
-      {draft.mentions, []}
-    end
+  def get_to_and_cc_for_visibility("unlisted", follower_collection, parent_actor, mentions) do
+    to =
+      if parent_actor,
+        do: Enum.uniq([parent_actor, follower_collection | mentions]),
+        else: [follower_collection | mentions]
+
+    {to, [Pleroma.Constants.as_public()]}
   end
 
-  def get_to_and_cc(%{visibility: {:list, _}, mentions: mentions}), do: {mentions, []}
+  def get_to_and_cc_for_visibility("private", follower_collection, _, mentions) do
+    {[follower_collection | mentions], []}
+  end
+
+  def get_to_and_cc_for_visibility("direct", _, _, mentions) do
+    {mentions, []}
+  end
+
+  def get_to_and_cc_for_visibility({:list, _}, _, _, mentions) do
+    {mentions, []}
+  end
 
   def get_addressed_users(_, to) when is_list(to) do
     User.get_ap_ids_by_nicknames(to)
@@ -300,6 +322,14 @@ defmodule Pleroma.Web.CommonAPI.Utils do
     |> Formatter.linkify(options)
   end
 
+  def format_input(text, "text/x.misskeymarkdown", options) do
+    text
+    |> Formatter.markdown_to_html(%{breaks: true})
+    |> safe_mfm_to_html()
+    |> Formatter.linkify(options)
+    |> Formatter.html_escape("text/x.misskeymarkdown")
+  end
+
   def format_input(text, "text/markdown", options) do
     text
     |> Formatter.mentions_escape(options)
@@ -308,6 +338,16 @@ defmodule Pleroma.Web.CommonAPI.Utils do
     |> Formatter.html_escape("text/html")
   end
 
+  defp safe_mfm_to_html(html) do
+    html
+    |> MfmParser.Parser.parse()
+    |> MfmParser.Encoder.to_html()
+  rescue
+    _ -> html
+  catch
+    _, _ -> html
+  end
+
   def format_naive_asctime(date) do
     date |> DateTime.from_naive!("Etc/UTC") |> format_asctime
   end

lib/pleroma/web/embed_controller.ex

@@ -20,6 +20,7 @@ defmodule Pleroma.Web.EmbedController do
       conn
       |> delete_resp_header("x-frame-options")
       |> delete_resp_header("content-security-policy")
+      |> put_layout({Pleroma.Web.LayoutView, :embed})
       |> render("show.html",
         activity: activity,
         author: User.sanitize_html(author),

lib/pleroma/web/endpoint.ex

@@ -46,8 +46,10 @@ defmodule Pleroma.Web.Endpoint do
   plug(Pleroma.Web.Plugs.HTTPSecurityPlug)
   plug(Pleroma.Web.Plugs.UploadedMedia)
 
-  @static_cache_control "public, max-age=1209600"
+  @static_cache_control "public, max-age=1209600, immutable"
   @static_cache_disabled "public, no-cache"
+  # cache for a day
+  @favicon_cache_control "public, max=age=86400, immutable"
 
   # InstanceStatic needs to be before Plug.Static to be able to override shipped-static files
   # If you're adding new paths to `only:` you'll need to configure them in InstanceStatic as well
@@ -64,6 +66,15 @@ defmodule Pleroma.Web.Endpoint do
     }
   )
 
+  plug(Pleroma.Web.Plugs.FaviconPlug,
+    at: "/",
+    only: ["favicon.png"],
+    cache_control_for_etags: @favicon_cache_control,
+    headers: %{
+      "cache-control" => @favicon_cache_control
+    }
+  )
+
   plug(Pleroma.Web.Plugs.InstanceStatic,
     at: "/",
     gzip: true,

lib/pleroma/web/fallback/redirect_controller.ex

@@ -29,9 +29,18 @@ defmodule Pleroma.Web.Fallback.RedirectController do
     )
   end
 
-  def live_dashboard(conn, _params) do
+  def live_dashboard(conn, %{"path" => path}) do
+    query_params = conn.query_string
+
+    redirect_path =
+      if query_params == "" do
+        "/pleroma/live_dashboard/#{path}"
+      else
+        "/pleroma/live_dashboard/#{path}?#{query_params}"
+      end
+
     conn
-    |> redirect(to: "/pleroma/live_dashboard")
+    |> redirect(to: redirect_path)
   end
 
   def redirector(conn, _params, code \\ 200) do

lib/pleroma/web/federator.ex

@@ -11,6 +11,7 @@ defmodule Pleroma.Web.Federator do
   alias Pleroma.Web.ActivityPub.Utils
   alias Pleroma.Workers.PublisherWorker
   alias Pleroma.Workers.ReceiverWorker
+  alias Pleroma.Workers.SignatureRetryWorker
 
   require Logger
 
@@ -35,12 +36,21 @@ defmodule Pleroma.Web.Federator do
   end
 
   # Client API
-  def incoming_ap_doc(%{params: params, req_headers: req_headers}) do
-    ReceiverWorker.new(
+  def incoming_failed_signature_ap_doc(%{
+        method: method,
+        params: params,
+        req_headers: req_headers,
+        request_path: request_path,
+        query_string: query_string
+      }) do
+    SignatureRetryWorker.new(
       %{
-        "op" => "incoming_ap_doc",
+        "op" => "incoming_failed_signature_ap_doc",
+        "method" => method,
         "req_headers" => req_headers,
         "params" => params,
+        "request_path" => request_path,
+        "query_string" => query_string,
         "timeout" => :timer.seconds(20)
       },
       priority: 2

lib/pleroma/web/mastodon_api/controllers/account_controller.ex

@@ -26,7 +26,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do
   alias Pleroma.Web.OAuth.OAuthController
   alias Pleroma.Web.Plugs.OAuthScopesPlug
   alias Pleroma.Web.Plugs.RateLimiter
-  alias Pleroma.Web.TwitterAPI.TwitterAPI
+  alias Pleroma.Web.Registration
   alias Pleroma.Web.Utils.Params
 
   plug(Pleroma.Web.ApiSpec.CastAndValidate, replace_params: false)
@@ -111,8 +111,8 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do
         _params
       ) do
     with :ok <- validate_email_param(params),
-         :ok <- TwitterAPI.validate_captcha(app, params),
-         {:ok, user} <- TwitterAPI.register_user(params),
+         :ok <- Registration.validate_captcha(app, params),
+         {:ok, user} <- Registration.register_user(params),
          {_, {:ok, token}} <-
            {:login, OAuthController.login(user, app, app.scopes)} do
       OAuthController.after_token_exchange(conn, %{user: user, token: token})

lib/pleroma/web/mastodon_api/controllers/auth_controller.ex

@@ -7,7 +7,7 @@ defmodule Pleroma.Web.MastodonAPI.AuthController do
 
   import Pleroma.Web.ControllerHelper, only: [json_response: 3]
 
-  alias Pleroma.Web.TwitterAPI.TwitterAPI
+  alias Pleroma.Web.Registration
 
   action_fallback(Pleroma.Web.MastodonAPI.FallbackController)
 
@@ -17,7 +17,7 @@ defmodule Pleroma.Web.MastodonAPI.AuthController do
   def password_reset(conn, params) do
     nickname_or_email = params["email"] || params["nickname"]
 
-    TwitterAPI.password_reset(nickname_or_email)
+    Registration.password_reset(nickname_or_email)
 
     json_response(conn, :no_content, "")
   end

lib/pleroma/web/mastodon_api/controllers/list_controller.ex

@@ -28,27 +28,27 @@ defmodule Pleroma.Web.MastodonAPI.ListController do
 
   # POST /api/v1/lists
   def create(
-        %{assigns: %{user: user}, private: %{open_api_spex: %{body_params: %{title: title}}}} =
+        %{assigns: %{user: user}, private: %{open_api_spex: %{body_params: params}}} =
           conn,
         _
       ) do
-    with {:ok, %Pleroma.List{} = list} <- Pleroma.List.create(title, user) do
+    with {:ok, %Pleroma.List{} = list} <- Pleroma.List.create(params, user) do
       render(conn, "show.json", list: list)
     end
   end
 
-  # GET /api/v1/lists/:idOB
+  # GET /api/v1/lists/:id
   def show(%{assigns: %{list: list}} = conn, _) do
     render(conn, "show.json", list: list)
   end
 
   # PUT /api/v1/lists/:id
   def update(
-        %{assigns: %{list: list}, private: %{open_api_spex: %{body_params: %{title: title}}}} =
+        %{assigns: %{list: list}, private: %{open_api_spex: %{body_params: params}}} =
           conn,
         _
       ) do
-    with {:ok, list} <- Pleroma.List.rename(list, title) do
+    with {:ok, list} <- Pleroma.List.update(list, params) do
       render(conn, "show.json", list: list)
     end
   end

lib/pleroma/web/mastodon_api/controllers/status_controller.ex

@@ -81,10 +81,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do
 
   plug(OAuthScopesPlug, %{scopes: ["write:accounts"]} when action in [:pin, :unpin])
 
-  # Note: scope not present in Mastodon: read:bookmarks
   plug(OAuthScopesPlug, %{scopes: ["read:bookmarks"]} when action == :bookmarks)
 
-  # Note: scope not present in Mastodon: write:bookmarks
   plug(
     OAuthScopesPlug,
     %{scopes: ["write:bookmarks"]} when action in [:bookmark, :unbookmark]

lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex

@@ -45,6 +45,10 @@ defmodule Pleroma.Web.MastodonAPI.TimelineController do
       |> User.followed_hashtags()
       |> Enum.map(& &1.id)
 
+    excluded_list_members =
+      user
+      |> Pleroma.List.get_exclusive_list_members()
+
     params =
       params
       |> Map.put(:type, ["Create", "Announce"])
@@ -58,7 +62,7 @@ defmodule Pleroma.Web.MastodonAPI.TimelineController do
       |> Map.delete(:local)
 
     activities =
-      [user.ap_id | User.following(user)]
+      [user.ap_id | User.following(user) -- excluded_list_members]
       |> ActivityPub.fetch_activities(params)
       |> Enum.reverse()
 

lib/pleroma/web/mastodon_api/views/list_view.ex

@@ -13,7 +13,8 @@ defmodule Pleroma.Web.MastodonAPI.ListView do
   def render("show.json", %{list: list}) do
     %{
       id: to_string(list.id),
-      title: list.title
+      title: list.title,
+      exclusive: list.exclusive
     }
   end
 end

lib/pleroma/web/mastodon_api/views/poll_view.ex

@@ -71,6 +71,8 @@ defmodule Pleroma.Web.MastodonAPI.PollView do
     end)
   end
 
+  defp voters_count(%{data: %{"votersCount" => voters}}) when is_integer(voters), do: voters
+
   defp voters_count(%{data: %{"voters" => [_ | _] = voters}}) do
     length(voters)
   end

lib/pleroma/web/o_auth/password_controller.ex

@@ -2,7 +2,7 @@
 # Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
-defmodule Pleroma.Web.TwitterAPI.PasswordController do
+defmodule Pleroma.Web.OAuth.PasswordController do
   @moduledoc """
   The module contains functions for password reset.
   """
@@ -16,7 +16,7 @@ defmodule Pleroma.Web.TwitterAPI.PasswordController do
   alias Pleroma.PasswordResetToken
   alias Pleroma.Repo
   alias Pleroma.User
-  alias Pleroma.Web.TwitterAPI.TwitterAPI
+  alias Pleroma.Web.Registration
 
   plug(Pleroma.Web.Plugs.RateLimiter, [name: :request] when action == :request)
 
@@ -24,7 +24,7 @@ defmodule Pleroma.Web.TwitterAPI.PasswordController do
   def request(conn, params) do
     nickname_or_email = params["email"] || params["nickname"]
 
-    TwitterAPI.password_reset(nickname_or_email)
+    Registration.password_reset(nickname_or_email)
 
     json_response(conn, :no_content, "")
   end

lib/pleroma/web/o_auth/password_view.ex

@@ -2,7 +2,7 @@
 # Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
-defmodule Pleroma.Web.TwitterAPI.PasswordView do
+defmodule Pleroma.Web.OAuth.PasswordView do
   use Pleroma.Web, :view
   import Phoenix.HTML.Form
   alias Pleroma.Web.Gettext

lib/pleroma/web/o_auth/token_controller.ex

@@ -2,13 +2,13 @@
 # Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
-defmodule Pleroma.Web.TwitterAPI.Controller do
+defmodule Pleroma.Web.OAuth.TokenController do
   use Pleroma.Web, :controller
 
   alias Pleroma.User
   alias Pleroma.Web.OAuth.Token
+  alias Pleroma.Web.OAuth.TokenView
   alias Pleroma.Web.Plugs.OAuthScopesPlug
-  alias Pleroma.Web.TwitterAPI.TokenView
 
   require Logger
 

lib/pleroma/web/o_auth/token_view.ex

@@ -2,12 +2,12 @@
 # Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
-defmodule Pleroma.Web.TwitterAPI.TokenView do
+defmodule Pleroma.Web.OAuth.TokenView do
   use Pleroma.Web, :view
 
   def render("index.json", %{tokens: tokens}) do
     tokens
-    |> render_many(Pleroma.Web.TwitterAPI.TokenView, "show.json")
+    |> render_many(Pleroma.Web.OAuth.TokenView, "show.json")
     |> Enum.filter(&Enum.any?/1)
   end
 

lib/pleroma/web/pleroma_api/controllers/bookmark_folder_controller.ex

@@ -10,10 +10,8 @@ defmodule Pleroma.Web.PleromaAPI.BookmarkFolderController do
 
   plug(Pleroma.Web.ApiSpec.CastAndValidate)
 
-  # Note: scope not present in Mastodon: read:bookmarks
   plug(OAuthScopesPlug, %{scopes: ["read:bookmarks"]} when action == :index)
 
-  # Note: scope not present in Mastodon: write:bookmarks
   plug(
     OAuthScopesPlug,
     %{scopes: ["write:bookmarks"]} when action in [:create, :update, :delete]

lib/pleroma/web/pleroma_api/controllers/password_controller.ex

@@ -0,0 +1,52 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.PleromaAPI.PasswordController do
+  @moduledoc """
+  The module contains functions for password reset.
+  """
+
+  use Pleroma.Web, :controller
+
+  require Logger
+
+  import Pleroma.Web.ControllerHelper, only: [json_response: 3]
+
+  alias Pleroma.PasswordResetToken
+  alias Pleroma.Repo
+  alias Pleroma.User
+  alias Pleroma.Web.Registration
+
+  plug(Pleroma.Web.Plugs.RateLimiter, [name: :request] when action == :request)
+
+  @doc "POST /auth/password"
+  def request(conn, params) do
+    nickname_or_email = params["email"] || params["nickname"]
+
+    Registration.password_reset(nickname_or_email)
+
+    json_response(conn, :no_content, "")
+  end
+
+  def reset(conn, %{"token" => token}) do
+    with %{used: false} = token <- Repo.get_by(PasswordResetToken, %{token: token}),
+         false <- PasswordResetToken.expired?(token),
+         %User{} = user <- User.get_cached_by_id(token.user_id) do
+      render(conn, "reset.html", %{
+        token: token,
+        user: user
+      })
+    else
+      _e -> render(conn, "invalid_token.html")
+    end
+  end
+
+  def do_reset(conn, %{"data" => data}) do
+    with {:ok, _} <- PasswordResetToken.reset_password(data["token"], data) do
+      render(conn, "reset_success.html")
+    else
+      _e -> render(conn, "reset_failed.html")
+    end
+  end
+end

lib/pleroma/web/pleroma_api/controllers/token_controller.ex

@@ -0,0 +1,59 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.PleromaAPI.TokenController do
+  use Pleroma.Web, :controller
+
+  alias Pleroma.User
+  alias Pleroma.Web.OAuth.Token
+  alias Pleroma.Web.PleromaAPI.TokenView
+  alias Pleroma.Web.Plugs.OAuthScopesPlug
+
+  require Logger
+
+  plug(:skip_auth when action == :confirm_email)
+  plug(:skip_plug, OAuthScopesPlug when action in [:oauth_tokens, :revoke_token])
+
+  action_fallback(:errors)
+
+  def confirm_email(conn, %{"user_id" => uid, "token" => token}) do
+    with %User{} = user <- User.get_cached_by_id(uid),
+         true <- user.local and !user.is_confirmed and user.confirmation_token == token,
+         {:ok, _} <- User.confirm(user) do
+      redirect(conn, to: "/")
+    end
+  end
+
+  def oauth_tokens(%{assigns: %{user: user}} = conn, _params) do
+    with oauth_tokens <- Token.get_user_tokens(user) do
+      conn
+      |> put_view(TokenView)
+      |> render("index.json", %{tokens: oauth_tokens})
+    end
+  end
+
+  def revoke_token(%{assigns: %{user: user}} = conn, %{"id" => id} = _params) do
+    Token.delete_user_token(user, id)
+
+    json_reply(conn, 201, "")
+  end
+
+  defp errors(conn, {:param_cast, _}) do
+    conn
+    |> put_status(400)
+    |> json("Invalid parameters")
+  end
+
+  defp errors(conn, _) do
+    conn
+    |> put_status(500)
+    |> json("Something went wrong")
+  end
+
+  defp json_reply(conn, status, json) do
+    conn
+    |> put_resp_content_type("application/json")
+    |> send_resp(status, json)
+  end
+end

lib/pleroma/web/pleroma_api/controllers/util_controller.ex

@@ -2,12 +2,11 @@
 # Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
-defmodule Pleroma.Web.TwitterAPI.UtilController do
+defmodule Pleroma.Web.PleromaAPI.UtilController do
   use Pleroma.Web, :controller
 
   require Logger
 
-  alias Pleroma.Activity
   alias Pleroma.Config
   alias Pleroma.Emoji
   alias Pleroma.Healthcheck
@@ -17,19 +16,8 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do
   alias Pleroma.Web.Auth.WrapperAuthenticator, as: Authenticator
   alias Pleroma.Web.CommonAPI
   alias Pleroma.Web.Plugs.OAuthScopesPlug
-  alias Pleroma.Web.WebFinger
 
-  plug(
-    Pleroma.Web.ApiSpec.CastAndValidate,
-    [replace_params: false]
-    when action != :remote_subscribe and action != :show_subscribe_form
-  )
-
-  plug(
-    Pleroma.Web.Plugs.FederatingPlug
-    when action == :remote_subscribe
-    when action == :show_subscribe_form
-  )
+  plug(Pleroma.Web.ApiSpec.CastAndValidate, replace_params: false)
 
   plug(
     OAuthScopesPlug,
@@ -54,125 +42,7 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do
          ]
   )
 
-  defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.TwitterUtilOperation
-
-  def show_subscribe_form(conn, %{"nickname" => nick}) do
-    with %User{} = user <- User.get_cached_by_nickname(nick),
-         avatar = User.avatar_url(user) do
-      conn
-      |> render("subscribe.html", %{nickname: nick, avatar: avatar, error: false})
-    else
-      _e ->
-        render(conn, "subscribe.html", %{
-          nickname: nick,
-          avatar: nil,
-          error:
-            Pleroma.Web.Gettext.dpgettext(
-              "static_pages",
-              "remote follow error message - user not found",
-              "Could not find user"
-            )
-        })
-    end
-  end
-
-  def show_subscribe_form(conn, %{"status_id" => id}) do
-    with %Activity{} = activity <- Activity.get_by_id(id),
-         {:ok, ap_id} <- get_ap_id(activity),
-         %User{} = user <- User.get_cached_by_ap_id(activity.actor),
-         avatar = User.avatar_url(user) do
-      conn
-      |> render("status_interact.html", %{
-        status_link: ap_id,
-        status_id: id,
-        nickname: user.nickname,
-        avatar: avatar,
-        error: false
-      })
-    else
-      _e ->
-        render(conn, "status_interact.html", %{
-          status_id: id,
-          avatar: nil,
-          error:
-            Pleroma.Web.Gettext.dpgettext(
-              "static_pages",
-              "status interact error message - status not found",
-              "Could not find status"
-            )
-        })
-    end
-  end
-
-  def remote_subscribe(conn, %{"nickname" => nick, "profile" => _}) do
-    show_subscribe_form(conn, %{"nickname" => nick})
-  end
-
-  def remote_subscribe(conn, %{"status_id" => id, "profile" => _}) do
-    show_subscribe_form(conn, %{"status_id" => id})
-  end
-
-  def remote_subscribe(conn, %{"user" => %{"nickname" => nick, "profile" => profile}}) do
-    with {:ok, %{"subscribe_address" => template}} <- WebFinger.finger(profile),
-         %User{ap_id: ap_id} <- User.get_cached_by_nickname(nick) do
-      conn
-      |> Phoenix.Controller.redirect(external: String.replace(template, "{uri}", ap_id))
-    else
-      _e ->
-        render(conn, "subscribe.html", %{
-          nickname: nick,
-          avatar: nil,
-          error:
-            Pleroma.Web.Gettext.dpgettext(
-              "static_pages",
-              "remote follow error message - unknown error",
-              "Something went wrong."
-            )
-        })
-    end
-  end
-
-  def remote_subscribe(conn, %{"status" => %{"status_id" => id, "profile" => profile}}) do
-    with {:ok, %{"subscribe_address" => template}} <- WebFinger.finger(profile),
-         %Activity{} = activity <- Activity.get_by_id(id),
-         {:ok, ap_id} <- get_ap_id(activity) do
-      conn
-      |> Phoenix.Controller.redirect(external: String.replace(template, "{uri}", ap_id))
-    else
-      _e ->
-        render(conn, "status_interact.html", %{
-          status_id: id,
-          avatar: nil,
-          error:
-            Pleroma.Web.Gettext.dpgettext(
-              "static_pages",
-              "status interact error message - unknown error",
-              "Something went wrong."
-            )
-        })
-    end
-  end
-
-  def remote_interaction(
-        %{private: %{open_api_spex: %{body_params: %{ap_id: ap_id, profile: profile}}}} = conn,
-        _params
-      ) do
-    with {:ok, %{"subscribe_address" => template}} <- WebFinger.finger(profile) do
-      conn
-      |> json(%{url: String.replace(template, "{uri}", ap_id)})
-    else
-      _e -> json(conn, %{error: "Couldn't find user"})
-    end
-  end
-
-  defp get_ap_id(activity) do
-    object = Pleroma.Object.normalize(activity, fetch: false)
-
-    case object do
-      %{data: %{"id" => ap_id}} -> {:ok, ap_id}
-      _ -> {:no_ap_id, nil}
-    end
-  end
+  defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.PleromaUtilOperation
 
   def frontend_configurations(conn, _params) do
     render(conn, "frontend_configurations.json")

lib/pleroma/web/pleroma_api/views/bookmark_folder_view.ex

@@ -9,11 +9,13 @@ defmodule Pleroma.Web.PleromaAPI.BookmarkFolderView do
   alias Pleroma.Emoji
 
   def render("show.json", %{folder: %BookmarkFolder{} = folder}) do
+    {emoji, emoji_url} = get_emoji(folder.emoji)
+
     %{
       id: folder.id |> to_string(),
       name: folder.name,
-      emoji: folder.emoji,
-      emoji_url: get_emoji_url(folder.emoji)
+      emoji: emoji,
+      emoji_url: emoji_url
     }
   end
 
@@ -21,20 +23,15 @@ defmodule Pleroma.Web.PleromaAPI.BookmarkFolderView do
     render_many(folders, __MODULE__, "show.json", Map.delete(opts, :folders))
   end
 
-  defp get_emoji_url(nil) do
-    nil
-  end
+  defp get_emoji(nil), do: {nil, nil}
 
-  defp get_emoji_url(emoji) do
+  defp get_emoji(emoji) do
     if Emoji.unicode?(emoji) do
-      nil
+      {emoji, nil}
     else
-      emoji = Emoji.get(emoji)
-
-      if emoji != nil do
-        Emoji.local_url(emoji.file)
-      else
-        nil
+      case Emoji.get(emoji) do
+        nil -> {nil, nil}
+        emoji_data -> {emoji, Emoji.local_url(emoji_data.file)}
       end
     end
   end

lib/pleroma/web/pleroma_api/views/util_view.ex

@@ -0,0 +1,13 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.PleromaAPI.UtilView do
+  use Pleroma.Web, :view
+  alias Pleroma.Config
+
+  def render("frontend_configurations.json", _) do
+    Config.get(:frontend_configurations, %{})
+    |> Enum.into(%{})
+  end
+end

lib/pleroma/web/plugs/favicon_plug.ex

@@ -0,0 +1,71 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2026 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.Plugs.FaviconPlug do
+  @behaviour Plug
+
+  @moduledoc """
+  This is a shim to call `Plug.Static` but with runtime `from` configuration for instance favicon.
+
+  Serves default or custom favicon.png with cacheable cache-control.
+  """
+
+  import Plug.Conn, only: [put_resp_header: 3, send_resp: 3, halt: 1]
+
+  require Logger
+
+  def init(opts) do
+    opts
+    |> Keyword.put(:from, "__unconfigured_favicon_static_plug")
+    |> Plug.Static.init()
+  end
+
+  def call(%{request_path: "/favicon.png"} = conn, opts) do
+    case find_favicon_dir() do
+      {:ok, dir} ->
+        call_static(conn, opts, dir)
+
+      :error ->
+        # Favicon should always be available and this should never occur.
+        # If it does, halt the pipeline before having unintended side-effects.
+        Logger.error("No favicon.png found! Is the default favicon deleted?")
+
+        conn
+        |> send_resp(404, "Not found")
+        |> halt()
+    end
+  end
+
+  def call(conn, _) do
+    conn
+  end
+
+  defp find_favicon_dir do
+    instance_dir = Pleroma.Config.get([:instance, :static_dir], "instance/static")
+    instance_path = Path.join(instance_dir, "favicon.png")
+
+    priv_dir = Application.app_dir(:pleroma, "priv/static")
+    priv_path = Path.join(priv_dir, "favicon.png")
+
+    cond do
+      File.exists?(instance_path) -> {:ok, instance_dir}
+      File.exists?(priv_path) -> {:ok, priv_dir}
+      true -> :error
+    end
+  end
+
+  defp call_static(conn, opts, from) do
+    opts =
+      opts
+      |> Map.put(:from, from)
+      |> Map.put(:content_types, false)
+
+    conn = set_content_type(conn)
+    Plug.Static.call(conn, opts)
+  end
+
+  defp set_content_type(conn) do
+    put_resp_header(conn, "content-type", "image/png")
+  end
+end

lib/pleroma/web/plugs/instance_static.ex

@@ -4,7 +4,7 @@
 
 defmodule Pleroma.Web.Plugs.InstanceStatic do
   require Pleroma.Constants
-  import Plug.Conn, only: [put_resp_header: 3]
+  import Plug.Conn, only: [put_resp_header: 3, put_status: 2, send_resp: 3, halt: 1]
 
   @moduledoc """
   This is a shim to call `Plug.Static` but with runtime `from` configuration.
@@ -51,10 +51,37 @@ defmodule Pleroma.Web.Plugs.InstanceStatic do
       |> Map.put(:from, from)
       |> Map.put(:content_types, false)
 
-    conn = set_content_type(conn, conn.request_path)
+    conn =
+      conn
+      |> set_content_type(conn.request_path)
+      |> Plug.Static.call(opts)
 
-    # Call Plug.Static with our sanitized content-type
-    Plug.Static.call(conn, opts)
+    if conn.halted do
+      conn
+    else
+      path = String.trim_leading(conn.request_path, "/")
+
+      if not File.exists?(file_path(path)) do
+        conn
+        |> put_status(:not_found)
+        |> send_404()
+        |> halt()
+      else
+        conn
+      end
+    end
+  end
+
+  defp send_404(conn) do
+    if String.ends_with?(String.downcase(conn.request_path), ".json") do
+      conn
+      |> put_resp_header("content-type", "application/json")
+      |> send_resp(404, Jason.encode!(%{error: "not found"}))
+    else
+      conn
+      |> put_resp_header("content-type", "text/plain")
+      |> send_resp(404, "Not found")
+    end
   end
 
   defp set_content_type(conn, "/emoji/" <> filepath) do

lib/pleroma/web/plugs/mapped_signature_to_identity_plug.ex

@@ -32,8 +32,8 @@ defmodule Pleroma.Web.Plugs.MappedSignatureToIdentityPlug do
       # remove me once testsuite uses mapped capabilities instead of what we do now
       {:user, nil} ->
         Logger.debug("Failed to map identity from signature (lookup failure)")
-        Logger.debug("key_id=#{inspect(key_id_from_conn(conn))}, actor=#{actor}")
-        conn
+        Logger.debug("key_id=#{inspect(key_id_from_conn(conn))}, actor=#{inspect(actor)}")
+        assign(conn, :valid_signature, false)
     end
   end
 

lib/pleroma/web/plugs/rate_limiter.ex

@@ -67,6 +67,8 @@ defmodule Pleroma.Web.Plugs.RateLimiter do
   import Plug.Conn
 
   alias Pleroma.Config
+  alias Pleroma.Config.Holder
+  alias Pleroma.EctoType.Config.RateLimit
   alias Pleroma.User
   alias Pleroma.Web.Plugs.RateLimiter.LimiterSupervisor
 
@@ -143,7 +145,7 @@ defmodule Pleroma.Web.Plugs.RateLimiter do
 
   def action_settings(plug_opts) do
     with limiter_name when is_atom(limiter_name) <- plug_opts[:name],
-         limits when not is_nil(limits) <- Config.get([:rate_limit, limiter_name]) do
+         {:ok, limits} <- fetch_and_normalize_limits(limiter_name) do
       bucket_name_root = Keyword.get(plug_opts, :bucket_name, limiter_name)
 
       %{
@@ -151,6 +153,72 @@ defmodule Pleroma.Web.Plugs.RateLimiter do
         limits: limits,
         opts: plug_opts
       }
+    else
+      :disabled -> nil
+    end
+  end
+
+  defp fetch_and_normalize_limits(limiter_name) do
+    limits = Config.get([:rate_limit, limiter_name])
+
+    case normalize_limits(limits) do
+      {:ok, limits} ->
+        {:ok, limits}
+
+      :disabled ->
+        :disabled
+
+      :error ->
+        default_limits =
+          Holder.default_config(:pleroma, :rate_limit)
+          |> get_default_limits(limiter_name)
+
+        case normalize_limits(default_limits) do
+          {:ok, normalized_limits} ->
+            warn_invalid_limits_once(limiter_name, limits)
+            {:ok, normalized_limits}
+
+          _ ->
+            warn_invalid_limits_once(limiter_name, limits)
+            :disabled
+        end
+    end
+  end
+
+  defp get_default_limits(%{} = rate_limit, limiter_name), do: Map.get(rate_limit, limiter_name)
+
+  defp get_default_limits(rate_limit, limiter_name) when is_list(rate_limit) do
+    if Keyword.keyword?(rate_limit) do
+      Keyword.get(rate_limit, limiter_name)
+    else
+      nil
+    end
+  end
+
+  defp get_default_limits(_, _), do: nil
+
+  @invalid_limits_warned_key {__MODULE__, :invalid_limits_warned}
+
+  defp warn_invalid_limits_once(limiter_name, limits) do
+    warned = :persistent_term.get(@invalid_limits_warned_key, MapSet.new())
+
+    if MapSet.member?(warned, limiter_name) do
+      :ok
+    else
+      :persistent_term.put(@invalid_limits_warned_key, MapSet.put(warned, limiter_name))
+
+      Logger.warning(
+        "Invalid rate limiter config for #{inspect(limiter_name)}: #{inspect(limits)}. Falling back to defaults or disabling this limiter."
+      )
+    end
+  end
+
+  defp normalize_limits(nil), do: :disabled
+
+  defp normalize_limits(limits) do
+    case RateLimit.cast(limits) do
+      {:ok, normalized_limits} -> {:ok, normalized_limits}
+      :error -> :error
     end
   end
 

lib/pleroma/web/plugs/uploaded_media.ex

@@ -17,7 +17,7 @@ defmodule Pleroma.Web.Plugs.UploadedMedia do
   # no slashes
   @path "media"
 
-  @default_cache_control_header "public, max-age=1209600"
+  @default_cache_control_header "public, max-age=1209600, immutable"
 
   def init(_opts) do
     static_plug_opts =

lib/pleroma/web/preload/providers/instance.ex

@@ -5,9 +5,9 @@
 defmodule Pleroma.Web.Preload.Providers.Instance do
   alias Pleroma.Web.MastodonAPI.InstanceView
   alias Pleroma.Web.Nodeinfo.Nodeinfo
+  alias Pleroma.Web.PleromaAPI.UtilView
   alias Pleroma.Web.Plugs.InstanceStatic
   alias Pleroma.Web.Preload.Providers.Provider
-  alias Pleroma.Web.TwitterAPI.UtilView
 
   @behaviour Provider
   @instance_url "/api/v1/instance"

lib/pleroma/web/registration.ex

@@ -2,7 +2,7 @@
 # Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
-defmodule Pleroma.Web.TwitterAPI.TwitterAPI do
+defmodule Pleroma.Web.Registration do
   import Pleroma.Web.Gettext
 
   alias Pleroma.Emails.Mailer

lib/pleroma/web/remote_interaction/remote_interaction_controller.ex

@@ -2,7 +2,7 @@
 # Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
-defmodule Pleroma.Web.TwitterAPI.RemoteFollowController do
+defmodule Pleroma.Web.RemoteInteraction.RemoteInteractionController do
   use Pleroma.Web, :controller
 
   require Logger
@@ -14,18 +14,27 @@ defmodule Pleroma.Web.TwitterAPI.RemoteFollowController do
   alias Pleroma.Web.Auth.TOTPAuthenticator
   alias Pleroma.Web.Auth.WrapperAuthenticator
   alias Pleroma.Web.CommonAPI
+  alias Pleroma.Web.WebFinger
 
   @status_types ["Article", "Event", "Note", "Video", "Page", "Question"]
 
+  plug(
+    Pleroma.Web.ApiSpec.CastAndValidate,
+    [replace_params: false]
+    when action == :remote_interaction
+  )
+
   plug(Pleroma.Web.Plugs.FederatingPlug)
 
   # Note: follower can submit the form (with password auth) not being signed in (having no token)
   plug(
     Pleroma.Web.Plugs.OAuthScopesPlug,
     %{fallback: :proceed_unauthenticated, scopes: ["follow", "write:follows"]}
-    when action in [:do_follow]
+    when action == :do_follow
   )
 
+  defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.RemoteInteractionOperation
+
   # GET /ostatus_subscribe
   #
   def follow(%{assigns: %{user: user}} = conn, %{"acct" => acct}) do
@@ -125,7 +134,7 @@ defmodule Pleroma.Web.TwitterAPI.RemoteFollowController do
   #
   def authorize_interaction(conn, %{"uri" => uri}) do
     conn
-    |> redirect(to: Routes.remote_follow_path(conn, :follow, %{acct: uri}))
+    |> redirect(to: Routes.remote_interaction_path(conn, :follow, %{acct: uri}))
   end
 
   defp handle_follow_error(conn, {:mfa_token, followee, _} = _) do
@@ -162,4 +171,122 @@ defmodule Pleroma.Web.TwitterAPI.RemoteFollowController do
     Logger.debug("Remote follow failed with error #{inspect(error)}")
     render(conn, "followed.html", %{error: "Something went wrong."})
   end
+
+  def show_subscribe_form(conn, %{"nickname" => nick}) do
+    with %User{} = user <- User.get_cached_by_nickname(nick),
+         avatar = User.avatar_url(user) do
+      conn
+      |> render("subscribe.html", %{nickname: nick, avatar: avatar, error: false})
+    else
+      _e ->
+        render(conn, "subscribe.html", %{
+          nickname: nick,
+          avatar: nil,
+          error:
+            Pleroma.Web.Gettext.dpgettext(
+              "static_pages",
+              "remote follow error message - user not found",
+              "Could not find user"
+            )
+        })
+    end
+  end
+
+  def show_subscribe_form(conn, %{"status_id" => id}) do
+    with %Activity{} = activity <- Activity.get_by_id(id),
+         {:ok, ap_id} <- get_ap_id(activity),
+         %User{} = user <- User.get_cached_by_ap_id(activity.actor),
+         avatar = User.avatar_url(user) do
+      conn
+      |> render("status_interact.html", %{
+        status_link: ap_id,
+        status_id: id,
+        nickname: user.nickname,
+        avatar: avatar,
+        error: false
+      })
+    else
+      _e ->
+        render(conn, "status_interact.html", %{
+          status_id: id,
+          avatar: nil,
+          error:
+            Pleroma.Web.Gettext.dpgettext(
+              "static_pages",
+              "status interact error message - status not found",
+              "Could not find status"
+            )
+        })
+    end
+  end
+
+  def remote_subscribe(conn, %{"nickname" => nick, "profile" => _}) do
+    show_subscribe_form(conn, %{"nickname" => nick})
+  end
+
+  def remote_subscribe(conn, %{"status_id" => id, "profile" => _}) do
+    show_subscribe_form(conn, %{"status_id" => id})
+  end
+
+  def remote_subscribe(conn, %{"user" => %{"nickname" => nick, "profile" => profile}}) do
+    with {:ok, %{"subscribe_address" => template}} <- WebFinger.finger(profile),
+         %User{ap_id: ap_id} <- User.get_cached_by_nickname(nick) do
+      conn
+      |> Phoenix.Controller.redirect(external: String.replace(template, "{uri}", ap_id))
+    else
+      _e ->
+        render(conn, "subscribe.html", %{
+          nickname: nick,
+          avatar: nil,
+          error:
+            Pleroma.Web.Gettext.dpgettext(
+              "static_pages",
+              "remote follow error message - unknown error",
+              "Something went wrong."
+            )
+        })
+    end
+  end
+
+  def remote_subscribe(conn, %{"status" => %{"status_id" => id, "profile" => profile}}) do
+    with {:ok, %{"subscribe_address" => template}} <- WebFinger.finger(profile),
+         %Activity{} = activity <- Activity.get_by_id(id),
+         {:ok, ap_id} <- get_ap_id(activity) do
+      conn
+      |> Phoenix.Controller.redirect(external: String.replace(template, "{uri}", ap_id))
+    else
+      _e ->
+        render(conn, "status_interact.html", %{
+          status_id: id,
+          avatar: nil,
+          error:
+            Pleroma.Web.Gettext.dpgettext(
+              "static_pages",
+              "status interact error message - unknown error",
+              "Something went wrong."
+            )
+        })
+    end
+  end
+
+  def remote_interaction(
+        %{private: %{open_api_spex: %{body_params: %{ap_id: ap_id, profile: profile}}}} = conn,
+        _params
+      ) do
+    with {:ok, %{"subscribe_address" => template}} <- WebFinger.finger(profile) do
+      conn
+      |> json(%{url: String.replace(template, "{uri}", ap_id)})
+    else
+      _e -> json(conn, %{error: "Couldn't find user"})
+    end
+  end
+
+  defp get_ap_id(activity) do
+    object = Pleroma.Object.normalize(activity, fetch: false)
+
+    case object do
+      %{data: %{"id" => ap_id}} -> {:ok, ap_id}
+      _ -> {:no_ap_id, nil}
+    end
+  end
 end

lib/pleroma/web/remote_interaction/remote_interaction_view.ex

@@ -2,9 +2,11 @@
 # Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
-defmodule Pleroma.Web.TwitterAPI.RemoteFollowView do
+defmodule Pleroma.Web.RemoteInteraction.RemoteInteractionView do
   use Pleroma.Web, :view
+  import Phoenix.HTML
   import Phoenix.HTML.Form
+  import Phoenix.HTML.Link
   alias Pleroma.Web.Gettext
 
   def avatar_url(user) do

lib/pleroma/web/router.ex

@@ -226,23 +226,28 @@ defmodule Pleroma.Web.Router do
     plug(Pleroma.Web.Plugs.StaticFEPlug)
   end
 
-  scope "/api/v1/pleroma", Pleroma.Web.TwitterAPI do
+  scope "/api/v1/pleroma", Pleroma.Web.OAuth do
     pipe_through(:pleroma_api)
 
     get("/password_reset/:token", PasswordController, :reset, as: :reset_password)
     post("/password_reset", PasswordController, :do_reset, as: :reset_password)
-    get("/emoji", UtilController, :emoji)
-    get("/captcha", UtilController, :captcha)
-    get("/healthcheck", UtilController, :healthcheck)
-    post("/remote_interaction", UtilController, :remote_interaction)
   end
 
   scope "/api/v1/pleroma", Pleroma.Web.PleromaAPI do
     pipe_through(:pleroma_api)
 
+    get("/emoji", UtilController, :emoji)
+    get("/captcha", UtilController, :captcha)
+    get("/healthcheck", UtilController, :healthcheck)
     get("/federation_status", InstancesController, :show)
   end
 
+  scope "/api/v1/pleroma", Pleroma.Web.RemoteInteraction do
+    pipe_through(:pleroma_api)
+
+    post("/remote_interaction", RemoteInteractionController, :remote_interaction)
+  end
+
   scope "/api/v1/pleroma", Pleroma.Web do
     pipe_through(:pleroma_api)
     post("/uploader_callback/:upload_path", UploaderController, :callback)
@@ -531,18 +536,18 @@ defmodule Pleroma.Web.Router do
     end
   end
 
-  scope "/", Pleroma.Web.TwitterAPI do
+  scope "/", Pleroma.Web.RemoteInteraction do
     pipe_through(:pleroma_html)
 
-    post("/main/ostatus", UtilController, :remote_subscribe)
-    get("/main/ostatus", UtilController, :show_subscribe_form)
-    get("/ostatus_subscribe", RemoteFollowController, :follow)
-    post("/ostatus_subscribe", RemoteFollowController, :do_follow)
+    post("/main/ostatus", RemoteInteractionController, :remote_subscribe)
+    get("/main/ostatus", RemoteInteractionController, :show_subscribe_form)
+    get("/ostatus_subscribe", RemoteInteractionController, :follow)
+    post("/ostatus_subscribe", RemoteInteractionController, :do_follow)
 
-    get("/authorize_interaction", RemoteFollowController, :authorize_interaction)
+    get("/authorize_interaction", RemoteInteractionController, :authorize_interaction)
   end
 
-  scope "/api/pleroma", Pleroma.Web.TwitterAPI do
+  scope "/api/pleroma", Pleroma.Web.PleromaAPI do
     pipe_through(:authenticated_api)
 
     post("/change_email", UtilController, :change_email)
@@ -900,7 +905,7 @@ defmodule Pleroma.Web.Router do
   scope "/api", Pleroma.Web do
     pipe_through(:config)
 
-    get("/pleroma/frontend_configurations", TwitterAPI.UtilController, :frontend_configurations)
+    get("/pleroma/frontend_configurations", PleromaAPI.UtilController, :frontend_configurations)
   end
 
   scope "/api", Pleroma.Web do
@@ -908,7 +913,7 @@ defmodule Pleroma.Web.Router do
 
     get(
       "/account/confirm_email/:user_id/:token",
-      TwitterAPI.Controller,
+      OAuth.TokenController,
       :confirm_email,
       as: :confirm_email
     )
@@ -920,11 +925,11 @@ defmodule Pleroma.Web.Router do
     get("/openapi", OpenApiSpex.Plug.RenderSpec, [])
   end
 
-  scope "/api", Pleroma.Web, as: :authenticated_twitter_api do
+  scope "/api", Pleroma.Web, as: :authenticated_pleroma_api do
     pipe_through(:authenticated_api)
 
-    get("/oauth_tokens", TwitterAPI.Controller, :oauth_tokens)
-    delete("/oauth_tokens/:id", TwitterAPI.Controller, :revoke_token)
+    get("/oauth_tokens", OAuth.TokenController, :oauth_tokens)
+    delete("/oauth_tokens/:id", OAuth.TokenController, :revoke_token)
   end
 
   scope "/", Pleroma.Web do
@@ -1073,7 +1078,9 @@ defmodule Pleroma.Web.Router do
   scope "/", Pleroma.Web do
     pipe_through(:pleroma_html)
 
-    post("/auth/password", TwitterAPI.PasswordController, :request)
+    post("/auth/password", OAuth.PasswordController, :request)
+
+    get("/embed/:id", EmbedController, :show)
   end
 
   scope "/proxy/", Pleroma.Web do
@@ -1135,7 +1142,7 @@ defmodule Pleroma.Web.Router do
     get("/:maybe_nickname_or_id", RedirectController, :redirector_with_meta)
     match(:*, "/api/pleroma/*path", LegacyPleromaApiRerouterPlug, [])
     get("/api/*path", RedirectController, :api_not_implemented)
-    get("/phoenix/live_dashboard", RedirectController, :live_dashboard)
+    get("/phoenix/live_dashboard/*path", RedirectController, :live_dashboard)
     get("/*path", RedirectController, :redirector_with_preload)
 
     options("/*path", RedirectController, :empty)

lib/pleroma/web/templates/remote_interaction/remote_interaction/follow.html.eex

@@ -4,7 +4,7 @@
     <h2><%= Gettext.dpgettext("static_pages", "remote follow header", "Remote follow") %></h2>
     <img height="128" width="128" src="<%= avatar_url(@followee) %>">
     <p><%= @followee.nickname %></p>
-    <%= form_for @conn, Routes.remote_follow_path(@conn, :do_follow), [as: "user"], fn f -> %>
+    <%= form_for @conn, Routes.remote_interaction_path(@conn, :do_follow), [as: "user"], fn f -> %>
     <%= hidden_input f, :id, value: @followee.id %>
     <%= submit Gettext.dpgettext("static_pages", "remote follow authorization button", "Authorize") %>
     <% end %>

lib/pleroma/web/templates/remote_interaction/remote_interaction/follow_login.html.eex

@@ -4,7 +4,7 @@
 <h2><%= Gettext.dpgettext("static_pages", "remote follow header, need login", "Log in to follow") %></h2>
 <p><%= @followee.nickname %></p>
 <img height="128" width="128" src="<%= avatar_url(@followee) %>">
-<%= form_for @conn, Routes.remote_follow_path(@conn, :do_follow), [as: "authorization"], fn f -> %>
+<%= form_for @conn, Routes.remote_interaction_path(@conn, :do_follow), [as: "authorization"], fn f -> %>
 <%= text_input f, :name, placeholder: Gettext.dpgettext("static_pages", "placeholder text for username entry", "Username"), required: true, autocomplete: "username" %>
 <br>
 <%= password_input f, :password, placeholder: Gettext.dpgettext("static_pages", "placeholder text for password entry", "Password"), required: true, autocomplete: "password" %>

lib/pleroma/web/templates/remote_interaction/remote_interaction/follow_mfa.html.eex

@@ -4,7 +4,7 @@
 <h2><%= Gettext.dpgettext("static_pages", "remote follow mfa header", "Two-factor authentication") %></h2>
 <p><%= @followee.nickname %></p>
 <img height="128" width="128" src="<%= avatar_url(@followee) %>">
-<%= form_for @conn, Routes.remote_follow_path(@conn, :do_follow), [as: "mfa"], fn f -> %>
+<%= form_for @conn, Routes.remote_interaction_path(@conn, :do_follow), [as: "mfa"], fn f -> %>
 <%= text_input f, :code, placeholder: Gettext.dpgettext("static_pages", "placeholder text for auth code entry", "Authentication code"), required: true %>
 <br>
 <%= hidden_input f, :id, value: @followee.id %>

lib/pleroma/web/templates/remote_interaction/remote_interaction/status_interact.html.eex

@@ -2,7 +2,7 @@
   <h2><%= Gettext.dpgettext("static_pages", "status interact error", "Error: %{error}", error: @error) %></h2>
 <% else %>
   <h2><%= raw Gettext.dpgettext("static_pages", "status interact header", "Interacting with %{nickname}'s %{status_link}", nickname: safe_to_string(html_escape(@nickname)), status_link: safe_to_string(link(Gettext.dpgettext("static_pages", "status interact header - status link text", "status"), to: @status_link))) %></h2>
-  <%= form_for @conn, Routes.util_path(@conn, :remote_subscribe), [as: "status"], fn f -> %>
+  <%= form_for @conn, Routes.remote_interaction_path(@conn, :remote_subscribe), [as: "status"], fn f -> %>
   <%= hidden_input f, :status_id, value: @status_id %>
   <%= text_input f, :profile, placeholder: Gettext.dpgettext("static_pages", "placeholder text for account id", "Your account ID, e.g. lain@quitter.se") %>
   <%= submit Gettext.dpgettext("static_pages", "status interact authorization button", "Interact") %>

lib/pleroma/web/templates/remote_interaction/remote_interaction/subscribe.html.eex

@@ -2,7 +2,7 @@
   <h2><%= Gettext.dpgettext("static_pages", "remote follow error", "Error: %{error}", error: @error) %></h2>
 <% else %>
   <h2><%= Gettext.dpgettext("static_pages", "remote follow header", "Remotely follow %{nickname}", nickname: @nickname) %></h2>
-  <%= form_for @conn, Routes.util_path(@conn, :remote_subscribe), [as: "user"], fn f -> %>
+  <%= form_for @conn, Routes.remote_interaction_path(@conn, :remote_subscribe), [as: "user"], fn f -> %>
   <%= hidden_input f, :nickname, value: @nickname %>
   <%= text_input f, :profile, placeholder: Gettext.dpgettext("static_pages", "placeholder text for account id", "Your account ID, e.g. lain@quitter.se") %>
   <%= submit Gettext.dpgettext("static_pages", "remote follow authorization button for following with a remote account", "Follow") %>

lib/pleroma/web/templates/static_fe/static_fe/profile.html.eex

@@ -2,7 +2,7 @@
   <h1><%= link instance_name(), to: "/" %></h1>
 
   <h3>
-    <form class="pull-right collapse" method="POST" action="<%= Helpers.util_path(@conn, :remote_subscribe) %>">
+    <form class="pull-right collapse" method="POST" action="<%= Helpers.remote_interaction_path(@conn, :remote_subscribe) %>">
       <input type="hidden" name="nickname" value="<%= @user.nickname %>">
       <input type="hidden" name="profile" value="">
       <button type="submit" class="collapse"><%= Gettext.dpgettext("static_pages", "static fe profile page remote follow button", "Remote follow") %></button>

lib/pleroma/web/twitter_api/views/util_view.ex

@@ -1,31 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Web.TwitterAPI.UtilView do
-  use Pleroma.Web, :view
-  import Phoenix.HTML
-  import Phoenix.HTML.Form
-  import Phoenix.HTML.Link
-  alias Pleroma.Config
-  alias Pleroma.Web.Endpoint
-  alias Pleroma.Web.Gettext
-
-  def status_net_config(instance) do
-    """
-    <config>
-    <site>
-    <name>#{Keyword.get(instance, :name)}</name>
-    <site>#{Endpoint.url()}</site>
-    <textlimit>#{Keyword.get(instance, :limit)}</textlimit>
-    <closed>#{!Keyword.get(instance, :registrations_open)}</closed>
-    </site>
-    </config>
-    """
-  end
-
-  def render("frontend_configurations.json", _) do
-    Config.get(:frontend_configurations, %{})
-    |> Enum.into(%{})
-  end
-end

lib/pleroma/workers/cron/digest_emails_worker.ex

@@ -7,7 +7,7 @@ defmodule Pleroma.Workers.Cron.DigestEmailsWorker do
   The worker to send digest emails.
   """
 
-  use Oban.Worker, queue: "mailer"
+  use Oban.Worker, queue: :background
 
   alias Pleroma.Config
   alias Pleroma.Emails

lib/pleroma/workers/receiver_worker.ex

@@ -4,40 +4,37 @@
 
 defmodule Pleroma.Workers.ReceiverWorker do
   alias Pleroma.Instances
-  alias Pleroma.Signature
-  alias Pleroma.User
   alias Pleroma.Web.Federator
+  alias Pleroma.Workers.SignatureRetryWorker
 
   use Oban.Worker, queue: :federator_incoming, max_attempts: 5, unique: [period: :infinity]
 
   @impl true
+  def perform(%Job{args: %{"op" => "incoming_ap_doc", "params" => params} = args} = job) do
+    if signature_retry_job?(args) do
+      perform_signature_retry(job)
+    else
+      perform_incoming(params)
+    end
+  end
 
-  def perform(%Job{
-        args: %{
-          "op" => "incoming_ap_doc",
-          "method" => method,
-          "params" => params,
-          "req_headers" => req_headers,
-          "request_path" => request_path,
-          "query_string" => query_string
-        }
-      }) do
-    # Oban's serialization converts our tuple headers to lists.
-    # Revert it for the signature validation.
-    req_headers = Enum.into(req_headers, [], &List.to_tuple(&1))
+  def perform(%Job{args: %{"op" => "incoming_ap_doc"} = args} = job) do
+    if signature_retry_job?(args) do
+      perform_signature_retry(job)
+    else
+      process_errors(:missing_incoming_ap_doc_params)
+    end
+  end
 
-    conn_data = %Plug.Conn{
-      method: method,
-      params: params,
-      req_headers: req_headers,
-      request_path: request_path,
-      query_string: query_string
-    }
+  defp perform_signature_retry(%Job{args: args} = job) do
+    SignatureRetryWorker.perform(%Job{
+      job
+      | args: Map.put(args, "op", "incoming_failed_signature_ap_doc")
+    })
+  end
 
-    with {:ok, %User{}} <- User.get_or_fetch_by_ap_id(conn_data.params["actor"]),
-         {:ok, _public_key} <- Signature.refetch_public_key(conn_data),
-         {:signature, true} <- {:signature, Signature.validate_signature(conn_data)},
-         {:ok, res} <- Federator.perform(:incoming_ap_doc, params) do
+  defp perform_incoming(params) do
+    with {:ok, res} <- Federator.perform(:incoming_ap_doc, params) do
       unless Instances.reachable?(params["actor"]) do
         domain = URI.parse(params["actor"]).host
         Oban.insert(Pleroma.Workers.ReachabilityWorker.new(%{"domain" => domain}))
@@ -49,17 +46,8 @@ defmodule Pleroma.Workers.ReceiverWorker do
     end
   end
 
-  def perform(%Job{args: %{"op" => "incoming_ap_doc", "params" => params}}) do
-    with {:ok, res} <- Federator.perform(:incoming_ap_doc, params) do
-      unless Instances.reachable?(params["actor"]) do
-        domain = URI.parse(params["actor"]).host
-        Oban.insert(Pleroma.Workers.ReachabilityWorker.new(%{"domain" => domain}))
-      end
-
-      {:ok, res}
-    else
-      e -> process_errors(e)
-    end
+  defp signature_retry_job?(args) do
+    Enum.any?(~w(method req_headers request_path query_string), &Map.has_key?(args, &1))
   end
 
   @impl true
@@ -85,10 +73,12 @@ defmodule Pleroma.Workers.ReceiverWorker do
       {:error, {:reject, _} = reason} -> {:cancel, reason}
       # HTTP Sigs
       {:signature, false} -> {:cancel, :invalid_signature}
+      {:same_actor, false} -> {:cancel, :actor_signature_mismatch}
       # Origin / URL validation failed somewhere possibly due to spoofing
       {:error, :origin_containment_failed} -> {:cancel, :origin_containment_failed}
       # Unclear if this can be reached
       {:error, {:side_effects, {:error, :no_object_actor}} = reason} -> {:cancel, reason}
+      :missing_incoming_ap_doc_params -> {:cancel, :missing_incoming_ap_doc_params}
       # Catchall
       {:error, _} = e -> e
       e -> {:error, e}

lib/pleroma/workers/signature_retry_worker.ex

@@ -0,0 +1,254 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Workers.SignatureRetryWorker do
+  alias Pleroma.Instances
+  alias Pleroma.Signature
+  alias Pleroma.User
+  alias Pleroma.Web.ActivityPub.Utils
+  alias Pleroma.Web.Federator
+  alias Pleroma.Web.Plugs.MappedSignatureToIdentityPlug
+
+  require Logger
+
+  use Oban.Worker, queue: :federator_incoming, max_attempts: 5, unique: [period: :infinity]
+
+  @impl true
+  def perform(%Job{
+        args: %{
+          "op" => "incoming_failed_signature_ap_doc",
+          "method" => method,
+          "params" => params,
+          "req_headers" => req_headers,
+          "request_path" => request_path,
+          "query_string" => query_string
+        }
+      })
+      when is_binary(method) and is_map(params) and is_list(req_headers) and
+             is_binary(request_path) and is_binary(query_string) do
+    case normalize_req_headers(req_headers) do
+      {:ok, req_headers} ->
+        conn_data = %Plug.Conn{
+          assigns: %{valid_signature: true},
+          method: method,
+          params: params,
+          req_headers: req_headers,
+          request_path: request_path,
+          query_string: query_string
+        }
+
+        signature_actor_result = signature_actor_id(conn_data)
+
+        with actor_id = Utils.get_ap_id(params["actor"]),
+             {:signature_actor, {:ok, signature_actor_id}} <-
+               {:signature_actor, signature_actor_result},
+             {:same_actor, true} <- {:same_actor, signature_actor_id == actor_id},
+             {:ok, %User{}} <- User.get_or_fetch_by_ap_id(actor_id),
+             {:ok, _public_key} <- Signature.refetch_public_key(conn_data),
+             {:signature, true} <- {:signature, validate_signature(conn_data)},
+             {:same_actor, true} <- {:same_actor, validate_same_actor(conn_data)},
+             {:ok, res} <- Federator.perform(:incoming_ap_doc, params) do
+          unless Instances.reachable?(params["actor"]) do
+            domain = URI.parse(params["actor"]).host
+            Oban.insert(Pleroma.Workers.ReachabilityWorker.new(%{"domain" => domain}))
+          end
+
+          {:ok, res}
+        else
+          e -> process_errors(e, retry_log_context(params, request_path, signature_actor_result))
+        end
+
+      e ->
+        process_errors(e, retry_log_context(params, request_path, nil))
+    end
+  end
+
+  def perform(%Job{args: %{"op" => "incoming_failed_signature_ap_doc"} = args}) do
+    process_errors(
+      :missing_signature_retry_metadata,
+      retry_log_context(Map.get(args, "params"), Map.get(args, "request_path"), nil)
+    )
+  end
+
+  def perform(%Job{args: args}) when is_map(args) do
+    process_errors(
+      :missing_signature_retry_metadata,
+      retry_log_context(Map.get(args, "params"), Map.get(args, "request_path"), nil)
+    )
+  end
+
+  def perform(%Job{}), do: process_errors(:missing_signature_retry_metadata)
+
+  @impl true
+  def timeout(%_{args: %{"timeout" => timeout}}), do: timeout
+
+  def timeout(_job), do: :timer.seconds(5)
+
+  defp normalize_req_headers(req_headers) do
+    req_headers
+    |> Enum.reduce_while({:ok, []}, fn
+      {key, value}, {:ok, acc} when is_binary(key) and is_binary(value) ->
+        {:cont, {:ok, [{key, value} | acc]}}
+
+      [key, value], {:ok, acc} when is_binary(key) and is_binary(value) ->
+        {:cont, {:ok, [{key, value} | acc]}}
+
+      _, _ ->
+        {:halt, {:error, :invalid_signature_retry_metadata}}
+    end)
+    |> case do
+      {:ok, headers} -> {:ok, Enum.reverse(headers)}
+      error -> error
+    end
+  end
+
+  defp validate_same_actor(conn_data) do
+    case MappedSignatureToIdentityPlug.call(conn_data, []) do
+      %Plug.Conn{assigns: %{valid_signature: true}} ->
+        true
+
+      _ ->
+        false
+    end
+  end
+
+  defp validate_signature(conn_data) do
+    Signature.validate_signature(conn_data)
+  rescue
+    _ -> false
+  catch
+    _, _ -> false
+  end
+
+  defp signature_actor_id(conn_data) do
+    Signature.get_actor_id(conn_data)
+  rescue
+    _ -> {:error, :invalid_signature}
+  catch
+    _, _ -> {:error, :invalid_signature}
+  end
+
+  defp process_errors(errors, context \\ %{})
+
+  defp process_errors({:error, {:error, _} = error}, context), do: process_errors(error, context)
+
+  defp process_errors(errors, context) do
+    result =
+      case errors do
+        # User fetch failures
+        {:error, :not_found} = reason ->
+          {:cancel, reason}
+
+        {:error, :forbidden} = reason ->
+          {:cancel, reason}
+
+        # Inactive user
+        {:error, {:user_active, false} = reason} ->
+          {:cancel, reason}
+
+        # Validator will error and return a changeset error
+        # e.g., duplicate activities or if the object was deleted
+        {:error, {:validate, {:error, _changeset} = reason}} ->
+          {:cancel, reason}
+
+        # Duplicate detection during Normalization
+        {:error, :already_present} ->
+          {:cancel, :already_present}
+
+        # MRFs will return a reject
+        {:error, {:reject, _} = reason} ->
+          {:cancel, reason}
+
+        # HTTP Sigs
+        {:signature_actor, {:error, _}} ->
+          {:cancel, :invalid_signature}
+
+        {:signature, false} ->
+          {:cancel, :invalid_signature}
+
+        {:same_actor, false} ->
+          {:cancel, :actor_signature_mismatch}
+
+        # Origin / URL validation failed somewhere possibly due to spoofing
+        {:error, :origin_containment_failed} ->
+          {:cancel, :origin_containment_failed}
+
+        # Unclear if this can be reached
+        {:error, {:side_effects, {:error, :no_object_actor}} = reason} ->
+          {:cancel, reason}
+
+        # Fail closed if the retry cannot reconstruct the original request.
+        :missing_signature_retry_metadata ->
+          {:cancel, :missing_signature_retry_metadata}
+
+        {:error, :invalid_signature_retry_metadata} ->
+          {:cancel, :invalid_signature_retry_metadata}
+
+        # Catchall
+        {:error, _} = e ->
+          e
+
+        e ->
+          {:error, e}
+      end
+
+    log_signature_retry_rejection(result, context)
+    result
+  end
+
+  defp retry_log_context(params, request_path, signature_actor_result) when is_map(params) do
+    signature_actor =
+      case signature_actor_result do
+        {:ok, actor} when is_binary(actor) -> actor
+        actor when is_binary(actor) -> actor
+        _ -> nil
+      end
+
+    %{
+      activity_id: params["id"],
+      payload_actor: Utils.get_ap_id(params["actor"]),
+      request_path: request_path,
+      signature_actor: signature_actor,
+      type: params["type"]
+    }
+  end
+
+  defp retry_log_context(_params, request_path, signature_actor_result) do
+    signature_actor =
+      case signature_actor_result do
+        {:ok, actor} when is_binary(actor) -> actor
+        actor when is_binary(actor) -> actor
+        _ -> nil
+      end
+
+    %{
+      activity_id: nil,
+      payload_actor: nil,
+      request_path: request_path,
+      signature_actor: signature_actor,
+      type: nil
+    }
+  end
+
+  defp log_signature_retry_rejection({:cancel, reason}, context)
+       when reason in [
+              :actor_signature_mismatch,
+              :invalid_signature,
+              :invalid_signature_retry_metadata,
+              :missing_signature_retry_metadata,
+              :origin_containment_failed
+            ] do
+    Logger.warning(
+      "Failed-signature inbox retry rejected " <>
+        "reason=#{inspect(reason)} " <>
+        "payload_actor=#{inspect(context[:payload_actor])} " <>
+        "signature_actor=#{inspect(context[:signature_actor])} " <>
+        "activity_id=#{inspect(context[:activity_id])} " <>
+        "type=#{inspect(context[:type])} " <>
+        "request_path=#{inspect(context[:request_path])}"
+    )
+  end
+
+  defp log_signature_retry_rejection(_result, _context), do: :ok
+end