Merge branch 'develop' into fix/card-skip-quote-link

lib/mix/pleroma.ex

@@ -17,7 +17,7 @@ defmodule Mix.Pleroma do
     :oban,
     :logger_backends
   ]
-  @cachex_children ["object", "user", "scrubber", "web_resp"]
+  @cachex_children ["object", "user", "scrubber", "web_resp", "http_backoff"]
   @doc "Common functions to be reused in mix tasks"
   def start_pleroma do
     Pleroma.Config.Holder.save_default()

lib/mix/tasks/pleroma/email.ex

@@ -33,7 +33,7 @@ defmodule Mix.Tasks.Pleroma.Email do
 
     Pleroma.User.Query.build(%{
       local: true,
-      is_active: true,
+      deactivated: false,
       is_confirmed: false,
       invisible: false
     })

lib/mix/tasks/pleroma/instance.ex

@@ -162,38 +162,20 @@ defmodule Mix.Tasks.Pleroma.Instance do
         )
         |> Path.expand()
 
-      {strip_uploads_location_message, strip_uploads_location_default} =
-        if Pleroma.Utils.command_available?("exiftool") do
-          {"Do you want to strip location (GPS) data from uploaded images? This requires exiftool, it was detected as installed. (y/n)",
-           "y"}
-        else
-          {"Do you want to strip location (GPS) data from uploaded images? This requires exiftool, it was detected as not installed, please install it if you answer yes. (y/n)",
-           "n"}
-        end
-
       strip_uploads_location =
         get_option(
           options,
           :strip_uploads_location,
-          strip_uploads_location_message,
-          strip_uploads_location_default
+          "Do you want to strip location (GPS) data from uploaded images? (y/n)",
+          "y"
         ) === "y"
 
-      {read_uploads_description_message, read_uploads_description_default} =
-        if Pleroma.Utils.command_available?("exiftool") do
-          {"Do you want to read data from uploaded files so clients can use it to prefill fields like image description? This requires exiftool, it was detected as installed. (y/n)",
-           "y"}
-        else
-          {"Do you want to read data from uploaded files so clients can use it to prefill fields like image description? This requires exiftool, it was detected as not installed, please install it if you answer yes. (y/n)",
-           "n"}
-        end
-
       read_uploads_description =
         get_option(
           options,
           :read_uploads_description,
-          read_uploads_description_message,
-          read_uploads_description_default
+          "Do you want to read metadata from uploaded images so clients can prefill the media description field? (y/n)",
+          "y"
         ) === "y"
 
       anonymize_uploads =
@@ -324,14 +306,14 @@ defmodule Mix.Tasks.Pleroma.Instance do
   defp upload_filters(filters) when is_map(filters) do
     enabled_filters =
       if filters.strip_location do
-        [Pleroma.Upload.Filter.Exiftool.StripLocation]
+        [Pleroma.Upload.Filter.Exif.StripLocation]
       else
         []
       end
 
     enabled_filters =
       if filters.read_description do
-        enabled_filters ++ [Pleroma.Upload.Filter.Exiftool.ReadDescription]
+        enabled_filters ++ [Pleroma.Upload.Filter.Exif.ReadDescription]
       else
         enabled_filters
       end

lib/mix/tasks/pleroma/user.ex

@@ -362,7 +362,7 @@ defmodule Mix.Tasks.Pleroma.User do
 
     Pleroma.User.Query.build(%{
       local: true,
-      is_active: true,
+      deactivated: false,
       is_moderator: false,
       is_admin: false,
       invisible: false
@@ -380,7 +380,7 @@ defmodule Mix.Tasks.Pleroma.User do
 
     Pleroma.User.Query.build(%{
       local: true,
-      is_active: true,
+      deactivated: false,
       is_moderator: false,
       is_admin: false,
       invisible: false

lib/pleroma/application.ex

@@ -66,6 +66,7 @@ defmodule Pleroma.Application do
 
     if match?({Tesla.Adapter.Finch, _}, adapter) do
       Logger.info("Starting Finch")
+      use_castore_for_otp_cacerts()
       Finch.start_link(name: MyFinch)
     end
 
@@ -197,6 +198,7 @@ defmodule Pleroma.Application do
       ),
       build_cachex("rel_me", limit: 2500),
       build_cachex("host_meta", default_ttl: :timer.minutes(120), limit: 5_000),
+      build_cachex("http_backoff", limit: 10_000),
       build_cachex("translations", default_ttl: :timer.hours(24), limit: 5_000)
     ]
   end
@@ -304,6 +306,15 @@ defmodule Pleroma.Application do
     end
   end
 
+  # Mint trusts the OTP CA store, which comes from the OS. Make it the CAStore
+  # bundle that the other adapters trust. OTP 24 has no such store and Mint
+  # uses CAStore there anyway.
+  defp use_castore_for_otp_cacerts do
+    if Code.ensure_loaded?(:public_key) and function_exported?(:public_key, :cacerts_load, 1) do
+      :ok = :public_key.cacerts_load(CAStore.file_path())
+    end
+  end
+
   defp http_children_hackney do
     pools = [:federation, :media]
 

lib/pleroma/application_requirements.ex

@@ -168,8 +168,6 @@ defmodule Pleroma.ApplicationRequirements do
 
   defp check_system_commands!(:ok) do
     filter_commands_statuses = [
-      check_filter(Pleroma.Upload.Filter.Exiftool.StripLocation, "exiftool"),
-      check_filter(Pleroma.Upload.Filter.Exiftool.ReadDescription, "exiftool"),
       check_filter(Pleroma.Upload.Filter.Mogrify, "mogrify"),
       check_filter(Pleroma.Upload.Filter.Mogrifun, "mogrify"),
       check_filter(Pleroma.Upload.Filter.AnalyzeMetadata, "ffprobe")

lib/pleroma/config/deprecation_warnings.ex

@@ -22,40 +22,44 @@ defmodule Pleroma.Config.DeprecationWarnings do
 
   @logger_formatter_config_knobs [:colors, :format, :metadata, :truncate, :utc_log]
 
+  @exiftool_filter_replacements %{
+    Pleroma.Upload.Filter.Exiftool => Pleroma.Upload.Filter.Exif.StripLocation,
+    Pleroma.Upload.Filter.Exiftool.StripLocation => Pleroma.Upload.Filter.Exif.StripLocation,
+    Pleroma.Upload.Filter.Exiftool.ReadDescription => Pleroma.Upload.Filter.Exif.ReadDescription
+  }
+
   def check_exiftool_filter do
     filters = Config.get([Pleroma.Upload]) |> Keyword.get(:filters, [])
 
-    if Pleroma.Upload.Filter.Exiftool in filters do
-      Logger.warning("""
-      !!!DEPRECATION WARNING!!!
-      Your config is using Exiftool as a filter instead of Exiftool.StripLocation. This should work for now, but you are advised to change to the new configuration to prevent possible issues later:
+    deprecated_filters =
+      filters
+      |> Enum.filter(&Map.has_key?(@exiftool_filter_replacements, &1))
+      |> Enum.uniq()
 
-      ```
-      config :pleroma, Pleroma.Upload,
-        filters: [Pleroma.Upload.Filter.Exiftool]
-      ```
+    case deprecated_filters do
+      [] ->
+        :ok
 
-      Is now
+      deprecated_filters ->
+        replacements =
+          Enum.map_join(deprecated_filters, fn filter ->
+            "\n* `#{inspect(filter)}` is now `#{inspect(@exiftool_filter_replacements[filter])}`"
+          end)
 
+        Logger.warning("""
+        !!!DEPRECATION WARNING!!!
+        Your config is using Exiftool upload filters. Pleroma no longer needs exiftool, the filters were replaced by built-in ones. The old names should work for now, but you are advised to change to the new configuration to prevent possible issues later:
+        #{replacements}
+        """)
 
-      ```
-      config :pleroma, Pleroma.Upload,
-        filters: [Pleroma.Upload.Filter.Exiftool.StripLocation]
-      ```
-      """)
+        new_config =
+          filters
+          |> Enum.map(&Map.get(@exiftool_filter_replacements, &1, &1))
+          |> Enum.uniq()
 
-      new_config =
-        filters
-        |> Enum.map(fn
-          Pleroma.Upload.Filter.Exiftool -> Pleroma.Upload.Filter.Exiftool.StripLocation
-          filter -> filter
-        end)
+        Config.put([Pleroma.Upload, :filters], new_config)
 
-      Config.put([Pleroma.Upload, :filters], new_config)
-
-      :error
-    else
-      :ok
+        :error
     end
   end
 

lib/pleroma/config/transfer_task.ex

@@ -48,6 +48,7 @@ defmodule Pleroma.Config.TransferTask do
       settings =
         (Repo.all(ConfigDB) ++ deleted_settings)
         |> Enum.reject(&ConfigDB.static?/1)
+        |> Enum.reject(&(&1.group == :invalid_atom or &1.key == :invalid_atom))
         |> Enum.map(&merge_with_default/1)
 
       started_applications = Application.started_applications()

lib/pleroma/ecto_type/config/atom.ex

@@ -5,6 +5,8 @@
 defmodule Pleroma.EctoType.Config.Atom do
   use Ecto.Type
 
+  require Logger
+
   def type, do: :atom
 
   def cast(key) when is_atom(key) do
@@ -17,8 +19,17 @@ defmodule Pleroma.EctoType.Config.Atom do
 
   def cast(_), do: :error
 
+  # A key or group naming a module that no longer exists (e.g. a removed uploader)
+  # must not crash loading
   def load(key) do
     {:ok, Pleroma.ConfigDB.string_to_elixir_types(key)}
+  rescue
+    ArgumentError ->
+      Logger.warning(
+        "ConfigDB: unknown module #{key} in a setting key or group, the setting is not applied"
+      )
+
+      {:ok, :invalid_atom}
   end
 
   def dump(key) when is_atom(key), do: {:ok, inspect(key)}

lib/pleroma/emails/mailer.ex

@@ -36,10 +36,11 @@ defmodule Pleroma.Emails.Mailer do
   def deliver(email, config \\ [])
 
   def deliver(email, config) do
-    # temporary hackney fix until hackney max_connections bug is fixed
-    # https://git.pleroma.social/pleroma/pleroma/-/issues/2101
+    # Swoosh HTTP adapters use Hackney, which otherwise trusts its certifi bundle
     email =
-      Swoosh.Email.put_private(email, :hackney_options, ssl_options: [versions: [:"tlsv1.2"]])
+      Swoosh.Email.put_private(email, :hackney_options,
+        ssl_options: Pleroma.HTTP.TLS.ssl_options()
+      )
 
     case enabled?() do
       true -> Swoosh.Mailer.deliver(email, parse_config(config))
@@ -67,6 +68,21 @@ defmodule Pleroma.Emails.Mailer do
   end
 
   defp parse_config(config) do
-    Swoosh.Mailer.parse_config(@otp_app, __MODULE__, @mailer_config, config)
+    @otp_app
+    |> Swoosh.Mailer.parse_config(__MODULE__, @mailer_config, config)
+    |> put_mua_cacertfile()
+  end
+
+  @doc false
+  # Mua otherwise trusts the OS CA store. Keep a CA that the admin set.
+  def put_mua_cacertfile(config) do
+    ssl = Keyword.get(config, :ssl, [])
+
+    if config[:adapter] == Swoosh.Adapters.Mua and
+         not Keyword.has_key?(ssl, :cacertfile) and not Keyword.has_key?(ssl, :cacerts) do
+      Keyword.put(config, :ssl, [{:cacertfile, CAStore.file_path()} | ssl])
+    else
+      config
+    end
   end
 end

lib/pleroma/filter.ex

@@ -208,19 +208,30 @@ defmodule Pleroma.Filter do
   end
 
   def compose_regex([_ | _] = filters, format) do
+    boundary = if format == :postgres, do: "\\y", else: "\\b"
+
     phrases =
       filters
-      |> Enum.map(& &1.phrase)
+      |> Enum.map(&phrase_regex(&1, boundary))
       |> Enum.join("|")
 
     case format do
-      :postgres ->
-        "\\y(#{phrases})\\y"
-
-      :re ->
-        ~r/\b#{phrases}\b/i
+      :postgres -> "(#{phrases})"
+      :re -> Regex.compile!("(?:#{phrases})", "iu")
     end
   end
 
   def compose_regex(_, _), do: nil
+
+  # Phrases are literal text. Like Mastodon, whole-word filters only get a
+  # word boundary where the phrase starts or ends with a word character.
+  defp phrase_regex(%__MODULE__{phrase: phrase, whole_word: false}, _boundary),
+    do: Regex.escape(phrase)
+
+  defp phrase_regex(%__MODULE__{phrase: phrase}, boundary) do
+    prefix = if String.match?(phrase, ~r/^\w/u), do: boundary, else: ""
+    suffix = if String.match?(phrase, ~r/\w$/u), do: boundary, else: ""
+
+    prefix <> Regex.escape(phrase) <> suffix
+  end
 end

lib/pleroma/formatter.ex

@@ -3,12 +3,15 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Formatter do
+  require Logger
+
   alias Pleroma.HTML
   alias Pleroma.User
 
   @safe_mention_regex ~r/^(\s*(?<mentions>(@.+?\s+){1,})+)(?<rest>.*)/s
   @link_regex ~r"((?:http(s)?:\/\/)?[\w.-]+(?:\.[\w\.-]+)+[\w\-\._~%:/?#[\]@!\$&'\(\)\*\+,;=.]+)|[0-9a-z+\-\.]+:[0-9a-z$-_.+!*'(),]+"ui
   @markdown_characters_regex ~r/(`|\*|_|{|}|[|]|\(|\)|#|\+|-|\.|!)/
+  @markdown_void_elements ~w[area base br col command embed hr img input keygen link meta param source track wbr]
 
   defp linkify_opts do
     Pleroma.Config.get(Pleroma.Formatter) ++
@@ -124,14 +127,157 @@ defmodule Pleroma.Formatter do
   end
 
   def markdown_to_html(text) do
-    Earmark.as_html!(text, %Earmark.Options{compact_output: true, smartypants: false})
+    markdown_to_html(text, %{})
   end
 
   def markdown_to_html(text, opts) do
-    Earmark.as_html!(
-      text,
-      %Earmark.Options{compact_output: true, smartypants: false} |> Map.merge(opts)
-    )
+    breaks = Map.get(opts, :breaks, false)
+
+    {_status, ast, _messages} = EarmarkParser.as_ast(text, breaks: breaks, gfm: true)
+
+    ast
+    |> maybe_strip_email_autolinks(text, Map.get(opts, :strip_email_autolinks, false))
+    |> render_markdown_nodes(false)
+    |> IO.iodata_to_binary()
+  rescue
+    exception ->
+      Logger.warning("Markdown rendering failed in #{inspect(exception.__struct__)}")
+      markdown_fallback(text, Map.get(opts, :breaks, false))
+  end
+
+  defp markdown_fallback(text, breaks) do
+    text = text |> escape_html() |> IO.iodata_to_binary() |> maybe_render_markdown_breaks(breaks)
+    "<p>" <> text <> "</p>"
+  end
+
+  defp maybe_render_markdown_breaks(text, true), do: String.replace(text, ~r/\r?\n/, "<br>")
+  defp maybe_render_markdown_breaks(text, false), do: text
+
+  defp maybe_strip_email_autolinks(ast, _text, false), do: ast
+
+  defp maybe_strip_email_autolinks(ast, text, true) do
+    emails =
+      ~r/<([^ >]+@[^ >]+)>/
+      |> Regex.scan(text, capture: :all_but_first)
+      |> Enum.map(fn [email] -> String.replace_prefix(email, "mailto:", "") end)
+      |> MapSet.new()
+
+    strip_email_autolinks(ast, emails)
+  end
+
+  defp strip_email_autolinks(nodes, emails) when is_list(nodes) do
+    Enum.map(nodes, &strip_email_autolinks(&1, emails))
+  end
+
+  defp strip_email_autolinks({"a", attrs, [email], _meta} = node, emails)
+       when is_binary(email) do
+    if List.keyfind(attrs, "href", 0) == {"href", "mailto:" <> email} and
+         MapSet.member?(emails, email) do
+      email
+    else
+      node
+    end
+  end
+
+  defp strip_email_autolinks({tag, attrs, children, meta}, emails) do
+    {tag, attrs, strip_email_autolinks(children, emails), meta}
+  end
+
+  defp strip_email_autolinks(text, _emails) when is_binary(text), do: text
+
+  defp render_markdown_nodes(nodes, verbatim) when is_list(nodes) do
+    Enum.map(nodes, &render_markdown_node(&1, verbatim))
+  end
+
+  defp render_markdown_node(text, true) when is_binary(text),
+    do: strip_dangerous_markdown_html(text)
+
+  defp render_markdown_node(text, false) when is_binary(text) do
+    text
+    |> strip_dangerous_markdown_html()
+    |> escape_markdown_html()
+  end
+
+  defp render_markdown_node({:comment, _attrs, _children, _meta}, _verbatim), do: []
+
+  defp render_markdown_node({tag, attrs, children, meta}, verbatim) do
+    normalized_tag = String.downcase(tag)
+
+    if normalized_tag in ["script", "style"] do
+      []
+    else
+      render_markdown_element(tag, normalized_tag, attrs, children, meta, verbatim)
+    end
+  end
+
+  defp render_markdown_element("code", _normalized_tag, attrs, children, _meta, _verbatim) do
+    ["<code", render_markdown_attrs(attrs), ">", render_markdown_code(children), "</code>"]
+  end
+
+  defp render_markdown_element(tag, "hr", attrs, _children, _meta, _verbatim) do
+    attrs = Enum.reject(attrs, &match?({"class", class} when class in ~w[thin medium thick], &1))
+    ["<", tag, render_markdown_attrs(attrs), "/>"]
+  end
+
+  defp render_markdown_element(tag, normalized_tag, attrs, _children, _meta, _verbatim)
+       when normalized_tag in @markdown_void_elements do
+    ["<", tag, render_markdown_attrs(attrs), ">"]
+  end
+
+  defp render_markdown_element(tag, _normalized_tag, attrs, children, meta, verbatim) do
+    children_verbatim = verbatim or Map.get(meta, :verbatim, false)
+
+    [
+      "<",
+      tag,
+      render_markdown_attrs(attrs),
+      ">",
+      render_markdown_nodes(children, children_verbatim),
+      "</",
+      tag,
+      ">"
+    ]
+  end
+
+  defp render_markdown_attrs(attrs) do
+    Enum.map(attrs, fn {name, value} ->
+      value = value |> to_string() |> HtmlEntities.decode() |> sanitize_markdown_url(name)
+      [" ", to_string(name), "=\"", escape_html(value), "\""]
+    end)
+  end
+
+  defp render_markdown_code(children) do
+    Enum.map(children, &escape_html/1)
+  end
+
+  defp sanitize_markdown_url(value, name) do
+    if String.downcase(to_string(name)) in ["href", "src"] do
+      # Percent-encode instead of cutting, so the URL stays complete but can
+      # never close the attribute or open a tag.
+      String.replace(value, ~r/["<>]/, fn
+        "\"" -> "%22"
+        "<" -> "%3C"
+        ">" -> "%3E"
+      end)
+    else
+      value
+    end
+  end
+
+  defp strip_dangerous_markdown_html(text) do
+    text
+    |> String.replace(~r/<(script|style)\b[^>]*>.*?<\/\1>/is, "")
+    |> String.replace(~r/<(script|style)\b[^>]*>.*\z/is, "")
+    |> String.replace(~r/<\/(script|style)\s*>/i, "")
+  end
+
+  defp escape_markdown_html(text) do
+    text |> HtmlEntities.decode() |> escape_html()
+  end
+
+  defp escape_html(text) do
+    {:safe, escaped} = Phoenix.HTML.html_escape(text)
+    escaped
   end
 
   def html_escape({text, mentions, hashtags}, type) do

lib/pleroma/gun/conn.ex

@@ -14,7 +14,9 @@ defmodule Pleroma.Gun.Conn do
       opts
       |> Enum.into(%{})
       |> Map.put_new(:connect_timeout, pool_opts[:connect_timeout] || 5_000)
-      |> Map.put_new(:supervise, false)
+      # Under gun's supervisor, a failed connect does not kill the caller
+      # through the link. Gun still stops when its owner goes down.
+      |> Map.put_new(:supervise, true)
       |> maybe_add_tls_opts(uri)
 
     do_open(uri, opts)
@@ -117,8 +119,7 @@ defmodule Pleroma.Gun.Conn do
     host = Pleroma.HTTP.AdapterHelper.parse_host(host)
     opts = Map.put(opts, :transport, transport(uri.scheme))
 
-    with {:ok, conn} <- Gun.open(host, port, opts),
-         {:ok, protocol} <- await_up(conn, opts[:connect_timeout]) do
+    with {:ok, conn, protocol} <- open_direct(host, port, opts) do
       {:ok, conn, protocol, nil}
     else
       error ->
@@ -130,6 +131,40 @@ defmodule Pleroma.Gun.Conn do
     end
   end
 
+  defp open_direct(host, port, opts) do
+    result =
+      with {:ok, conn} <- Gun.open(host, port, opts),
+           {:ok, protocol} <- await_up(conn, opts[:connect_timeout]) do
+        {:ok, conn, protocol}
+      end
+
+    case inet6_fallback(result, host, opts) do
+      {:ok, inet6_opts} -> open_direct(host, port, inet6_opts)
+      :error -> result
+    end
+  end
+
+  # Gun's default when :tcp_opts is not set.
+  @default_tcp_opts [send_timeout: 15_000, send_timeout_close: true]
+
+  @doc """
+  Gun resolves hostnames with `inet_tcp` (A records only) unless `:inet6` is
+  in `:tcp_opts`, for both the :tcp and :tls transports. When the A lookup
+  finds nothing, returns the options for one more attempt over IPv6.
+  """
+  @spec inet6_fallback(term(), Pleroma.HTTP.AdapterHelper.host(), map()) :: {:ok, map()} | :error
+  def inet6_fallback({:error, {:down, {:shutdown, :nxdomain}}}, host, opts) when is_list(host) do
+    tcp_opts = Map.get(opts, :tcp_opts, @default_tcp_opts)
+
+    if :inet6 in tcp_opts do
+      :error
+    else
+      {:ok, Map.put(opts, :tcp_opts, [:inet6 | tcp_opts])}
+    end
+  end
+
+  def inet6_fallback(_result, _host, _opts), do: :error
+
   defp destination_opts(%URI{host: host, port: port}) do
     host = Pleroma.HTTP.AdapterHelper.parse_host(host)
     %{host: host, port: port}

lib/pleroma/html/uri_attributes.ex

@@ -0,0 +1,40 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.HTML.URIAttributes do
+  @moduledoc """
+  URI attribute rules for the scrubbers.
+
+  Works like `FastSanitize.Sanitizer.Meta.allow_tag_with_uri_attributes/3`, but
+  checks the scheme the way a browser reads the URL. Browsers remove tabs and
+  newlines anywhere and control characters and spaces at the start, so
+  `java\\tscript:` is a `javascript:` link to them, while `URI.parse/1` finds no
+  scheme and FastSanitize lets it through.
+  """
+
+  defmacro allow_tag_with_uri_attributes(tag, attrs, valid_schemes) do
+    for attr <- attrs do
+      quote do
+        def scrub_attribute(unquote(tag), {unquote(attr), "&" <> _}), do: nil
+
+        def scrub_attribute(unquote(tag), {unquote(attr), uri} = attribute) do
+          if Pleroma.HTML.URIAttributes.valid_scheme?(uri, unquote(valid_schemes)),
+            do: attribute
+        end
+      end
+    end
+  end
+
+  @spec valid_scheme?(String.t(), [String.t()]) :: boolean()
+  def valid_scheme?(uri, valid_schemes) do
+    scheme =
+      uri
+      |> String.replace(~r/[\t\n\r]/, "")
+      |> String.replace(~r/\A[\x00-\x20]+/, "")
+      |> URI.parse()
+      |> Map.get(:scheme)
+
+    is_nil(scheme) or scheme in valid_schemes
+  end
+end

lib/pleroma/http.ex

@@ -79,8 +79,14 @@ defmodule Pleroma.HTTP do
       adapter,
       adapter_opts
     )
+    |> drop_partial_body()
   end
 
+  # Hackney 4.9 returns the bytes read before the peer closed. No caller wants
+  # them, and callers that cache errors would keep them.
+  defp drop_partial_body({:error, {:closed, body}}) when is_binary(body), do: {:error, :closed}
+  defp drop_partial_body(result), do: result
+
   @spec request(Client.t(), keyword()) :: {:ok, Env.t()} | {:error, any()}
   def request(client, request), do: Tesla.request(client, request)
 
@@ -130,5 +136,9 @@ defmodule Pleroma.HTTP do
   end
 
   defp default_middleware,
-    do: [Tesla.Middleware.FollowRedirects, Pleroma.Tesla.Middleware.EncodeUrl]
+    do: [
+      Tesla.Middleware.FollowRedirects,
+      Pleroma.Tesla.Middleware.EncodeUrl,
+      Pleroma.Tesla.Middleware.MaxBody
+    ]
 end

lib/pleroma/http/adapter_helper.ex

@@ -20,8 +20,9 @@ defmodule Pleroma.HTTP.AdapterHelper do
 
   @callback options(keyword(), URI.t()) :: keyword()
 
+  # AdminFE stores an empty proxy field as "", which means no proxy.
   @spec format_proxy(String.t() | tuple() | nil) :: proxy() | nil
-  def format_proxy(nil), do: nil
+  def format_proxy(proxy) when proxy in [nil, ""], do: nil
 
   def format_proxy(proxy_url) do
     case parse_proxy(proxy_url) do
@@ -32,7 +33,7 @@ defmodule Pleroma.HTTP.AdapterHelper do
   end
 
   @spec maybe_add_proxy(keyword(), proxy() | nil) :: keyword()
-  def maybe_add_proxy(opts, nil), do: opts
+  def maybe_add_proxy(opts, proxy) when proxy in [nil, ""], do: opts
   def maybe_add_proxy(opts, proxy), do: Keyword.put_new(opts, :proxy, proxy)
 
   @doc """
@@ -46,6 +47,24 @@ defmodule Pleroma.HTTP.AdapterHelper do
     |> adapter_helper().options(uri)
   end
 
+  @doc """
+  The configured Tesla adapter with TLS options, for clients that do not go
+  through `Pleroma.HTTP`.
+  """
+  @spec tesla_adapter() :: module() | {module(), keyword()}
+  def tesla_adapter do
+    case adapter() do
+      Tesla.Adapter.Hackney ->
+        {Tesla.Adapter.Hackney, Pleroma.HTTP.TLS.put_hackney_ssl_options([])}
+
+      Tesla.Adapter.Gun ->
+        {Tesla.Adapter.Gun, certificates_verification: true}
+
+      adapter ->
+        adapter
+    end
+  end
+
   defp adapter, do: Application.get_env(:tesla, :adapter)
 
   defp adapter_helper do

lib/pleroma/http/adapter_helper/gun.ex

@@ -28,6 +28,7 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do
 
     @defaults
     |> Keyword.merge(config_opts)
+    |> ssl_options_to_tls_opts()
     |> add_scheme_opts(uri)
     |> AdapterHelper.maybe_add_proxy(proxy)
     |> Keyword.merge(incoming_opts)
@@ -35,6 +36,15 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do
     |> maybe_stream()
   end
 
+  # [:http, :adapter, :ssl_options] is Hackney's name for the TLS options,
+  # and AdminFE sets it. Keep those settings when Gun is the adapter.
+  defp ssl_options_to_tls_opts(opts) do
+    case Keyword.pop(opts, :ssl_options) do
+      {nil, opts} -> opts
+      {ssl_options, opts} -> Keyword.put_new(opts, :tls_opts, ssl_options)
+    end
+  end
+
   defp add_scheme_opts(opts, %{scheme: "http"}), do: opts
 
   defp add_scheme_opts(opts, %{scheme: "https"}) do

lib/pleroma/http/adapter_helper/hackney.ex

@@ -28,6 +28,25 @@ defmodule Pleroma.HTTP.AdapterHelper.Hackney do
     |> add_scheme_opts(uri)
     |> maybe_add_with_body()
     |> Pleroma.HTTP.AdapterHelper.maybe_add_proxy(proxy)
+    |> Keyword.put(:follow_redirect, false)
+    |> Keyword.put(:force_redirect, false)
+    |> normalize_proxy()
+    |> Pleroma.HTTP.TLS.put_hackney_ssl_options()
+  end
+
+  # Hackney 4 ignores a proxy whose host is an IP tuple and connects directly.
+  # ConfigDB turns a numeric proxy host into such a tuple.
+  defp normalize_proxy(opts) do
+    case Keyword.fetch(opts, :proxy) do
+      {:ok, {type, host, port}} when is_tuple(host) ->
+        Keyword.put(opts, :proxy, {type, :inet.ntoa(host), port})
+
+      {:ok, {host, port}} when is_tuple(host) ->
+        Keyword.put(opts, :proxy, {:inet.ntoa(host), port})
+
+      _ ->
+        opts
+    end
   end
 
   defp add_scheme_opts(opts, _), do: opts

lib/pleroma/http/backoff.ex

@@ -0,0 +1,153 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.HTTP.Backoff do
+  @moduledoc """
+  GET wrapper that stops sending requests to a host after it answers
+  429 or 503, until the time given by `Retry-After` or `X-RateLimit-Reset`.
+
+  Based on Akkoma's `Pleroma.HTTP.Backoff`.
+  """
+
+  alias Pleroma.HTTP
+
+  require Logger
+
+  @cachex Pleroma.Config.get([:cachex, :provider], Cachex)
+  @cache :http_backoff_cache
+
+  @default_seconds 5 * 60
+  @max_seconds 60 * 60
+  @max_job_age 2 * 60 * 60
+
+  @spec get(String.t(), list(), keyword()) :: {:ok, Tesla.Env.t()} | {:error, any()}
+  def get(url, headers \\ [], options \\ []) when is_binary(url) do
+    host = url |> URI.parse() |> Map.get(:host) |> downcase()
+
+    case @cachex.get(@cache, host) do
+      {:ok, true} ->
+        {:error, :ratelimit}
+
+      _ ->
+        url
+        |> HTTP.get(headers, options)
+        |> check_backoff(host)
+    end
+  end
+
+  defp downcase(nil), do: nil
+  defp downcase(host), do: String.downcase(host)
+
+  defp check_backoff({:ok, %{status: status, headers: headers} = env}, host)
+       when status in [429, 503] do
+    host = response_host(env, host)
+    seconds = backoff_seconds(headers, DateTime.utc_now())
+    Logger.warning("#{host} answered #{status}, backing off for #{seconds} seconds")
+    @cachex.put(@cache, host, true, ttl: :timer.seconds(seconds))
+    {:error, :ratelimit}
+  end
+
+  defp check_backoff(response, _host), do: response
+
+  # After a redirect, the 429/503 came from the last host, not the requested one.
+  defp response_host(%{url: url}, host) when is_binary(url) do
+    URI.parse(url).host |> downcase() || host
+  end
+
+  defp response_host(_env, host), do: host
+
+  @doc """
+  Turns an Oban job's `{:error, :ratelimit}` result into a snooze for the
+  default backoff, so the job's attempts aren't all spent while the host is
+  backed off. Jobs inserted more than #{@max_job_age} seconds ago get the
+  error back and fail as usual.
+  """
+  @spec snooze_ratelimited(any(), Oban.Job.t()) :: any()
+  def snooze_ratelimited({:error, :ratelimit} = error, %Oban.Job{inserted_at: inserted_at}) do
+    if is_nil(inserted_at) or DateTime.diff(DateTime.utc_now(), inserted_at) < @max_job_age do
+      {:snooze, @default_seconds}
+    else
+      error
+    end
+  end
+
+  def snooze_ratelimited(result, _job), do: result
+
+  @doc """
+  Seconds to back off for, from the first valid `Retry-After` or
+  `X-RateLimit-Reset` header. Missing, invalid or past values give the
+  default of #{@default_seconds} seconds; values are capped at #{@max_seconds}.
+  """
+  @spec backoff_seconds(list(), DateTime.t()) :: pos_integer()
+  def backoff_seconds(headers, now) do
+    [
+      parse_retry_after(header(headers, "retry-after"), now),
+      parse_ratelimit_reset(header(headers, "x-ratelimit-reset"), now)
+    ]
+    |> Enum.find(@default_seconds, &is_integer/1)
+    |> min(@max_seconds)
+  end
+
+  defp header(headers, name) do
+    Enum.find_value(headers, fn
+      {key, value} when is_binary(key) and is_binary(value) ->
+        if String.downcase(key) == name, do: String.trim(value)
+
+      _ ->
+        nil
+    end)
+  end
+
+  # Retry-After is either delta-seconds or an HTTP-date (RFC 9110, 10.2.3)
+  defp parse_retry_after(nil, _now), do: nil
+
+  defp parse_retry_after(value, now) do
+    case Integer.parse(value) do
+      {seconds, ""} when seconds > 0 -> seconds
+      {_, ""} -> nil
+      _ -> parse_timestamp(value, now)
+    end
+  end
+
+  # Servers send X-RateLimit-Reset as a timestamp (ISO 8601 like Mastodon, or
+  # Unix seconds) or as seconds until the reset.
+  @unix_timestamp_min 1_000_000_000
+
+  defp parse_ratelimit_reset(nil, _now), do: nil
+
+  defp parse_ratelimit_reset(value, now) do
+    case Integer.parse(value) do
+      {timestamp, ""} when timestamp >= @unix_timestamp_min ->
+        timestamp - DateTime.to_unix(now)
+
+      {seconds, ""} ->
+        seconds
+
+      _ ->
+        parse_timestamp(value, now)
+    end
+    |> positive()
+  end
+
+  defp positive(seconds) when is_integer(seconds) and seconds > 0, do: seconds
+  defp positive(_), do: nil
+
+  defp parse_timestamp(nil, _now), do: nil
+
+  defp parse_timestamp(value, now) do
+    with {:ok, %DateTime{} = datetime} <- parse_datetime(value),
+         seconds when seconds > 0 <- DateTime.diff(datetime, now) do
+      seconds
+    else
+      _ -> nil
+    end
+  end
+
+  defp parse_datetime(value) do
+    with {:error, _} <- Timex.parse(value, "{RFC1123}"),
+         {:ok, datetime, _offset} <- DateTime.from_iso8601(value) do
+      {:ok, datetime}
+    end
+  end
+end

lib/pleroma/http/tls.ex

@@ -0,0 +1,49 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.HTTP.TLS do
+  @moduledoc """
+  TLS client options that trust the CAStore bundle.
+
+  Hackney otherwise trusts its own certifi bundle. Hackney 1 drops all of its
+  default TLS options, including certificate checks, as soon as any
+  `ssl_options` are given. Hackney 4 merges them, so `partial_chain` must be
+  set too: Hackney's own one trusts any certificate in the peer chain that is
+  in certifi.
+  """
+
+  @spec ssl_options(keyword()) :: keyword()
+  def ssl_options(overrides \\ []) do
+    defaults = [
+      verify: :verify_peer,
+      cacertfile: CAStore.file_path(),
+      depth: 20,
+      customize_hostname_check: [match_fun: :public_key.pkix_verify_hostname_match_fun(:https)],
+      partial_chain: &__MODULE__.reject_partial_chain/1
+    ]
+
+    # Hackney drops cacerts when cacertfile is set too.
+    defaults =
+      if Keyword.has_key?(overrides, :cacerts),
+        do: Keyword.delete(defaults, :cacertfile),
+        else: defaults
+
+    Keyword.merge(defaults, overrides)
+  end
+
+  @doc false
+  def reject_partial_chain(_certs), do: :unknown_ca
+
+  @spec put_hackney_ssl_options(keyword()) :: keyword()
+  def put_hackney_ssl_options(opts) do
+    # Hackney 4 ignores the top-level insecure option and falls back to certifi.
+    if opts[:insecure] do
+      Keyword.put(opts, :ssl_options, verify: :verify_none)
+    else
+      opts
+      |> Keyword.put(:ssl_options, ssl_options(Keyword.get(opts, :ssl_options, [])))
+      |> Keyword.put(:proxy_ssl_options, ssl_options(Keyword.get(opts, :proxy_ssl_options, [])))
+    end
+  end
+end

lib/pleroma/instances/instance.ex

@@ -91,6 +91,13 @@ defmodule Pleroma.Instances.Instance do
 
   def reachable?(url_or_host) when is_binary(url_or_host), do: true
 
+  @doc "Creates a record for the instance unless one exists. Existing records are kept."
+  def ensure_exists(url_or_host) when is_binary(url_or_host) do
+    %Instance{host: host(url_or_host)}
+    |> changeset(%{})
+    |> Repo.insert(on_conflict: :nothing, conflict_target: :host)
+  end
+
   def set_reachable(url_or_host) when is_binary(url_or_host) do
     host = host(url_or_host)
 

lib/pleroma/notification.ex

@@ -821,6 +821,9 @@ defmodule Pleroma.Notification do
       object.data["actor"] == user.ap_id ->
         false
 
+      not is_binary(object.data["content"]) ->
+        false
+
       not is_nil(regex = Pleroma.Filter.compose_regex(user, :re)) ->
         Regex.match?(regex, object.data["content"])
 

lib/pleroma/object/containment.ex

@@ -44,6 +44,9 @@ defmodule Pleroma.Object.Containment do
     id
   end
 
+  # remote data can carry any shape here; callers treat nil as "no actor"
+  def get_actor(_), do: nil
+
   def get_object(%{"object" => id}) when is_binary(id) do
     id
   end
@@ -108,8 +111,46 @@ defmodule Pleroma.Object.Containment do
 
   def contain_origin_from_id(_id, _data), do: :error
 
+  @doc """
+  Checks that a fetched document's id is the URL it was fetched from
+  (after redirects).
+
+  Fragments, a trailing slash, the case of the host and differences
+  that our URL encoding removes before a request are ignored.
+  Documents without an id never match.
+
+  Based on Akkoma's `contain_id_to_fetch/2`.
+  """
+  def contain_id_to_fetch(url, %{"id" => id}) when is_binary(url) and is_binary(id) do
+    if normalize_uri(url) == normalize_uri(id), do: :ok, else: :error
+  end
+
+  def contain_id_to_fetch(_url, _data), do: :error
+
+  defp normalize_uri(uri) do
+    uri = uri |> Pleroma.Utils.URIEncoding.encode_url() |> URI.parse()
+
+    %URI{
+      uri
+      | fragment: nil,
+        host: uri.host && String.downcase(uri.host, :ascii),
+        path: uri.path && String.replace_suffix(uri.path, "/", "")
+    }
+    |> URI.to_string()
+  end
+
   def contain_child(%{"object" => %{"id" => id, "attributedTo" => _} = object}),
     do: contain_origin(id, object)
 
   def contain_child(_), do: :ok
+
+  @doc "Checks that two URIs are on the same host. URIs without a host never match."
+  def same_origin(id, other_id) when is_binary(id) and is_binary(other_id) do
+    case {URI.parse(id), URI.parse(other_id)} do
+      {%URI{host: host}, %URI{host: host}} when is_binary(host) and host != "" -> :ok
+      _ -> :error
+    end
+  end
+
+  def same_origin(_id, _other_id), do: :error
 end

lib/pleroma/object/fetcher.ex

@@ -11,6 +11,7 @@ defmodule Pleroma.Object.Fetcher do
   alias Pleroma.Web.ActivityPub.InternalFetchActor
   alias Pleroma.Web.ActivityPub.MRF
   alias Pleroma.Web.ActivityPub.ObjectValidator
+  alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes
   alias Pleroma.Web.ActivityPub.Pipeline
   alias Pleroma.Web.ActivityPub.Transmogrifier
   alias Pleroma.Web.Federator
@@ -46,9 +47,14 @@ defmodule Pleroma.Object.Fetcher do
     end
   end
 
-  def refetch_object(%Object{data: %{"id" => id}} = object) do
+  # The stored id is canonical, so the refetch must not redirect, and the
+  # remote must not swap in another object or author.
+  # Based on Akkoma 9061d148b and fee57eb37.
+  def refetch_object(%Object{data: %{"id" => id} = data} = object) do
     with {:local, false} <- {:local, Object.local?(object)},
-         {:ok, new_data} <- fetch_and_contain_remote_object_from_id(id),
+         {:ok, new_data} <- fetch_and_contain(id, true),
+         {:id, true} <- {:id, new_data["id"] == id},
+         {:actor, true} <- {:actor, actor_of(new_data) == actor_of(data)},
          {:ok, object} <- reinject_object(object, new_data) do
       {:ok, object}
     else
@@ -57,6 +63,8 @@ defmodule Pleroma.Object.Fetcher do
     end
   end
 
+  defp actor_of(data), do: CommonFixes.fix_actor(data)["actor"]
+
   @typep fetcher_errors ::
            :error | :reject | :allowed_depth | :fetch | :containment | :transmogrifier
 
@@ -140,17 +148,39 @@ defmodule Pleroma.Object.Fetcher do
   def fetch_and_contain_remote_object_from_id(%{"id" => id}),
     do: fetch_and_contain_remote_object_from_id(id)
 
-  def fetch_and_contain_remote_object_from_id(id) when is_binary(id) do
+  def fetch_and_contain_remote_object_from_id(id) when is_binary(id),
+    do: fetch_and_contain(id, false)
+
+  def fetch_and_contain_remote_object_from_id(_id),
+    do: {:error, "id must be a string"}
+
+  # The fetched document's id must be the URL it came from. Display URLs
+  # (e.g. Mastodon's /@user/123) serve the object under another id, so
+  # if the id differs we refetch once from that id, which has to be on the
+  # same host, must not redirect, and must then match exactly.
+  # Based on Akkoma 8684964c5 and 940792f8b.
+  defp fetch_and_contain(id, canonical?) do
     Logger.debug("Fetching object #{id} via AP")
 
     with {:scheme, true} <- {:scheme, String.starts_with?(id, "http")},
          {_, true} <- {:mrf, MRF.id_filter(id)},
          {_, :ok} <- {:local_fetch, Containment.contain_local_fetch(id)},
-         {:ok, body} <- get_object(id),
+         {:ok, final_url, body} <- get_object(id),
+         {_, true} <- {:redirect, not canonical? or same_url?(final_url, id)},
          {:ok, data} <- safe_json_decode(body),
-         :ok <- Containment.contain_origin_from_id(id, data) do
+         :ok <- Containment.contain_origin_from_id(id, data),
+         {_, _, :ok} <- {:id, data, Containment.contain_id_to_fetch(final_url, data)} do
       {:ok, data}
     else
+      {:id, %{"id" => canonical_id}, :error} when is_binary(canonical_id) and not canonical? ->
+        fetch_and_contain(canonical_id, true)
+
+      {:id, _, :error} ->
+        {:error, :id_mismatch}
+
+      {:redirect, false} ->
+        {:error, :canonical_id_redirect}
+
       {:scheme, _} ->
         {:error, "Unsupported URI scheme"}
 
@@ -168,8 +198,7 @@ defmodule Pleroma.Object.Fetcher do
     end
   end
 
-  def fetch_and_contain_remote_object_from_id(_id),
-    do: {:error, "id must be a string"}
+  defp same_url?(url, other), do: Containment.contain_id_to_fetch(url, %{"id" => other}) == :ok
 
   defp check_crossdomain_redirect(final_host, _original_url) when is_nil(final_host) do
     {:cross_domain_redirect, false}
@@ -187,7 +216,7 @@ defmodule Pleroma.Object.Fetcher do
       |> maybe_date_fetch(date)
       |> sign_fetch(id, date)
 
-    case HTTP.get(id, headers) do
+    case HTTP.Backoff.get(id, headers) do
       {:ok, %{body: body, status: code, headers: headers, url: final_url}}
       when code in 200..299 ->
         remote_host = if final_url, do: URI.parse(final_url).host, else: nil
@@ -195,7 +224,7 @@ defmodule Pleroma.Object.Fetcher do
         with {:cross_domain_redirect, false} <- check_crossdomain_redirect(remote_host, id),
              {_, content_type} <- List.keyfind(headers, "content-type", 0),
              {:ok, _media_type} <- verify_content_type(content_type) do
-          {:ok, body}
+          {:ok, if(final_url in [nil, ""], do: id, else: final_url), body}
         else
           {:cross_domain_redirect, true} ->
             {:error, {:cross_domain_redirect, true}}
@@ -209,7 +238,7 @@ defmodule Pleroma.Object.Fetcher do
         case List.keyfind(headers, "content-type", 0) do
           {_, content_type} ->
             case verify_content_type(content_type) do
-              {:ok, _} -> {:ok, body}
+              {:ok, _} -> {:ok, id, body}
               error -> error
             end
 

lib/pleroma/reverse_proxy.ex

@@ -112,7 +112,13 @@ defmodule Pleroma.ReverseProxy do
   def call(_conn, _url, _opts \\ [])
 
   def call(conn = %{method: method}, url, opts) when method in @methods do
-    client_opts = Keyword.merge(@default_options, Keyword.get(opts, :http, []))
+    client_opts =
+      @default_options
+      |> Keyword.merge(Keyword.get(opts, :http, []))
+      |> Keyword.put_new(
+        :recv_timeout,
+        receive_timeout(opts)
+      )
 
     req_headers = build_req_headers(conn.req_headers, opts)
 
@@ -506,7 +512,16 @@ defmodule Pleroma.ReverseProxy do
     end
   end
 
-  defp check_read_duration(_, _), do: {:ok, :no_duration_limit, :no_duration_limit}
+  defp check_read_duration(_, _), do: {:ok, :no_duration_limit}
+
+  defp receive_timeout(opts) do
+    case Keyword.get(opts, :max_read_duration, @max_read_duration) do
+      timeout when is_integer(timeout) and timeout > 0 -> timeout
+      _ -> :infinity
+    end
+  end
+
+  defp increase_read_duration(:no_duration_limit), do: {:ok, :no_duration_limit}
 
   defp increase_read_duration({previous_duration, started})
        when is_integer(previous_duration) and is_integer(started) do

lib/pleroma/reverse_proxy/client/hackney.ex

@@ -28,17 +28,18 @@ defmodule Pleroma.ReverseProxy.Client.Hackney do
   @impl true
   def request(method, url, headers, body, opts \\ []) do
     opts =
-      Keyword.put_new(opts, :path_encode_fun, fn path ->
-        path
-      end)
+      opts
+      |> Keyword.put_new(:path_encode_fun, fn path -> path end)
+      |> Pleroma.HTTP.TLS.put_hackney_ssl_options()
 
     if opts[:follow_redirect] != false do
       {_state, req_opts} = Access.get_and_update(opts, :follow_redirect, fn a -> {a, false} end)
       env = %{method: method, headers: headers, body: body, req_opts: req_opts}
-      res = :hackney.request(method, url, headers, body, req_opts)
+      res = do_request(method, url, headers, body, req_opts)
 
       case res do
-        {:ok, code, resp_headers, _client} when code in @redirect_statuses ->
+        {:ok, code, resp_headers, client} when code in @redirect_statuses ->
+          close(client)
           redirect(url, resp_headers, env, @redirect_limit)
 
         {:ok, code, resp_headers} when code in @redirect_statuses ->
@@ -48,11 +49,34 @@ defmodule Pleroma.ReverseProxy.Client.Hackney do
           res
       end
     else
-      :hackney.request(method, url, headers, body, opts)
+      do_request(method, url, headers, body, opts)
     end
   end
 
   @impl true
+  def stream_body(%{released: true}), do: :done
+
+  def stream_body(%{ref: ref, recv_timeout: recv_timeout} = client) do
+    _ = :hackney.stream_next(ref)
+
+    case receive_response(ref, recv_timeout) do
+      {:ok, data} when is_binary(data) ->
+        {:ok, data, client}
+
+      # Reaching the end of the body does not free the connection: Hackney parks
+      # a finished conn instead of stopping it, and an unpooled one has neither
+      # an idle timeout nor a live owner monitor to reap it, so it would sit
+      # there forever pinning every binary it read. Release it here, mirroring
+      # what the gun client does, so callers only close what they abandon.
+      {:ok, :done} ->
+        close(client)
+        :done
+
+      {:error, error} ->
+        {:error, error}
+    end
+  end
+
   def stream_body(ref) do
     case :hackney.stream_body(ref) do
       :done -> :done
@@ -62,8 +86,98 @@ defmodule Pleroma.ReverseProxy.Client.Hackney do
   end
 
   @impl true
+  def close(%{released: true}), do: :ok
+
+  def close(%{ref: ref}), do: :hackney.close(ref)
+
   def close(ref), do: :hackney.close(ref)
 
+  defp do_request(method, url, headers, body, opts) do
+    recv_timeout = Keyword.get(opts, :recv_timeout, :infinity)
+
+    req_opts =
+      opts
+      |> Keyword.put(:async, :once)
+      |> Keyword.put(:stream_to, self())
+
+    with {:ok, ref} <- :hackney.request(method, url, headers, body, req_opts) do
+      result =
+        with {:ok, {:status, code, _reason}} <- receive_response(ref, recv_timeout),
+             {:ok, {:headers, resp_headers}} <- receive_response(ref, recv_timeout) do
+          {:ok, code, resp_headers, build_client(ref, recv_timeout, method, code, opts)}
+        end
+
+      case result do
+        {:ok, _, _, _} = response ->
+          response
+
+        error ->
+          close(ref)
+          flush_responses(ref)
+          error
+      end
+    end
+  end
+
+  # A response that cannot carry a body (204, 304, or a reply to HEAD) is
+  # complete as soon as the headers are parsed. Hackney queues `done` for us and,
+  # if the connection is pooled, checks it straight back in, so the ref stops
+  # being ours at that point: pulling the stream or stopping the connection
+  # afterwards would disturb whichever request checks it out next. Mark it so
+  # `stream_body/1` and `close/1` leave it alone.
+  #
+  # This relies on hackney never running its own redirect handling for us --
+  # `request/5` forces `follow_redirect: false` -- otherwise a redirect status
+  # would be checked back in too and would need the same treatment.
+  defp build_client(ref, recv_timeout, method, code, opts) do
+    client = %{ref: ref, recv_timeout: recv_timeout}
+
+    if released?(method, code) and pooled?(opts) do
+      flush_responses(ref)
+      Map.put(client, :released, true)
+    else
+      client
+    end
+  end
+
+  defp released?(method, _code) when method in [:head, :HEAD, "head", "HEAD"], do: true
+  defp released?(_method, code) when code in [204, 304], do: true
+  defp released?(_method, _code), do: false
+
+  # Mirrors hackney's own pooling decision: :hackney.use_pool/1, plus the fact
+  # that a proxy tunnel always bypasses the pool. An unpooled connection is never
+  # checked in, and parks in `connected` with no idle timeout, so it leaks
+  # forever unless we stop it ourselves.
+  defp pooled?(opts) do
+    no_proxy?(Keyword.get(opts, :proxy)) and use_pool?(Keyword.get(opts, :pool))
+  end
+
+  # Hackney reads `false` as "no proxy", same as an absent option.
+  defp no_proxy?(nil), do: true
+  defp no_proxy?(false), do: true
+  defp no_proxy?(_proxy), do: false
+
+  defp use_pool?(false), do: false
+  defp use_pool?(nil), do: Application.get_env(:hackney, :use_default_pool, true)
+  defp use_pool?(_pool_name), do: true
+
+  defp receive_response(ref, recv_timeout) do
+    receive do
+      {:hackney_response, ^ref, {:error, error}} -> {:error, error}
+      {:hackney_response, ^ref, response} -> {:ok, response}
+    after
+      recv_timeout -> {:error, :timeout}
+    end
+  end
+
+  defp flush_responses(ref) do
+    receive do
+      {:hackney_response, ^ref, _response} -> flush_responses(ref)
+    after
+      0 -> :ok
+    end
+  end
+
   defp redirect(url, resp_headers, env, limit) when limit == 0 do
     new_url = absolute_redirect_url(url, resp_headers)
 
@@ -71,17 +185,18 @@ defmodule Pleroma.ReverseProxy.Client.Hackney do
       "#{__MODULE__}: Handling redirect #{url} -> #{new_url}; redirect limit was reached - returning response after final redirect"
     )
 
-    :hackney.request(env.method, new_url, env.headers, env.body, env.req_opts)
+    do_request(env.method, new_url, env.headers, env.body, env.req_opts)
   end
 
   defp redirect(url, resp_headers, env, limit) do
     new_url = absolute_redirect_url(url, resp_headers)
     Logger.debug("#{__MODULE__}: handling redirect #{url} -> #{new_url}; limit = #{limit}")
 
-    res = :hackney.request(env.method, new_url, env.headers, env.body, env.req_opts)
+    res = do_request(env.method, new_url, env.headers, env.body, env.req_opts)
 
     case res do
-      {:ok, code, new_resp_headers, _client} when code in @redirect_statuses ->
+      {:ok, code, new_resp_headers, client} when code in @redirect_statuses ->
+        close(client)
         redirect(new_url, new_resp_headers, env, limit - 1)
 
       {:ok, code, new_resp_headers} when code in @redirect_statuses ->

lib/pleroma/search/database_search.ex

@@ -7,6 +7,7 @@ defmodule Pleroma.Search.DatabaseSearch do
   alias Pleroma.Config
   alias Pleroma.Object.Fetcher
   alias Pleroma.Pagination
+  alias Pleroma.Repo
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.Visibility
 
@@ -23,22 +24,49 @@ defmodule Pleroma.Search.DatabaseSearch do
     offset = Keyword.get(options, :offset, 0)
     author = Keyword.get(options, :author)
 
-    try do
-      Activity
-      |> Activity.with_preloaded_object()
-      |> Activity.restrict_deactivated_users()
-      |> restrict_public(user)
-      |> query_with(index_type, search_query)
-      |> maybe_restrict_local(user)
-      |> maybe_restrict_author(author)
-      |> maybe_restrict_blocked(user)
-      |> Pagination.fetch_paginated(
-        %{"offset" => offset, "limit" => limit, "skip_order" => index_type == :rum},
-        :offset
-      )
-      |> maybe_fetch(user, search_query)
-    rescue
-      _ -> maybe_fetch([], user, search_query)
+    activities =
+      try do
+        with_fuzzy_limit(fn ->
+          Activity
+          |> Activity.with_preloaded_object()
+          |> Activity.restrict_deactivated_users()
+          |> restrict_public(user)
+          |> query_with(index_type, search_query)
+          |> maybe_restrict_local(user)
+          |> maybe_restrict_author(author)
+          |> maybe_restrict_blocked(user)
+          |> Pagination.fetch_paginated(
+            %{"offset" => offset, "limit" => limit, "skip_order" => index_type == :rum},
+            :offset
+          )
+        end)
+      rescue
+        _ -> []
+      end
+
+    # Outside the query transaction, as it may fetch over the network.
+    maybe_fetch(activities, user, search_query, options)
+  end
+
+  @doc """
+  Runs `fun` with `gin_fuzzy_search_limit` set, if configured under
+  `Pleroma.Search.DatabaseSearch`. The limit makes GIN scans faster but their
+  results incomplete and random, so it is only set for search queries.
+  """
+  def with_fuzzy_limit(fun) do
+    case Config.get([__MODULE__, :gin_fuzzy_search_limit]) do
+      limit when is_integer(limit) and limit > 0 ->
+        {:ok, result} =
+          Repo.transaction(fn ->
+            # SET LOCAL takes no parameters; limit is an integer.
+            Repo.query!("SET LOCAL gin_fuzzy_search_limit TO #{limit}")
+            fun.()
+          end)
+
+        result
+
+      _ ->
+        fun.()
     end
   end
 
@@ -132,7 +160,27 @@ defmodule Pleroma.Search.DatabaseSearch do
 
   defp restrict_local(q), do: where(q, local: true)
 
-  def maybe_fetch(activities, user, search_query) do
+  @doc """
+  Adds the status a URL query points to, fetching it if needed. As in Mastodon,
+  this only happens with `resolve: true` and on the first page, and for
+  anonymous users only if `[:instance, :limit_to_local_content]` is `false`.
+  """
+  def maybe_fetch(activities, user, search_query, options) do
+    if fetch_allowed?(user, options),
+      do: fetch(activities, user, search_query),
+      else: activities
+  end
+
+  defp fetch_allowed?(user, options) do
+    options[:resolve] == true and (options[:offset] || 0) == 0 and
+      case Config.get([:instance, :limit_to_local_content], :unauthenticated) do
+        :all -> false
+        :unauthenticated -> match?(%User{}, user)
+        false -> true
+      end
+  end
+
+  defp fetch(activities, user, search_query) do
     with true <- Regex.match?(~r/https?:/, search_query),
          {:ok, object} <- Fetcher.fetch_object_from_id(search_query),
          %Activity{} = activity <- Activity.get_create_by_object_ap_id(object.data["id"]),

lib/pleroma/search/meilisearch.ex

@@ -111,11 +111,11 @@ defmodule Pleroma.Search.Meilisearch do
         |> maybe_restrict_local(user)
         |> maybe_restrict_author(author)
         |> maybe_restrict_blocked(user)
-        |> maybe_fetch(user, query)
+        |> maybe_fetch(user, query, options)
         |> order_by([object: obj], desc: obj.data["published"])
         |> Pleroma.Repo.all()
       rescue
-        _ -> maybe_fetch([], user, query)
+        _ -> maybe_fetch([], user, query, options)
       end
     end
   end

lib/pleroma/search/parade_db.ex

@@ -12,7 +12,7 @@ defmodule Pleroma.Search.ParadeDB do
 
   import Pleroma.Search.DatabaseSearch,
     only: [
-      maybe_fetch: 3,
+      maybe_fetch: 4,
       maybe_restrict_author: 2,
       maybe_restrict_blocked: 2,
       maybe_restrict_local: 2
@@ -149,19 +149,19 @@ defmodule Pleroma.Search.ParadeDB do
 
     case search_ids(query, limit, offset, author) do
       {:ok, []} ->
-        maybe_fetch([], user, query)
+        maybe_fetch([], user, query, options)
 
       {:ok, ids} ->
         ids
-        |> fetch_activities(user, query, author)
+        |> fetch_activities(user, query, author, options)
 
       {:error, error} ->
         Logger.error("ParadeDB search failed: #{inspect(error)}")
-        maybe_fetch([], user, query)
+        maybe_fetch([], user, query, options)
     end
   end
 
-  defp fetch_activities(ids, user, query, author) do
+  defp fetch_activities(ids, user, query, author, options) do
     dumped_ids =
       ids
       |> Enum.map(&dump_activity_id/1)
@@ -183,7 +183,7 @@ defmodule Pleroma.Search.ParadeDB do
       )
     )
     |> Pleroma.Repo.all()
-    |> maybe_fetch(user, query)
+    |> maybe_fetch(user, query, options)
   end
 
   defp maybe_restrict_public(query, %Pleroma.User{}) do

lib/pleroma/search/qdrant_search.ex

@@ -10,7 +10,7 @@ defmodule Pleroma.Search.QdrantSearch do
   alias __MODULE__.OpenAIClient
   alias __MODULE__.QdrantClient
 
-  import Pleroma.Search.DatabaseSearch, only: [maybe_fetch: 3]
+  import Pleroma.Search.DatabaseSearch, only: [maybe_fetch: 4]
 
   @impl true
   def create_index do
@@ -131,7 +131,7 @@ defmodule Pleroma.Search.QdrantSearch do
       |> Activity.restrict_deactivated_users()
       |> Ecto.Query.order_by([a], fragment("array_position(?, ?)", ^ids, a.id))
       |> Pleroma.Repo.all()
-      |> maybe_fetch(user, original_query)
+      |> maybe_fetch(user, original_query, options)
     else
       _ ->
         []
@@ -160,7 +160,7 @@ defmodule Pleroma.Search.QdrantSearch.OpenAIClient do
   end
 
   defp client do
-    Tesla.client(middleware())
+    Tesla.client(middleware(), Pleroma.HTTP.AdapterHelper.tesla_adapter())
   end
 
   defp middleware do
@@ -191,7 +191,7 @@ defmodule Pleroma.Search.QdrantSearch.QdrantClient do
   end
 
   defp client do
-    Tesla.client(middleware())
+    Tesla.client(middleware(), Pleroma.HTTP.AdapterHelper.tesla_adapter())
   end
 
   defp middleware do

lib/pleroma/signature.ex

@@ -95,6 +95,84 @@ defmodule Pleroma.Signature do
     Timex.format!(date, "{WDshort}, {0D} {Mshort} {YYYY} {h24}:{m}:{s} GMT")
   end
 
+  @doc """
+  Checks the parts of an incoming signature that the cryptographic check does
+  not cover, so that a captured request cannot be replayed later or with a
+  different body.
+
+  The signature must cover `date` or `(created)`, and the signed time must be
+  within the configured window. Requests other than GET and HEAD must also
+  sign `digest`, and DigestPlug must have hashed their body. Based on Akkoma's http_signatures fork and Mastodon's
+  SignedRequest.
+  """
+  @spec check_signature_policy(Plug.Conn.t(), DateTime.t()) :: :ok | {:error, atom()}
+  def check_signature_policy(%Plug.Conn{} = conn, now \\ DateTime.utc_now()) do
+    signature = HTTPSignatures.signature_for_conn(conn)
+    signed = Map.get(signature, "headers", [])
+
+    cond do
+      "date" not in signed and "(created)" not in signed ->
+        {:error, :date_not_signed}
+
+      conn.method not in ["GET", "HEAD"] and "digest" not in signed ->
+        {:error, :digest_not_signed}
+
+      # The plug puts the digest computed by DigestPlug into the header. With
+      # no computed digest (e.g. multipart), nothing checked the body.
+      conn.method not in ["GET", "HEAD"] and is_nil(conn.assigns[:digest]) ->
+        {:error, :body_not_hashed}
+
+      true ->
+        with :ok <- check_signed_date(conn, signed, now) do
+          check_signed_created(signature, signed, now)
+        end
+    end
+  end
+
+  defp check_signed_date(conn, signed, now) do
+    if "date" in signed do
+      # Exactly one header: a second one could make this check and the
+      # signature check see different dates.
+      with [date] <- Plug.Conn.get_req_header(conn, "date"),
+           {:ok, %DateTime{} = date} <- Timex.parse(date, "{RFC1123}") do
+        check_time_window(date, now)
+      else
+        _ -> {:error, :invalid_date}
+      end
+    else
+      :ok
+    end
+  end
+
+  defp check_signed_created(signature, signed, now) do
+    if "(created)" in signed do
+      with created when is_binary(created) <- signature["created"],
+           {created, ""} <- Integer.parse(created),
+           {:ok, created} <- DateTime.from_unix(created) do
+        check_time_window(created, now)
+      else
+        _ -> {:error, :invalid_date}
+      end
+    else
+      :ok
+    end
+  end
+
+  defp check_time_window(signed_at, now) do
+    age = DateTime.diff(now, signed_at)
+
+    cond do
+      age > Pleroma.Config.get([:activitypub, :signature_max_age], 7200) ->
+        {:error, :date_too_old}
+
+      -age > Pleroma.Config.get([:activitypub, :signature_max_clock_skew], 3600) ->
+        {:error, :date_in_future}
+
+      true ->
+        :ok
+    end
+  end
+
   @spec validate_signature(Plug.Conn.t(), String.t()) :: boolean()
   def validate_signature(%Plug.Conn{} = conn, request_target) do
     # Newer drafts for HTTP signatures now use @request-target instead of the

lib/pleroma/stats.ex

@@ -8,10 +8,11 @@ defmodule Pleroma.Stats do
   import Ecto.Query
 
   alias Pleroma.CounterCache
+  alias Pleroma.Instances.Instance
   alias Pleroma.Repo
   alias Pleroma.User
 
-  @interval :timer.seconds(60)
+  @interval :timer.seconds(300)
 
   def start_link(_) do
     GenServer.start_link(
@@ -64,14 +65,12 @@ defmodule Pleroma.Stats do
           }
         }
   def calculate_stat_data do
+    # Hosts are unique in instances, and remote users add theirs. Scanning
+    # all remote users for their domains was one of the costliest queries.
+    # Favicon lookups for local accounts can add our own host, so skip it.
     peers =
-      from(
-        u in User,
-        select: fragment("distinct split_part(?, '@', 2)", u.nickname),
-        where: u.local != ^true
-      )
+      from(i in Instance, where: i.host != ^local_host(), select: i.host)
       |> Repo.all()
-      |> Enum.filter(& &1)
 
     domain_count = Enum.count(peers)
 
@@ -97,6 +96,9 @@ defmodule Pleroma.Stats do
     }
   end
 
+  # Read from the config: this also runs before the endpoint has started.
+  defp local_host, do: Pleroma.Config.get([Pleroma.Web.Endpoint, :url, :host])
+
   @spec get_status_visibility_count(String.t() | nil) :: map()
   def get_status_visibility_count(instance \\ nil) do
     if is_nil(instance) do

lib/pleroma/tesla/middleware/max_body.ex

@@ -0,0 +1,70 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Tesla.Middleware.MaxBody do
+  @moduledoc """
+  Enforces the `:max_body` adapter option for every adapter.
+
+  `Pleroma.HTTP.Adapter.Gun` applies the limit as it accumulates a `:plain`
+  body, so an oversized response is abandoned mid-transfer. Since 4.9 Hackney
+  does the same with the `max_body` request option. Before that it ignored
+  `with_body` and always handed back a fully read binary (`hackney:request/5`),
+  so `Tesla.Adapter.Hackney` never took the branch that would honour
+  `:max_body`.
+
+  The check bounds what callers are handed, whatever the adapter does, and
+  refuses up front when `content-length` already announces too much.
+
+  Streaming bodies are left alone: neither adapter limits those, and the caller
+  that owns the stream is the one that has to drain and release it.
+  """
+
+  @behaviour Tesla.Middleware
+
+  @impl Tesla.Middleware
+  def call(%Tesla.Env{opts: opts} = env, next, _opts) do
+    case opts[:adapter][:max_body] do
+      limit when is_integer(limit) and limit >= 0 -> run(env, next, limit)
+      _ -> Tesla.run(env, next)
+    end
+  end
+
+  defp run(env, next, limit) do
+    with {:ok, env} <- Tesla.run(env, next),
+         :ok <- check(env, limit) do
+      {:ok, env}
+    end
+  end
+
+  defp check(%Tesla.Env{body: body} = env, limit) when is_binary(body) do
+    with :ok <- check_content_length(env, limit) do
+      check_body(body, limit)
+    end
+  end
+
+  # body_as: :chunks and response: :stream hand back a client the caller drains
+  # itself, applying its own limit (Pleroma.Web.RichMedia.Helpers). There is
+  # nothing to measure, and failing on the header alone would strand the pool
+  # lease that only the caller holds a handle to.
+  defp check(_env, _limit), do: :ok
+
+  # A bodyless response may still announce the entity's size.
+  defp check_content_length(%Tesla.Env{method: :head}, _limit), do: :ok
+
+  defp check_content_length(%Tesla.Env{status: status}, _limit) when status in [204, 304],
+    do: :ok
+
+  defp check_content_length(env, limit) do
+    with value when is_binary(value) <- Tesla.get_header(env, "content-length"),
+         {length, ""} <- Integer.parse(value) do
+      if length > limit, do: {:error, :body_too_large}, else: :ok
+    else
+      _ -> :ok
+    end
+  end
+
+  defp check_body(body, limit) do
+    if byte_size(body) > limit, do: {:error, :body_too_large}, else: :ok
+  end
+end

lib/pleroma/upload/filter/exif/read_description.ex

@@ -0,0 +1,32 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2026 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Upload.Filter.Exif.ReadDescription do
+  @moduledoc """
+  Reads a description from image metadata when the upload has none yet.
+
+  Checks EXIF ImageDescription, then IPTC Caption-Abstract (and XMP for GIF). Blank
+  values and values longer than `[:instance, :description_limit]` are ignored.
+  """
+
+  @behaviour Pleroma.Upload.Filter
+
+  @impl true
+  def filter(%Pleroma.Upload{description: description}) when is_binary(description),
+    do: {:ok, :noop}
+
+  def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _} = upload) do
+    max_length = Pleroma.Config.get([:instance, :description_limit])
+
+    case ExifStripper.Description.read_file(file, max_length: max_length) do
+      description when is_binary(description) ->
+        {:ok, :filtered, %{upload | description: description}}
+
+      nil ->
+        {:ok, :noop}
+    end
+  end
+
+  def filter(_), do: {:ok, :noop}
+end

lib/pleroma/upload/filter/exif/strip_location.ex

@@ -0,0 +1,25 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2026 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Upload.Filter.Exif.StripLocation do
+  @moduledoc """
+  Strips GPS metadata from uploaded images and overwrites the file in place.
+
+  Supports JPEG, PNG, WebP, TIFF and ISO-BMFF images (HEIC, AVIF). This is lossless:
+  the image data is not re-encoded. Other image types are left untouched.
+  """
+
+  @behaviour Pleroma.Upload.Filter
+
+  @impl true
+  def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do
+    case ExifStripper.GpsStripper.strip_gps_data(file, file) do
+      :ok -> {:ok, :filtered}
+      :noop -> {:ok, :noop}
+      {:error, _} = error -> error
+    end
+  end
+
+  def filter(_), do: {:ok, :noop}
+end

lib/pleroma/upload/filter/exiftool.ex

@@ -0,0 +1,14 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2026 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Upload.Filter.Exiftool do
+  @moduledoc """
+  Deprecated: use `Pleroma.Upload.Filter.Exif.StripLocation`. This module delegates to it.
+  """
+
+  @behaviour Pleroma.Upload.Filter
+
+  @impl true
+  defdelegate filter(upload), to: Pleroma.Upload.Filter.Exif.StripLocation
+end

lib/pleroma/upload/filter/exiftool/read_description.ex

@@ -4,51 +4,11 @@
 
 defmodule Pleroma.Upload.Filter.Exiftool.ReadDescription do
   @moduledoc """
-  Gets a valid description from the related EXIF tags and provides them in the response if no description is provided yet.
-  It will first check ImageDescription, when that doesn't probide a valid description, it will check iptc:Caption-Abstract.
-  A valid description means the fields are filled in and not too long (see `:instance, :description_limit`).
+  Deprecated: use `Pleroma.Upload.Filter.Exif.ReadDescription`. This module delegates to it.
   """
+
   @behaviour Pleroma.Upload.Filter
 
-  def filter(%Pleroma.Upload{description: description})
-      when is_binary(description),
-      do: {:ok, :noop}
-
-  def filter(%Pleroma.Upload{tempfile: file} = upload),
-    do: {:ok, :filtered, upload |> Map.put(:description, read_description_from_exif_data(file))}
-
-  def filter(_, _), do: {:ok, :noop}
-
-  defp read_description_from_exif_data(file) do
-    nil
-    |> read_when_empty(file, "-ImageDescription")
-    |> read_when_empty(file, "-iptc:Caption-Abstract")
-  end
-
-  defp read_when_empty(current_description, _, _) when is_binary(current_description),
-    do: current_description
-
-  defp read_when_empty(_, file, tag) do
-    if File.exists?(file) do
-      try do
-        {tag_content, 0} =
-          System.cmd("exiftool", ["-m", "-b", "-s3", tag, file],
-            stderr_to_stdout: false,
-            parallelism: true
-          )
-
-        tag_content = String.trim(tag_content)
-
-        if tag_content != "" and
-             String.length(tag_content) <=
-               Pleroma.Config.get([:instance, :description_limit]),
-           do: tag_content,
-           else: nil
-      rescue
-        _ in ErlangError -> nil
-      end
-    else
-      nil
-    end
-  end
+  @impl true
+  defdelegate filter(upload), to: Pleroma.Upload.Filter.Exif.ReadDescription
 end

lib/pleroma/upload/filter/exiftool/strip_location.ex

@@ -4,30 +4,11 @@
 
 defmodule Pleroma.Upload.Filter.Exiftool.StripLocation do
   @moduledoc """
-  Strips GPS related EXIF tags and overwrites the file in place.
-  Also strips or replaces filesystem metadata e.g., timestamps.
+  Deprecated: use `Pleroma.Upload.Filter.Exif.StripLocation`. This module delegates to it.
   """
+
   @behaviour Pleroma.Upload.Filter
 
-  # Formats not compatible with exiftool at this time
-  def filter(%Pleroma.Upload{content_type: "image/heic"}), do: {:ok, :noop}
-  def filter(%Pleroma.Upload{content_type: "image/webp"}), do: {:ok, :noop}
-  def filter(%Pleroma.Upload{content_type: "image/svg" <> _}), do: {:ok, :noop}
-
-  def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do
-    try do
-      case System.cmd("exiftool", ["-m", "-overwrite_original", "-gps:all=", "-png:all=", file],
-             stderr_to_stdout: true,
-             parallelism: true
-           ) do
-        {_response, 0} -> {:ok, :filtered}
-        {error, _} -> {:error, error}
-      end
-    rescue
-      e in ErlangError ->
-        {:error, "#{__MODULE__}: #{inspect(e)}"}
-    end
-  end
-
-  def filter(_), do: {:ok, :noop}
+  @impl true
+  defdelegate filter(upload), to: Pleroma.Upload.Filter.Exif.StripLocation
 end

lib/pleroma/user.ex

@@ -969,7 +969,7 @@ defmodule Pleroma.User do
     candidates = Config.get([:instance, :autofollowed_nicknames])
 
     autofollowed_users =
-      User.Query.build(%{nickname: candidates, local: true, is_active: true})
+      User.Query.build(%{nickname: candidates, local: true, deactivated: false})
       |> Repo.all()
 
     follow_all(user, autofollowed_users)
@@ -1401,7 +1401,7 @@ defmodule Pleroma.User do
 
   @spec get_followers_query(User.t(), pos_integer() | nil) :: Ecto.Query.t()
   def get_followers_query(%User{} = user, nil) do
-    User.Query.build(%{followers: user, is_active: true})
+    User.Query.build(%{followers: user, deactivated: false})
   end
 
   def get_followers_query(%User{} = user, page) do
@@ -1476,7 +1476,7 @@ defmodule Pleroma.User do
       |> where([u], not u.hide_follows)
       |> select([u], u.id)
 
-    User.Query.build(%{is_active: true})
+    User.Query.build(%{deactivated: false})
     |> where([u], u.id not in ^[user.id, current_user.id])
     |> join(:inner, [u], r in FollowingRelationship,
       as: :followers_relationships,
@@ -1611,7 +1611,7 @@ defmodule Pleroma.User do
   @spec get_users_from_set([String.t()], keyword()) :: [User.t()]
   def get_users_from_set(ap_ids, opts \\ []) do
     local_only = Keyword.get(opts, :local_only, true)
-    criteria = %{ap_id: ap_ids, is_active: true}
+    criteria = %{ap_id: ap_ids, deactivated: false}
     criteria = if local_only, do: Map.put(criteria, :local, true), else: criteria
 
     User.Query.build(criteria)
@@ -1622,7 +1622,7 @@ defmodule Pleroma.User do
   def get_recipients_from_activity(%Activity{recipients: to, actor: actor}) do
     to = [actor | to]
 
-    query = User.Query.build(%{recipients_from_activity: to, local: true, is_active: true})
+    query = User.Query.build(%{recipients_from_activity: to, local: true, deactivated: false})
 
     query
     |> Repo.all()
@@ -2473,7 +2473,7 @@ defmodule Pleroma.User do
 
   @spec all_superusers() :: [User.t()]
   def all_superusers do
-    User.Query.build(%{super_users: true, local: true, is_active: true})
+    User.Query.build(%{super_users: true, local: true, deactivated: false})
     |> Repo.all()
   end
 

lib/pleroma/web/activity_pub/activity_pub.ex

@@ -11,6 +11,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   alias Pleroma.Conversation.Participation
   alias Pleroma.Filter
   alias Pleroma.Hashtag
+  alias Pleroma.Instances.Instance
   alias Pleroma.Maps
   alias Pleroma.Notification
   alias Pleroma.Object
@@ -21,6 +22,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   alias Pleroma.Upload
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.MRF
+  alias Pleroma.Web.ActivityPub.ObjectValidators.UserValidator
   alias Pleroma.Web.ActivityPub.Transmogrifier
   alias Pleroma.Web.Streamer
   alias Pleroma.Web.WebFinger
@@ -464,21 +466,28 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   end
 
   def fetch_activities_for_context_query(context, opts) do
-    public = [Constants.as_public()]
+    user = opts[:user]
 
     recipients =
-      if opts[:user],
-        do: [opts[:user].ap_id | User.following(opts[:user])] ++ public,
-        else: public
+      cond do
+        user && user.local ->
+          [Constants.as_public(), as_local_public(), user.ap_id | User.following(user)]
+
+        user ->
+          [Constants.as_public(), user.ap_id | User.following(user)]
+
+        true ->
+          [Constants.as_public()]
+      end
 
     from(activity in Activity)
     |> maybe_preload_objects(opts)
     |> maybe_preload_bookmarks(opts)
     |> maybe_set_thread_muted_field(opts)
-    |> restrict_unauthenticated(opts[:user])
+    |> restrict_unauthenticated(user)
     |> restrict_blocked(opts)
     |> restrict_blockers_visibility(opts)
-    |> restrict_recipients(recipients, opts[:user])
+    |> restrict_recipients(recipients, user)
     |> restrict_filtered(opts)
     |> where(
       [activity],
@@ -1924,6 +1933,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   def user_data_from_user_object(data, additional \\ []) do
     with {:ok, data} <- MRF.filter(data),
+         {:ok, data, _} <- UserValidator.validate(data, []),
          {:ok, data} <- object_to_user_data(data, additional) do
       {:ok, data}
     else
@@ -1946,6 +1956,10 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
         Logger.info("Rejected user #{ap_id}: #{inspect(reason)}")
         {:error, e}
 
+      {:error, :ratelimit = e} ->
+        Logger.debug("Backing off from fetching user #{ap_id}")
+        {:error, e}
+
       {:error, e} ->
         Logger.error("Could not decode user at fetch #{ap_id}, #{inspect(e)}")
         {:error, e}
@@ -2053,10 +2067,13 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   def enqueue_pin_fetches(_), do: nil
 
   def make_user_from_ap_id(ap_id, additional \\ []) do
-    user = User.get_cached_by_ap_id(ap_id)
-
     with {:ok, data} <- fetch_and_prepare_user_from_ap_id(ap_id, additional) do
       enqueue_pin_fetches(data)
+      # Keeps the instances table complete, so it can list our peers.
+      Instance.ensure_exists(data.ap_id)
+
+      # the fetched actor's id differs from ap_id when it was fetched from a display URL
+      user = User.get_cached_by_ap_id(data.ap_id)
 
       if user do
         user

lib/pleroma/web/activity_pub/activity_pub_controller.ex

@@ -22,8 +22,11 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
   alias Pleroma.Web.ControllerHelper
   alias Pleroma.Web.Endpoint
   alias Pleroma.Web.Federator
+  alias Pleroma.Web.OAuth.Token
   alias Pleroma.Web.Plugs.EnsureAuthenticatedPlug
   alias Pleroma.Web.Plugs.FederatingPlug
+  alias Pleroma.Web.Plugs.OAuthScopesPlug
+  alias Pleroma.Web.Plugs.RateLimiter
 
   require Logger
 
@@ -46,6 +49,27 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
     when action in [:read_inbox, :update_outbox, :whoami, :upload_media]
   )
 
+  # Checks scopes only for OAuth tokens (signatures and Basic auth already skip the check).
+  plug(:check_token_scopes, %{scopes: ["read:accounts"]} when action == :whoami)
+  plug(:check_token_scopes, %{scopes: ["read:statuses"]} when action == :read_inbox)
+  plug(:check_token_scopes, %{scopes: ["write"]} when action == :update_outbox)
+  plug(:check_token_scopes, %{scopes: ["write:media"]} when action == :upload_media)
+
+  plug(
+    :check_token_scopes,
+    %{fallback: :proceed_unauthenticated, scopes: ["read:statuses"]}
+    when action in [:outbox, :object_replies]
+  )
+
+  plug(
+    :check_token_scopes,
+    %{fallback: :proceed_unauthenticated, scopes: ["read:accounts"]}
+    when action in [:followers, :following]
+  )
+
+  # Shares the per-user bucket with Mastodon API status actions and emoji reactions.
+  plug(RateLimiter, [name: :statuses_actions] when action == :update_outbox)
+
   plug(Majic.Plug, [pool: Pleroma.MajicPool] when action in [:upload_media])
 
   plug(
@@ -57,6 +81,11 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
   plug(:log_inbox_metadata when action in [:inbox])
   plug(:relay_active? when action in [:relay])
 
+  defp check_token_scopes(%{assigns: %{token: %Token{}}} = conn, opts),
+    do: OAuthScopesPlug.call(conn, opts)
+
+  defp check_token_scopes(conn, _), do: conn
+
   defp relay_active?(conn, _) do
     if Pleroma.Config.get([:instance, :allow_relay]) do
       conn
@@ -316,7 +345,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
            {:sender_exists, User.get_or_fetch_by_ap_id(params["actor"])},
          {:recipient_active, true} <- {:recipient_active, recipient.is_active},
          {:sender_active, true} <- {:sender_active, actor.is_active},
-         true <- Utils.recipient_in_message(recipient, actor, params),
          params <- Utils.maybe_splice_recipient(recipient.ap_id, params) do
       Federator.incoming_ap_doc(params)
       json(conn, "ok")
@@ -361,18 +389,25 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
   end
 
   def inbox(%{assigns: %{valid_signature: false}} = conn, params) do
-    if unsupported_http_message_signature?(conn) do
-      reject_unsupported_http_message_signature(conn)
-    else
-      Federator.incoming_failed_signature_ap_doc(%{
-        method: conn.method,
-        req_headers: conn.req_headers,
-        request_path: conn.request_path,
-        params: params,
-        query_string: conn.query_string
-      })
+    cond do
+      unsupported_http_message_signature?(conn) ->
+        reject_unsupported_http_message_signature(conn)
 
-      json(conn, "ok")
+      reason = conn.assigns[:signature_error] ->
+        conn
+        |> put_status(:unauthorized)
+        |> json("error, invalid HTTP signature (#{reason})")
+
+      true ->
+        Federator.incoming_failed_signature_ap_doc(%{
+          method: conn.method,
+          req_headers: conn.req_headers,
+          request_path: conn.request_path,
+          params: params,
+          query_string: conn.query_string
+        })
+
+        json(conn, "ok")
     end
   end
 

lib/pleroma/web/activity_pub/mrf.ex

@@ -148,14 +148,13 @@ defmodule Pleroma.Web.ActivityPub.MRF do
   @spec subdomains_regex([String.t()]) :: [Regex.t()]
   def subdomains_regex(domains) when is_list(domains) do
     for domain <- domains do
-      try do
-        target = String.replace(domain, "*.", "(.*\\.)*")
-        ~r<^#{target}$>i
-      rescue
-        e ->
-          Logger.error("MRF: Invalid subdomain Regex: #{domain}")
-          reraise e, __STACKTRACE__
-      end
+      {prefix, rest} =
+        case domain do
+          "*." <> rest -> {"(.*\\.)?", rest}
+          _ -> {"", domain}
+        end
+
+      ~r<^#{prefix}#{Regex.escape(rest)}$>i
     end
   end
 
@@ -164,6 +163,39 @@ defmodule Pleroma.Web.ActivityPub.MRF do
     Enum.any?(domains, fn domain -> Regex.match?(domain, host) end)
   end
 
+  @doc """
+  Returns a function that is false for a host if SimplePolicy is enabled and
+  rejects it, or has an accept list without it. Used for deliveries and signed
+  fetches, not only for incoming activities. Build it once and reuse it: it
+  compiles the instance lists.
+  """
+  @spec instance_filter() :: (String.t() | nil -> boolean())
+  def instance_filter do
+    if Pleroma.Web.ActivityPub.MRF.SimplePolicy in get_policies() do
+      reject = simple_policy_domains(:reject)
+      accept = simple_policy_domains(:accept)
+
+      fn
+        host when is_binary(host) ->
+          not subdomain_match?(reject, host) and (accept == [] or subdomain_match?(accept, host))
+
+        _ ->
+          false
+      end
+    else
+      &is_binary/1
+    end
+  end
+
+  @spec instance_allowed?(String.t() | nil) :: boolean()
+  def instance_allowed?(host), do: instance_filter().(host)
+
+  defp simple_policy_domains(key) do
+    Pleroma.Config.get([:mrf_simple, key], [])
+    |> instance_list_from_tuples()
+    |> subdomains_regex()
+  end
+
   @spec instance_list_from_tuples([{String.t(), String.t()}]) :: [String.t()]
   def instance_list_from_tuples(list) do
     Enum.map(list, fn {instance, _} -> instance end)

lib/pleroma/web/activity_pub/mrf/object_age_policy.ex

@@ -36,16 +36,34 @@ defmodule Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy do
     end
   end
 
+  @spec delete_and_count(list(), term()) :: {integer(), list()}
+  defp delete_and_count(list, element), do: delete_and_count(list, element, {0, [], list})
+
+  defp delete_and_count([], _element, {0, _nlist, olist}), do: {0, olist}
+  defp delete_and_count([], _element, {count, nlist, _olist}), do: {count, Enum.reverse(nlist)}
+
+  defp delete_and_count([h | r], h, {count, nlist, olist}),
+    do: delete_and_count(r, h, {count + 1, nlist, olist})
+
+  defp delete_and_count([h | r], element, {count, nlist, olist}),
+    do: delete_and_count(r, element, {count, [h | nlist], olist})
+
+  defp insert_if_needed(list, oldcount, element) do
+    if oldcount <= 0 || Enum.member?(list, element) do
+      list
+    else
+      [element | list]
+    end
+  end
+
   defp check_delist(activity, actions) do
     if :delist in actions do
       with %User{} = user <- User.get_cached_by_ap_id(activity["actor"]) do
-        to =
-          List.delete(activity["to"] || [], Pleroma.Constants.as_public()) ++
-            [user.follower_address]
+        {pubcnt, to} = delete_and_count(activity["to"] || [], Pleroma.Constants.as_public())
+        {flwcnt, cc} = delete_and_count(activity["cc"] || [], user.follower_address)
 
-        cc =
-          List.delete(activity["cc"] || [], user.follower_address) ++
-            [Pleroma.Constants.as_public()]
+        cc = insert_if_needed(cc, pubcnt, Pleroma.Constants.as_public())
+        to = insert_if_needed(to, flwcnt, user.follower_address)
 
         activity =
           activity
@@ -67,8 +85,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy do
   defp check_strip_followers(activity, actions) do
     if :strip_followers in actions do
       with %User{} = user <- User.get_cached_by_ap_id(activity["actor"]) do
-        to = List.delete(activity["to"] || [], user.follower_address)
-        cc = List.delete(activity["cc"] || [], user.follower_address)
+        {_, to} = delete_and_count(activity["to"] || [], user.follower_address)
+        {_, cc} = delete_and_count(activity["cc"] || [], user.follower_address)
 
         activity =
           activity

lib/pleroma/web/activity_pub/mrf/simple_policy.ex

@@ -333,12 +333,12 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do
           %{
             key: :reject,
             description:
-              "List of instances to reject activities from (except deletes) and the reason for doing so"
+              "List of instances to reject activities from (except deletes) and the reason for doing so. We also do not deliver to them, and refuse their signed fetches in authorized fetch mode"
           },
           %{
             key: :accept,
             description:
-              "List of instances to only accept activities from (except deletes) and the reason for doing so"
+              "List of instances to only accept activities from (except deletes) and the reason for doing so. If set, we also only deliver to them, and refuse signed fetches from others in authorized fetch mode"
           },
           %{
             key: :followers_only,

lib/pleroma/web/activity_pub/object_validator.ex

@@ -11,7 +11,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
 
   @behaviour Pleroma.Web.ActivityPub.ObjectValidator.Validating
 
-  import Pleroma.Constants, only: [activity_types: 0, object_types: 0]
+  import Pleroma.Constants, only: [activity_types: 0, actor_types: 0, object_types: 0]
 
   alias Pleroma.Activity
   alias Pleroma.EctoType.ActivityPub.ObjectValidators
@@ -37,6 +37,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
   alias Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.UndoValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.UpdateValidator
+  alias Pleroma.Web.ActivityPub.ObjectValidators.UserValidator
 
   @impl true
   def validate(object, meta)
@@ -77,10 +78,16 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
            |> Ecto.Changeset.apply_action(:insert) do
       object = stringify_keys(object)
       undone_object = Activity.get_by_ap_id(object["object"])
+      outgoing_blocks = Pleroma.Config.get([:activitypub, :outgoing_blocks])
+      # if we're undoing a block, and do not permit federating that:
+      do_not_federate =
+        Keyword.get(meta, :do_not_federate) ||
+          (Map.get(undone_object.data, "type") == "Block" && !outgoing_blocks)
 
       meta =
         meta
         |> Keyword.put(:object_data, undone_object.data)
+        |> Keyword.put(:do_not_federate, do_not_federate)
 
       {:ok, object, meta}
     end
@@ -226,7 +233,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
   end
 
   def validate(%{"type" => type} = object, meta) when type == "Update" do
-    with {:ok, object} <-
+    with :ok <- validate_updated_actor(object, meta),
+         {:ok, object} <-
            object
            |> UpdateValidator.cast_and_validate(meta)
            |> Ecto.Changeset.apply_action(:insert) do
@@ -247,6 +255,19 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
 
   def validate(o, m), do: {:error, {:validator_not_set, {o, m}}}
 
+  defp validate_updated_actor(%{"object" => %{"type" => type} = actor}, meta)
+       when type in actor_types() do
+    with {_, false} <- {:local, Access.get(meta, :local, false)},
+         {:ok, _, _} <- UserValidator.validate(actor, meta) do
+      :ok
+    else
+      {:local, true} -> :ok
+      error -> error
+    end
+  end
+
+  defp validate_updated_actor(_, _), do: :ok
+
   def cast_and_apply_and_stringify_with_history(object) do
     do_separate_with_history(object, fn object ->
       with {:ok, object_data} <- cast_and_apply(object),

lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex

@@ -213,7 +213,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidator do
 
   defp fix_misskey_content(object), do: object
 
-  defp fix(data) do
+  def fix(data) do
     data
     |> CommonFixes.fix_actor()
     |> CommonFixes.fix_object_defaults()

lib/pleroma/web/activity_pub/object_validators/emoji_react_validator.ex

@@ -57,7 +57,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.EmojiReactValidator do
       |> CommonFixes.fix_actor()
       |> CommonFixes.fix_activity_addressing()
 
-    data = Map.put_new(data, "tag", [])
+    data = fix_tag(data)
 
     case Object.normalize(data["object"]) do
       %Object{} = object ->
@@ -70,6 +70,10 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.EmojiReactValidator do
     end
   end
 
+  # JSON-LD compaction turns a single-element array into a bare object
+  defp fix_tag(%{"tag" => %{} = tag} = data), do: Map.put(data, "tag", [tag])
+  defp fix_tag(data), do: Map.put_new(data, "tag", [])
+
   defp fix_emoji_qualification(%{"content" => emoji} = data) do
     new_emoji = Pleroma.Emoji.fully_qualify_emoji(emoji)
 

lib/pleroma/web/activity_pub/object_validators/question_validator.ex

@@ -6,10 +6,9 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do
   use Ecto.Schema
 
   alias Pleroma.EctoType.ActivityPub.ObjectValidators
-  alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes
+  alias Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations
   alias Pleroma.Web.ActivityPub.ObjectValidators.QuestionOptionsValidator
-  alias Pleroma.Web.ActivityPub.Transmogrifier
 
   import Ecto.Changeset
 
@@ -62,11 +61,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do
 
   defp fix(data) do
     data
-    |> CommonFixes.fix_actor()
-    |> CommonFixes.fix_object_defaults()
-    |> CommonFixes.fix_quote_url()
-    |> CommonFixes.fix_likes()
-    |> Transmogrifier.fix_emoji()
+    |> ArticleNotePageValidator.fix()
     |> fix_closed()
   end
 

lib/pleroma/web/activity_pub/object_validators/user_validator.ex

@@ -0,0 +1,53 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.ObjectValidators.UserValidator do
+  @moduledoc """
+  Checks that a remote actor document only points at its own host.
+
+  We deliver to the inbox and shared inbox, use the collections for
+  addressing and visibility checks, and map signature key ids back to
+  the actor. If an actor could point any of these at another host, it
+  could redirect our signed deliveries or claim another actor's
+  collections. Optional fields are only checked when present; null and
+  empty strings count as missing.
+
+  Based on Akkoma's UserValidator.
+  """
+
+  @behaviour Pleroma.Web.ActivityPub.ObjectValidator.Validating
+
+  alias Pleroma.Object.Containment
+
+  @collections ["inbox", "followers", "following", "featured"]
+
+  @impl true
+  def validate(%{"id" => id} = data, meta) when is_binary(id) do
+    fields =
+      Enum.map(@collections, &{&1, data[&1]}) ++
+        [
+          {"sharedInbox", get_in_map(data, ["endpoints", "sharedInbox"])},
+          {"publicKey", get_in_map(data, ["publicKey", "id"])}
+        ]
+
+    case Enum.find(fields, fn {_, value} -> not same_origin?(id, value) end) do
+      nil -> {:ok, data, meta}
+      {field, _} -> {:error, "#{field} is not on the same host as the actor"}
+    end
+  end
+
+  def validate(_data, _meta), do: {:error, "Actor has no id"}
+
+  defp same_origin?(_id, value) when value in [nil, ""], do: true
+  defp same_origin?(id, %{"id" => value}), do: same_origin?(id, value)
+  defp same_origin?(id, value), do: Containment.same_origin(id, value) == :ok
+
+  defp get_in_map(data, [key | rest]) do
+    case data do
+      %{^key => value} when rest == [] -> value
+      %{^key => value} -> get_in_map(value, rest)
+      _ -> nil
+    end
+  end
+end

lib/pleroma/web/activity_pub/publisher.ex

@@ -11,6 +11,7 @@ defmodule Pleroma.Web.ActivityPub.Publisher do
   alias Pleroma.Object
   alias Pleroma.Repo
   alias Pleroma.User
+  alias Pleroma.Web.ActivityPub.MRF
   alias Pleroma.Web.ActivityPub.Publisher.Prepared
   alias Pleroma.Web.ActivityPub.Relay
   alias Pleroma.Workers.PublisherWorker
@@ -221,18 +222,25 @@ defmodule Pleroma.Web.ActivityPub.Publisher do
     end
   end
 
-  def should_federate?(nil, _), do: false
-  def should_federate?(_, true), do: true
+  def should_federate?(inbox, public), do: federation_filter(public).(inbox)
 
-  def should_federate?(inbox, _) do
-    %{host: host} = URI.parse(inbox)
+  # Compiles the instance lists once, for filtering many inboxes.
+  defp federation_filter(public) do
+    allowed? = MRF.instance_filter()
 
-    quarantined_instances =
+    quarantined =
       Config.get([:instance, :quarantined_instances], [])
-      |> Pleroma.Web.ActivityPub.MRF.instance_list_from_tuples()
-      |> Pleroma.Web.ActivityPub.MRF.subdomains_regex()
+      |> MRF.instance_list_from_tuples()
+      |> MRF.subdomains_regex()
 
-    !Pleroma.Web.ActivityPub.MRF.subdomain_match?(quarantined_instances, host)
+    fn
+      nil ->
+        false
+
+      inbox ->
+        %{host: host} = URI.parse(inbox)
+        allowed?.(host) and (public or not MRF.subdomain_match?(quarantined, host))
+    end
   end
 
   @spec recipients(User.t(), Activity.t()) :: [[User.t()]]
@@ -273,36 +281,18 @@ defmodule Pleroma.Web.ActivityPub.Publisher do
   defp maybe_use_sharedinbox(%User{shared_inbox: shared_inbox}), do: shared_inbox
 
   @doc """
-  Determine a user inbox to use based on heuristics.  These heuristics
-  are based on an approximation of the ``sharedInbox`` rules in the
-  [ActivityPub specification][ap-sharedinbox].
+  Determines the inbox to deliver an activity to.
 
-  Please do not edit this function (or its children) without reading
-  the spec, as editing the code is likely to introduce some breakage
-  without some familiarity.
-
-     [ap-sharedinbox]: https://www.w3.org/TR/activitypub/#shared-inbox-delivery
+  Like Mastodon, GoToSocial and Akkoma, this always prefers the shared inbox,
+  which saves duplicate deliveries to the same server. Only activities with
+  `bcc` or `bto` go to personal inboxes, because the receiver rebuilds that
+  addressing from the inbox the activity arrived at.
   """
-  def determine_inbox(
-        %Activity{data: activity_data},
-        %User{inbox: inbox} = user
-      ) do
-    to = activity_data["to"] || []
-    cc = activity_data["cc"] || []
-    type = activity_data["type"]
-
-    cond do
-      type == "Delete" ->
-        maybe_use_sharedinbox(user)
-
-      Pleroma.Constants.as_public() in to || Pleroma.Constants.as_public() in cc ->
-        maybe_use_sharedinbox(user)
-
-      length(to) + length(cc) > 1 ->
-        maybe_use_sharedinbox(user)
-
-      true ->
-        inbox
+  def determine_inbox(%Activity{data: activity_data}, %User{inbox: inbox} = user) do
+    if Enum.empty?(activity_data["bcc"] || []) and Enum.empty?(activity_data["bto"] || []) do
+      maybe_use_sharedinbox(user)
+    else
+      inbox
     end
   end
 
@@ -312,7 +302,7 @@ defmodule Pleroma.Web.ActivityPub.Publisher do
 
   def publish(%User{} = actor, %{data: %{"bcc" => bcc}} = activity)
       when is_list(bcc) and bcc != [] do
-    public = public?(activity)
+    should_federate? = activity |> public?() |> federation_filter()
 
     [priority_recipients, recipients] = recipients(actor, activity)
 
@@ -324,7 +314,7 @@ defmodule Pleroma.Web.ActivityPub.Publisher do
           determine_inbox(activity, user)
         end)
         |> Enum.uniq()
-        |> Enum.filter(fn inbox -> should_federate?(inbox, public) end)
+        |> Enum.filter(should_federate?)
         |> Instances.filter_reachable()
       end)
 
@@ -360,6 +350,8 @@ defmodule Pleroma.Web.ActivityPub.Publisher do
       Relay.publish(activity)
     end
 
+    should_federate? = federation_filter(public)
+
     [priority_inboxes, inboxes] =
       recipients(actor, activity)
       |> Enum.map(fn recipients ->
@@ -368,7 +360,7 @@ defmodule Pleroma.Web.ActivityPub.Publisher do
           determine_inbox(activity, user)
         end)
         |> Enum.uniq()
-        |> Enum.filter(fn inbox -> should_federate?(inbox, public) end)
+        |> Enum.filter(should_federate?)
       end)
 
     inboxes = inboxes -- priority_inboxes

lib/pleroma/web/activity_pub/transmogrifier.ex

@@ -799,7 +799,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
        ) do
     with %User{} = origin_user <- User.get_cached_by_ap_id(origin_actor),
          {:ok, %User{} = target_user} <- User.get_or_fetch_by_ap_id(target_actor),
-         true <- origin_actor in target_user.also_known_as do
+         {:ok, %User{} = target_user} <- refetch_unless_aliased(target_user, origin_actor) do
       ActivityPub.move(origin_user, target_user, false)
     else
       _e -> :error
@@ -1204,4 +1204,20 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   def maybe_fix_user_url(data), do: data
 
   def maybe_fix_user_object(data), do: maybe_fix_user_url(data)
+
+  # A Move usually follows right after the alias was added, which the cached
+  # remote target may not have yet.
+  defp refetch_unless_aliased(%User{} = target, origin_ap_id, refetch? \\ true) do
+    cond do
+      origin_ap_id in target.also_known_as ->
+        {:ok, target}
+
+      refetch? and not target.local ->
+        with {:ok, %User{} = target} <- User.fetch_by_ap_id(target.ap_id),
+             do: refetch_unless_aliased(target, origin_ap_id, false)
+
+      true ->
+        :error
+    end
+  end
 end

lib/pleroma/web/activity_pub/utils.ex

@@ -78,22 +78,6 @@ defmodule Pleroma.Web.ActivityPub.Utils do
       [params["to"], params["cc"], params["bto"], params["bcc"]]
       |> Enum.any?(&label_in_collection?(label, &1))
 
-  @spec unaddressed_message?(map()) :: boolean()
-  def unaddressed_message?(params),
-    do:
-      [params["to"], params["cc"], params["bto"], params["bcc"]]
-      |> Enum.all?(fn
-        nil -> true
-        [] -> true
-        _ -> false
-      end)
-
-  @spec recipient_in_message(User.t(), User.t(), map()) :: boolean()
-  def recipient_in_message(%User{ap_id: ap_id} = recipient, %User{} = actor, params),
-    do:
-      label_in_message?(ap_id, params) || unaddressed_message?(params) ||
-        User.following?(recipient, actor)
-
   defp extract_list(target) when is_binary(target), do: [target]
   defp extract_list(lst) when is_list(lst), do: lst
   defp extract_list(_), do: []

lib/pleroma/web/api_spec/cast_and_validate.ex

@@ -20,6 +20,9 @@ defmodule Pleroma.Web.ApiSpec.CastAndValidate do
 
   require Logger
 
+  # How often to drop offending query params and retry in permissive mode
+  @max_retries 20
+
   @impl Plug
   def init(opts) do
     opts
@@ -114,34 +117,49 @@ defmodule Pleroma.Web.ApiSpec.CastAndValidate do
          false = _strict,
          cast_opts
        ) do
-    case OpenApiSpex.cast_and_validate(spec, operation, conn, content_type) do
-      {:ok, conn} ->
-        {:ok, conn}
+    permissive_cast_and_validate(spec, operation, conn, content_type, cast_opts, @max_retries)
+  end
 
-      # Remove unexpected query params and cast/validate again
-      {:error, errors} ->
-        query_params =
-          Enum.reduce(errors, conn.query_params, fn
-            %{reason: :unexpected_field, name: name, path: [name]}, params ->
-              Map.delete(params, name)
+  # OpenApiSpex reports one offending param at a time, so drop it and cast/validate again
+  defp permissive_cast_and_validate(
+         spec,
+         operation,
+         %Conn{} = conn,
+         content_type,
+         cast_opts,
+         retries
+       ) do
+    with {:error, errors} = error <-
+           OpenApiSpex.cast_and_validate(spec, operation, conn, content_type, cast_opts) do
+      query_params = drop_invalid_query_params(errors, conn.query_params)
 
-            # Filter out empty params
-            %{reason: :invalid_type, path: [name_atom], value: ""}, params ->
-              Map.delete(params, to_string(name_atom))
-
-            %{reason: :invalid_enum, name: nil, path: path, value: value}, params ->
-              path = path |> Enum.reverse() |> tl() |> Enum.reverse() |> list_items_to_string()
-              update_in(params, path, &List.delete(&1, value))
-
-            _, params ->
-              params
-          end)
-
-        conn = %Conn{conn | query_params: query_params}
-        OpenApiSpex.cast_and_validate(spec, operation, conn, content_type, cast_opts)
+      if retries > 0 and query_params != conn.query_params do
+        conn = %{conn | query_params: query_params}
+        permissive_cast_and_validate(spec, operation, conn, content_type, cast_opts, retries - 1)
+      else
+        error
+      end
     end
   end
 
+  defp drop_invalid_query_params(errors, query_params) do
+    Enum.reduce(errors, query_params, fn
+      %{reason: :unexpected_field, name: name, path: [name]}, params ->
+        Map.delete(params, name)
+
+      # Filter out empty params
+      %{reason: :invalid_type, path: [name_atom], value: ""}, params ->
+        Map.delete(params, to_string(name_atom))
+
+      %{reason: :invalid_enum, name: nil, path: path, value: value}, params ->
+        path = path |> Enum.reverse() |> tl() |> Enum.reverse() |> list_items_to_string()
+        update_in(params, path, &List.delete(&1, value))
+
+      _, params ->
+        params
+    end)
+  end
+
   defp list_items_to_string(list) do
     Enum.map(list, fn
       i when is_atom(i) -> to_string(i)

lib/pleroma/web/api_spec/operations/account_operation.ex

@@ -48,7 +48,7 @@ defmodule Pleroma.Web.ApiSpec.AccountOperation do
       description: "Test to make sure that the user token works.",
       summary: "Verify account credentials",
       operationId: "AccountController.verify_credentials",
-      security: [%{"oAuth" => ["read:accounts"]}],
+      security: [%{"oAuth" => ["read:accounts"]}, %{"oAuth" => ["profile"]}],
       responses: %{
         200 => Operation.response("Account", "application/json", Account)
       }

lib/pleroma/web/api_spec/schemas/status.ex

@@ -234,6 +234,29 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Status do
         type: :integer,
         description: "How many statuses quoted this status."
       },
+      quote: %Schema{
+        type: :object,
+        nullable: true,
+        description:
+          "The status being quoted, if any (Mastodon 4.4+). Quoted statuses only have `quoted_status_id`.",
+        properties: %{
+          state: %Schema{
+            type: :string,
+            enum: ["accepted", "unauthorized", "deleted"],
+            description: "Whether the quoted status can be shown"
+          },
+          quoted_status: %Schema{
+            allOf: [%OpenApiSpex.Reference{"$ref": "#/components/schemas/Status"}],
+            nullable: true,
+            description: "The quoted status, if it can be shown"
+          },
+          quoted_status_id: %Schema{
+            type: :string,
+            nullable: true,
+            description: "ID of the quoted status, if it can be shown"
+          }
+        }
+      },
       reblog: %Schema{
         allOf: [%OpenApiSpex.Reference{"$ref": "#/components/schemas/Status"}],
         nullable: true,

lib/pleroma/web/common_api/utils.ex

@@ -324,7 +324,7 @@ defmodule Pleroma.Web.CommonAPI.Utils do
 
   def format_input(text, "text/x.misskeymarkdown", options) do
     text
-    |> Formatter.markdown_to_html(%{breaks: true})
+    |> Formatter.markdown_to_html(%{breaks: true, strip_email_autolinks: true})
     |> safe_mfm_to_html()
     |> Formatter.linkify(options)
     |> Formatter.html_escape("text/x.misskeymarkdown")
@@ -435,13 +435,56 @@ defmodule Pleroma.Web.CommonAPI.Utils do
           %{}
       end
 
-    tagged_mentions = maybe_extract_mentions(object_data)
+    tagged_mentions =
+      maybe_extract_mentions(object_data)
+      |> filter_tags_to_qualified(activity)
 
     recipients ++ tagged_mentions
   end
 
   def maybe_notify_mentioned_recipients(recipients, _), do: recipients
 
+  # Filter tagged mentions to ensure they can actually access the status
+  defp filter_tags_to_qualified(tagged_mentions, %Activity{
+         recipients: recipients,
+         data: data,
+         actor: actor
+       }) do
+    recipients =
+      if recipients && recipients != [],
+        do: recipients,
+        else:
+          (data["to"] || []) ++ (data["cc"] || []) ++ (data["bto"] || []) ++ (data["bcc"] || [])
+
+    cond do
+      recipients == [] ->
+        []
+
+      Pleroma.Constants.as_public() in recipients ->
+        tagged_mentions
+
+      true ->
+        author = User.get_cached_by_ap_id(actor)
+        to_followers = match?(%User{}, author) and author.follower_address in recipients
+
+        Enum.filter(tagged_mentions, fn tagged ->
+          cond do
+            tagged in recipients ->
+              true
+
+            to_followers ->
+              case User.get_cached_by_ap_id(tagged) do
+                %User{} = tagged_user -> User.following?(tagged_user, author)
+                _ -> false
+              end
+
+            true ->
+              false
+          end
+        end)
+    end
+  end
+
   def maybe_notify_followers(recipients, %Activity{data: %{"type" => "Move"}} = activity) do
     with %User{} = user <- User.get_cached_by_ap_id(activity.actor) do
       user

lib/pleroma/web/mastodon_api/controllers/account_controller.ex

@@ -24,6 +24,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do
   alias Pleroma.Web.MastodonAPI.MastodonAPI
   alias Pleroma.Web.MastodonAPI.StatusView
   alias Pleroma.Web.OAuth.OAuthController
+  alias Pleroma.Web.OAuth.Token
   alias Pleroma.Web.Plugs.OAuthScopesPlug
   alias Pleroma.Web.Plugs.RateLimiter
   alias Pleroma.Web.Registration
@@ -49,8 +50,12 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do
 
   plug(
     OAuthScopesPlug,
-    %{scopes: ["read:accounts"]}
-    when action in [:verify_credentials, :endorsements, :own_endorsements]
+    %{scopes: ["read:accounts", "profile"]} when action == :verify_credentials
+  )
+
+  plug(
+    OAuthScopesPlug,
+    %{scopes: ["read:accounts"]} when action in [:endorsements, :own_endorsements]
   )
 
   plug(
@@ -161,7 +166,18 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do
   end
 
   @doc "GET /api/v1/accounts/verify_credentials"
-  def verify_credentials(%{assigns: %{user: user}} = conn, _) do
+  def verify_credentials(%{assigns: %{user: user, token: %Token{scopes: scopes}}} = conn, _) do
+    if OAuthScopesPlug.filter_descendants(["read:accounts"], scopes) == [] do
+      # The profile scope only identifies the user: show what anyone can see.
+      render(conn, "show.json", user: user, skip_visibility_check: true)
+    else
+      render_own_account(conn, user)
+    end
+  end
+
+  def verify_credentials(%{assigns: %{user: user}} = conn, _), do: render_own_account(conn, user)
+
+  defp render_own_account(conn, user) do
     chat_token = Phoenix.Token.sign(conn, "user socket", user.id)
 
     render(conn, "show.json",

lib/pleroma/web/mastodon_api/controllers/status_controller.ex

@@ -92,13 +92,13 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do
 
   plug(
     RateLimiter,
-    [name: :status_id_action, bucket_name: "status_id_action:reblog_unreblog", params: [:id]]
+    [name: :status_id_action, bucket_name: "status_id_action:reblog_unreblog", params: ["id"]]
     when action in ~w(reblog unreblog)a
   )
 
   plug(
     RateLimiter,
-    [name: :status_id_action, bucket_name: "status_id_action:fav_unfav", params: [:id]]
+    [name: :status_id_action, bucket_name: "status_id_action:fav_unfav", params: ["id"]]
     when action in ~w(favourite unfavourite)a
   )
 

lib/pleroma/web/mastodon_api/views/marker_view.ex

@@ -5,13 +5,15 @@
 defmodule Pleroma.Web.MastodonAPI.MarkerView do
   use Pleroma.Web, :view
 
+  alias Pleroma.Web.CommonAPI.Utils
+
   def render("markers.json", %{markers: markers}) do
     Map.new(markers, fn m ->
       {m.timeline,
        %{
          last_read_id: m.last_read_id,
          version: m.lock_version,
-         updated_at: NaiveDateTime.to_iso8601(m.updated_at),
+         updated_at: Utils.to_masto_date(m.updated_at),
          pleroma: %{
            unread_count: m.unread_count
          }

lib/pleroma/web/mastodon_api/views/status_view.ex

@@ -455,6 +455,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
       language: get_language(object),
       emojis: build_emojis(object.data["emoji"]),
       quotes_count: object.data["quotesCount"] || 0,
+      quote: build_quote(object, quote_activity, quote_post, opts),
       pleroma: %{
         local: activity.local,
         conversation_id: get_context_id(activity),
@@ -675,7 +676,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
   def render("attachment_meta.json", %{
         attachment: %{"url" => [%{"width" => width, "height" => height} | _]}
       })
-      when is_integer(width) and is_integer(height) do
+      when is_integer(width) and is_integer(height) and width > 0 and height > 0 do
     %{
       original: %{
         width: width,
@@ -691,7 +692,9 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
     %{ancestors: ancestors, descendants: descendants} =
       activities
       |> Enum.reverse()
-      |> Enum.group_by(fn %{id: id} -> if id < activity.id, do: :ancestors, else: :descendants end)
+      |> Enum.group_by(fn %{id: id} ->
+        if id < activity.id, do: :ancestors, else: :descendants
+      end)
       |> Map.put_new(:ancestors, [])
       |> Map.put_new(:descendants, [])
 
@@ -742,6 +745,26 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
     end
   end
 
+  # Mastodon (4.4+) quote entity. Quoted statuses get a shallow quote that only
+  # has the id, as quotes are only rendered one level deep.
+  defp build_quote(%Object{data: %{"quoteUrl" => url}}, quote_activity, quote_post, opts)
+       when is_binary(url) and url != "" do
+    state =
+      cond do
+        is_nil(quote_activity) -> "deleted"
+        visible_for_user?(quote_activity, opts[:for]) -> "accepted"
+        true -> "unauthorized"
+      end
+
+    if opts[:show_quote] == false do
+      %{state: state, quoted_status_id: if(state == "accepted", do: to_string(quote_activity.id))}
+    else
+      %{state: state, quoted_status: if(state == "accepted", do: quote_post)}
+    end
+  end
+
+  defp build_quote(_object, _quote_activity, _quote_post, _opts), do: nil
+
   def get_quote(activity, %{quoted_activities: quoted_activities}) do
     object = Object.normalize(activity, fetch: false)
 

lib/pleroma/web/media_proxy.ex

@@ -41,10 +41,11 @@ defmodule Pleroma.Web.MediaProxy do
   end
 
   def url(url) when is_nil(url) or url == "", do: nil
+  def url("//" <> _ = url), do: url("https:" <> url)
   def url("/" <> _ = url), do: url
 
   def url(url) do
-    if enabled?() and url_proxiable?(url) do
+    if enabled?() and not data_url?(url) and url_proxiable?(url) do
       encode_url(url)
     else
       url
@@ -56,8 +57,11 @@ defmodule Pleroma.Web.MediaProxy do
     not local?(url) and not whitelisted?(url)
   end
 
-  def preview_url(url, preview_params \\ []) do
-    if preview_enabled?() do
+  def preview_url(url, preview_params \\ [])
+  def preview_url("//" <> _ = url, pparams), do: preview_url("https:" <> url, pparams)
+
+  def preview_url(url, preview_params) do
+    if preview_enabled?() and not data_url?(url) do
       encode_preview_url(url, preview_params)
     else
       url(url)
@@ -70,6 +74,8 @@ defmodule Pleroma.Web.MediaProxy do
   #   non-local non-whitelisted URLs through it and be sure that body size constraint is preserved.
   def preview_enabled?, do: enabled?() and !!Config.get([:media_preview_proxy, :enabled])
 
+  defp data_url?(url), do: url |> String.slice(0, 5) |> String.downcase() == "data:"
+
   def local?(url), do: String.starts_with?(url, Endpoint.url())
 
   def whitelisted?(url) do

lib/pleroma/web/media_proxy/media_proxy_controller.ex

@@ -7,7 +7,6 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
 
   alias Pleroma.Config
   alias Pleroma.Helpers.MediaHelper
-  alias Pleroma.Helpers.UriHelper
   alias Pleroma.ReverseProxy
   alias Pleroma.Web.MediaProxy
   alias Plug.Conn
@@ -67,9 +66,6 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
         static and content_type == "image/gif" ->
           handle_jpeg_preview(conn, media_proxy_url)
 
-        static ->
-          drop_static_param_and_redirect(conn)
-
         content_type == "image/gif" ->
           conn
           |> put_status(301)
@@ -166,15 +162,6 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
     end
   end
 
-  defp drop_static_param_and_redirect(conn) do
-    uri_without_static_param =
-      conn
-      |> current_url()
-      |> UriHelper.modify_uri_params(%{}, ["static"])
-
-    redirect(conn, external: uri_without_static_param)
-  end
-
   defp fallback_on_preview_error(conn, media_proxy_url) do
     redirect(conn, external: media_proxy_url)
   end

lib/pleroma/web/o_auth/token_controller.ex

@@ -13,7 +13,8 @@ defmodule Pleroma.Web.OAuth.TokenController do
   require Logger
 
   plug(:skip_auth when action == :confirm_email)
-  plug(:skip_plug, OAuthScopesPlug when action in [:oauth_tokens, :revoke_token])
+  plug(OAuthScopesPlug, %{scopes: ["read:security"]} when action == :oauth_tokens)
+  plug(OAuthScopesPlug, %{scopes: ["write:security"]} when action == :revoke_token)
 
   action_fallback(:errors)
 

lib/pleroma/web/pleroma_api/controllers/emoji_reaction_controller.ex

@@ -12,6 +12,7 @@ defmodule Pleroma.Web.PleromaAPI.EmojiReactionController do
   alias Pleroma.Web.CommonAPI
   alias Pleroma.Web.MastodonAPI.StatusView
   alias Pleroma.Web.Plugs.OAuthScopesPlug
+  alias Pleroma.Web.Plugs.RateLimiter
 
   plug(Pleroma.Web.ApiSpec.CastAndValidate)
   plug(OAuthScopesPlug, %{scopes: ["write:statuses"]} when action in [:create, :delete])
@@ -22,6 +23,14 @@ defmodule Pleroma.Web.PleromaAPI.EmojiReactionController do
     when action == :index
   )
 
+  plug(
+    RateLimiter,
+    [name: :status_id_action, bucket_name: "status_id_action:emoji_react", params: [:id]]
+    when action in [:create, :delete]
+  )
+
+  plug(RateLimiter, [name: :statuses_actions] when action in [:create, :delete])
+
   defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.EmojiReactionOperation
 
   action_fallback(Pleroma.Web.MastodonAPI.FallbackController)

lib/pleroma/web/pleroma_api/controllers/password_controller.ex

@@ -1,52 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Web.PleromaAPI.PasswordController do
-  @moduledoc """
-  The module contains functions for password reset.
-  """
-
-  use Pleroma.Web, :controller
-
-  require Logger
-
-  import Pleroma.Web.ControllerHelper, only: [json_response: 3]
-
-  alias Pleroma.PasswordResetToken
-  alias Pleroma.Repo
-  alias Pleroma.User
-  alias Pleroma.Web.Registration
-
-  plug(Pleroma.Web.Plugs.RateLimiter, [name: :request] when action == :request)
-
-  @doc "POST /auth/password"
-  def request(conn, params) do
-    nickname_or_email = params["email"] || params["nickname"]
-
-    Registration.password_reset(nickname_or_email)
-
-    json_response(conn, :no_content, "")
-  end
-
-  def reset(conn, %{"token" => token}) do
-    with %{used: false} = token <- Repo.get_by(PasswordResetToken, %{token: token}),
-         false <- PasswordResetToken.expired?(token),
-         %User{} = user <- User.get_cached_by_id(token.user_id) do
-      render(conn, "reset.html", %{
-        token: token,
-        user: user
-      })
-    else
-      _e -> render(conn, "invalid_token.html")
-    end
-  end
-
-  def do_reset(conn, %{"data" => data}) do
-    with {:ok, _} <- PasswordResetToken.reset_password(data["token"], data) do
-      render(conn, "reset_success.html")
-    else
-      _e -> render(conn, "reset_failed.html")
-    end
-  end
-end

lib/pleroma/web/pleroma_api/controllers/token_controller.ex

@@ -1,59 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Web.PleromaAPI.TokenController do
-  use Pleroma.Web, :controller
-
-  alias Pleroma.User
-  alias Pleroma.Web.OAuth.Token
-  alias Pleroma.Web.PleromaAPI.TokenView
-  alias Pleroma.Web.Plugs.OAuthScopesPlug
-
-  require Logger
-
-  plug(:skip_auth when action == :confirm_email)
-  plug(:skip_plug, OAuthScopesPlug when action in [:oauth_tokens, :revoke_token])
-
-  action_fallback(:errors)
-
-  def confirm_email(conn, %{"user_id" => uid, "token" => token}) do
-    with %User{} = user <- User.get_cached_by_id(uid),
-         true <- user.local and !user.is_confirmed and user.confirmation_token == token,
-         {:ok, _} <- User.confirm(user) do
-      redirect(conn, to: "/")
-    end
-  end
-
-  def oauth_tokens(%{assigns: %{user: user}} = conn, _params) do
-    with oauth_tokens <- Token.get_user_tokens(user) do
-      conn
-      |> put_view(TokenView)
-      |> render("index.json", %{tokens: oauth_tokens})
-    end
-  end
-
-  def revoke_token(%{assigns: %{user: user}} = conn, %{"id" => id} = _params) do
-    Token.delete_user_token(user, id)
-
-    json_reply(conn, 201, "")
-  end
-
-  defp errors(conn, {:param_cast, _}) do
-    conn
-    |> put_status(400)
-    |> json("Invalid parameters")
-  end
-
-  defp errors(conn, _) do
-    conn
-    |> put_status(500)
-    |> json("Something went wrong")
-  end
-
-  defp json_reply(conn, status, json) do
-    conn
-    |> put_resp_content_type("application/json")
-    |> send_resp(status, json)
-  end
-end

lib/pleroma/web/plugs/http_signature_plug.ex

@@ -45,20 +45,52 @@ defmodule Pleroma.Web.Plugs.HTTPSignaturePlug do
 
   defp maybe_assign_valid_signature(conn) do
     if has_signature_header?(conn) do
-      # we replace the digest header with the one we computed in DigestPlug
-      conn =
-        case conn do
-          %{assigns: %{digest: digest}} = conn -> put_req_header(conn, "digest", digest)
-          conn -> conn
-        end
+      conn = put_computed_digest(conn)
 
-      assign(conn, :valid_signature, Signature.validate_signature(conn))
+      case check_signature_policy(conn) do
+        :ok ->
+          assign(conn, :valid_signature, Signature.validate_signature(conn))
+
+        {:error, reason} ->
+          reject_signature(conn, reason)
+      end
     else
       Logger.debug("No signature header!")
       conn
     end
   end
 
+  # we replace the digest header with the one we computed in DigestPlug
+  defp put_computed_digest(%{assigns: %{digest: digest}} = conn),
+    do: put_req_header(conn, "digest", digest)
+
+  defp put_computed_digest(conn), do: conn
+
+  defp check_signature_policy(conn) do
+    with {:error, reason} <- Signature.check_signature_policy(conn) do
+      if Pleroma.Config.get([:activitypub, :signature_policy], :enforce) == :log do
+        Logger.warning(
+          "HTTP signature policy violation (#{reason}) for #{conn.method} #{conn.request_path}, allowed by config"
+        )
+
+        :ok
+      else
+        {:error, reason}
+      end
+    end
+  end
+
+  # The request is treated as badly signed. The reason is kept so the inbox
+  # can refuse it instead of queueing a failed-signature retry, which only
+  # checks the cryptographic signature.
+  defp reject_signature(conn, reason) do
+    Logger.debug("Rejecting HTTP signature: #{reason}")
+
+    conn
+    |> assign(:valid_signature, false)
+    |> assign(:signature_error, reason)
+  end
+
   defp maybe_assign_actor_id(%{assigns: %{valid_signature: true}} = conn) do
     adapter = Application.get_env(:http_signatures, :adapter)
 
@@ -101,7 +133,7 @@ defmodule Pleroma.Web.Plugs.HTTPSignaturePlug do
          conn.assigns[:actor_id] do
       %{host: host} = URI.parse(conn.assigns.actor_id)
 
-      if MRF.subdomain_match?(rejected_domains(), host) do
+      if MRF.subdomain_match?(rejected_domains(), host) or not MRF.instance_allowed?(host) do
         conn
         |> put_status(:unauthorized)
         |> halt()

lib/pleroma/web/web_finger.ex

@@ -158,7 +158,11 @@ defmodule Pleroma.Web.WebFinger do
   @cachex Pleroma.Config.get([:cachex, :provider], Cachex)
   def find_lrdd_template(domain) do
     @cachex.fetch!(:host_meta_cache, domain, fn _ ->
-      {:commit, fetch_lrdd_template(domain)}
+      case fetch_lrdd_template(domain) do
+        # retry once the backoff expires instead of caching the miss
+        {:error, :ratelimit} = error -> {:ignore, error}
+        result -> {:commit, result}
+      end
     end)
   rescue
     e -> {:error, "Cachex error: #{inspect(e)}"}
@@ -168,9 +172,13 @@ defmodule Pleroma.Web.WebFinger do
     # WebFinger is restricted to HTTPS - https://tools.ietf.org/html/rfc7033#section-9.1
     meta_url = "https://#{domain}/.well-known/host-meta"
 
-    with {:ok, %{status: status, body: body}} when status in 200..299 <- HTTP.get(meta_url) do
+    with {:ok, %{status: status, body: body}} when status in 200..299 <-
+           HTTP.Backoff.get(meta_url) do
       get_template_from_xml(body)
     else
+      {:error, :ratelimit} = error ->
+        error
+
       error ->
         Logger.warning("Can't find LRDD template in #{inspect(meta_url)}: #{inspect(error)}")
         {:error, :lrdd_not_found}
@@ -210,7 +218,7 @@ defmodule Pleroma.Web.WebFinger do
 
     with address when is_binary(address) <- get_address_from_domain(domain, account),
          {:ok, %{status: status, body: body, headers: headers}} when status in 200..299 <-
-           HTTP.get(
+           HTTP.Backoff.get(
              address,
              [{"accept", "application/xrd+xml,application/jrd+json"}]
            ) do

lib/pleroma/workers/publisher_worker.ex

@@ -6,6 +6,8 @@ defmodule Pleroma.Workers.PublisherWorker do
   alias Pleroma.Activity
   alias Pleroma.Config
   alias Pleroma.Instances
+  alias Pleroma.Web.ActivityPub.Publisher
+  alias Pleroma.Web.ActivityPub.Visibility
   alias Pleroma.Web.Federator
 
   use Oban.Worker, queue: :federator_outgoing, max_attempts: 13
@@ -32,26 +34,39 @@ defmodule Pleroma.Workers.PublisherWorker do
         end
       end)
 
-    # Cancel / skip the job if this server believed to be unreachable now
-    if not Instances.reachable?(params.inbox) do
-      {:cancel, :unreachable}
-    else
-      case federator().perform(:publish_one, params) do
-        {:ok, _} ->
-          :ok
+    cond do
+      # Cancel / skip the job if this server believed to be unreachable now
+      not Instances.reachable?(params.inbox) ->
+        {:cancel, :unreachable}
 
-        {:error, _} = error ->
-          # Only mark as unreachable on final failure
-          if job.attempt == job.max_attempts do
-            Instances.set_unreachable(params.inbox)
-          end
+      # The instance may have been blocked after the job was queued
+      not federate?(params) ->
+        {:cancel, :rejected}
 
-          error
+      true ->
+        case federator().perform(:publish_one, params) do
+          {:ok, _} ->
+            :ok
 
-        error ->
-          # Unexpected error, may have been client side
-          error
-      end
+          {:error, _} = error ->
+            # Only mark as unreachable on final failure
+            if job.attempt == job.max_attempts do
+              Instances.set_unreachable(params.inbox)
+            end
+
+            error
+
+          error ->
+            # Unexpected error, may have been client side
+            error
+        end
+    end
+  end
+
+  defp federate?(%{inbox: inbox, activity_id: activity_id}) do
+    case Activity.get_by_id(activity_id) do
+      %Activity{} = activity -> Publisher.should_federate?(inbox, Visibility.public?(activity))
+      nil -> true
     end
   end
 

lib/pleroma/workers/receiver_worker.ex

@@ -3,6 +3,7 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Workers.ReceiverWorker do
+  alias Pleroma.HTTP.Backoff
   alias Pleroma.Instances
   alias Pleroma.Web.Federator
   alias Pleroma.Workers.SignatureRetryWorker
@@ -14,7 +15,9 @@ defmodule Pleroma.Workers.ReceiverWorker do
     if signature_retry_job?(args) do
       perform_signature_retry(job)
     else
-      perform_incoming(params)
+      params
+      |> perform_incoming()
+      |> Backoff.snooze_ratelimited(job)
     end
   end
 

lib/pleroma/workers/signature_retry_worker.ex

@@ -3,6 +3,7 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Workers.SignatureRetryWorker do
+  alias Pleroma.HTTP.Backoff
   alias Pleroma.Instances
   alias Pleroma.Signature
   alias Pleroma.User
@@ -16,16 +17,18 @@ defmodule Pleroma.Workers.SignatureRetryWorker do
   use Oban.Worker, queue: :federator_incoming, max_attempts: 5, unique: [period: :infinity]
 
   @impl true
-  def perform(%Job{
-        args: %{
-          "op" => "incoming_failed_signature_ap_doc",
-          "method" => method,
-          "params" => params,
-          "req_headers" => req_headers,
-          "request_path" => request_path,
-          "query_string" => query_string
-        }
-      })
+  def perform(
+        %Job{
+          args: %{
+            "op" => "incoming_failed_signature_ap_doc",
+            "method" => method,
+            "params" => params,
+            "req_headers" => req_headers,
+            "request_path" => request_path,
+            "query_string" => query_string
+          }
+        } = job
+      )
       when is_binary(method) and is_map(params) and is_list(req_headers) and
              is_binary(request_path) and is_binary(query_string) do
     case normalize_req_headers(req_headers) do
@@ -58,7 +61,10 @@ defmodule Pleroma.Workers.SignatureRetryWorker do
 
           {:ok, res}
         else
-          e -> process_errors(e, retry_log_context(params, request_path, signature_actor_result))
+          e ->
+            e
+            |> process_errors(retry_log_context(params, request_path, signature_actor_result))
+            |> Backoff.snooze_ratelimited(job)
         end
 
       e ->