Merge branch 'develop' of git.pleroma.social:pleroma/pleroma into pleroma-feature/akkoma-prune-old-posts

lib/mix/tasks/pleroma/search/indexer.ex

@@ -0,0 +1,80 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Mix.Tasks.Pleroma.Search.Indexer do
+  import Mix.Pleroma
+  import Ecto.Query
+
+  alias Pleroma.Workers.SearchIndexingWorker
+
+  def run(["create_index"]) do
+    start_pleroma()
+
+    with :ok <- Pleroma.Config.get([Pleroma.Search, :module]).create_index() do
+      IO.puts("Index created")
+    else
+      e -> IO.puts("Could not create index: #{inspect(e)}")
+    end
+  end
+
+  def run(["drop_index"]) do
+    start_pleroma()
+
+    with :ok <- Pleroma.Config.get([Pleroma.Search, :module]).drop_index() do
+      IO.puts("Index dropped")
+    else
+      e -> IO.puts("Could not drop index: #{inspect(e)}")
+    end
+  end
+
+  def run(["index" | options]) do
+    {options, [], []} =
+      OptionParser.parse(
+        options,
+        strict: [
+          limit: :integer
+        ]
+      )
+
+    start_pleroma()
+
+    limit = Keyword.get(options, :limit, 100_000)
+
+    per_step = 1000
+    chunks = max(div(limit, per_step), 1)
+
+    1..chunks
+    |> Enum.each(fn step ->
+      q =
+        from(a in Pleroma.Activity,
+          limit: ^per_step,
+          offset: ^per_step * (^step - 1),
+          select: [:id],
+          order_by: [desc: :id]
+        )
+
+      {:ok, ids} =
+        Pleroma.Repo.transaction(fn ->
+          Pleroma.Repo.stream(q, timeout: :infinity)
+          |> Enum.map(fn a ->
+            a.id
+          end)
+        end)
+
+      IO.puts("Got #{length(ids)} activities, adding to indexer")
+
+      ids
+      |> Enum.chunk_every(100)
+      |> Enum.each(fn chunk ->
+        IO.puts("Adding #{length(chunk)} activities to indexing queue")
+
+        chunk
+        |> Enum.map(fn id ->
+          SearchIndexingWorker.new(%{"op" => "add_to_index", "activity" => id})
+        end)
+        |> Oban.insert_all()
+      end)
+    end)
+  end
+end

lib/pleroma/application.ex

@@ -14,6 +14,7 @@ defmodule Pleroma.Application do
   @name Mix.Project.config()[:name]
   @version Mix.Project.config()[:version]
   @repository Mix.Project.config()[:source_url]
+  @compile_env Mix.env()
 
   def name, do: @name
   def version, do: @version
@@ -51,7 +52,11 @@ defmodule Pleroma.Application do
     Pleroma.HTML.compile_scrubbers()
     Pleroma.Config.Oban.warn()
     Config.DeprecationWarnings.warn()
-    Pleroma.Web.Plugs.HTTPSecurityPlug.warn_if_disabled()
+
+    if @compile_env != :test do
+      Pleroma.Web.Plugs.HTTPSecurityPlug.warn_if_disabled()
+    end
+
     Pleroma.ApplicationRequirements.verify!()
     load_custom_modules()
     Pleroma.Docs.JSON.compile()
@@ -109,7 +114,8 @@ defmodule Pleroma.Application do
         streamer_registry() ++
         background_migrators() ++
         shout_child(shout_enabled?()) ++
-        [Pleroma.Gopher.Server]
+        [Pleroma.Gopher.Server] ++
+        [Pleroma.Search.Healthcheck]
 
     # See http://elixir-lang.org/docs/stable/elixir/Supervisor.html
     # for other strategies and supported options
@@ -162,7 +168,8 @@ defmodule Pleroma.Application do
         expiration: chat_message_id_idempotency_key_expiration(),
         limit: 500_000
       ),
-      build_cachex("rel_me", limit: 2500)
+      build_cachex("rel_me", limit: 2500),
+      build_cachex("host_meta", default_ttl: :timer.minutes(120), limit: 5000)
     ]
   end
 

lib/pleroma/application_requirements.ex

@@ -28,6 +28,7 @@ defmodule Pleroma.ApplicationRequirements do
     |> check_welcome_message_config!()
     |> check_rum!()
     |> check_repo_pool_size!()
+    |> check_mrfs()
     |> handle_result()
   end
 
@@ -234,4 +235,25 @@ defmodule Pleroma.ApplicationRequirements do
       true
     end
   end
+
+  defp check_mrfs(:ok) do
+    mrfs = Config.get!([:mrf, :policies])
+
+    missing_mrfs =
+      Enum.reduce(mrfs, [], fn x, acc ->
+        if Code.ensure_compiled(x) do
+          acc
+        else
+          acc ++ [x]
+        end
+      end)
+
+    if Enum.empty?(missing_mrfs) do
+      :ok
+    else
+      {:error, "The following MRF modules are configured but missing: #{inspect(missing_mrfs)}"}
+    end
+  end
+
+  defp check_mrfs(result), do: result
 end

lib/pleroma/constants.ex

@@ -19,7 +19,8 @@ defmodule Pleroma.Constants do
       "context_id",
       "deleted_activity_id",
       "pleroma_internal",
-      "generator"
+      "generator",
+      "rules"
     ]
   )
 

lib/pleroma/helpers/inet_helper.ex

@@ -16,4 +16,15 @@ defmodule Pleroma.Helpers.InetHelper do
   def parse_address(ip) do
     :inet.parse_address(ip)
   end
+
+  def parse_cidr(proxy) when is_binary(proxy) do
+    proxy =
+      cond do
+        "/" in String.codepoints(proxy) -> proxy
+        InetCidr.v4?(InetCidr.parse_address!(proxy)) -> proxy <> "/32"
+        InetCidr.v6?(InetCidr.parse_address!(proxy)) -> proxy <> "/128"
+      end
+
+    InetCidr.parse_cidr!(proxy, true)
+  end
 end

lib/pleroma/http_signatures_api.ex

@@ -0,0 +1,4 @@
+defmodule Pleroma.HTTPSignaturesAPI do
+  @callback validate_conn(conn :: Plug.Conn.t()) :: boolean
+  @callback signature_for_conn(conn :: Plug.Conn.t()) :: map
+end

lib/pleroma/notification.ex

@@ -73,6 +73,7 @@ defmodule Pleroma.Notification do
     pleroma:report
     reblog
     poll
+    status
   }
 
   def changeset(%Notification{} = notification, attrs) do
@@ -280,15 +281,10 @@ defmodule Pleroma.Notification do
         select: n.id
       )
 
-    {:ok, %{ids: {_, notification_ids}}} =
-      Multi.new()
-      |> Multi.update_all(:ids, query, set: [seen: true, updated_at: NaiveDateTime.utc_now()])
-      |> Marker.multi_set_last_read_id(user, "notifications")
-      |> Repo.transaction()
-
-    for_user_query(user)
-    |> where([n], n.id in ^notification_ids)
-    |> Repo.all()
+    Multi.new()
+    |> Multi.update_all(:ids, query, set: [seen: true, updated_at: NaiveDateTime.utc_now()])
+    |> Marker.multi_set_last_read_id(user, "notifications")
+    |> Repo.transaction()
   end
 
   @spec read_one(User.t(), String.t()) ::
@@ -299,10 +295,6 @@ defmodule Pleroma.Notification do
       |> Multi.update(:update, changeset(notification, %{seen: true}))
       |> Marker.multi_set_last_read_id(user, "notifications")
       |> Repo.transaction()
-      |> case do
-        {:ok, %{update: notification}} -> {:ok, notification}
-        {:error, :update, changeset, _} -> {:error, changeset}
-      end
     end
   end
 
@@ -361,37 +353,38 @@ defmodule Pleroma.Notification do
     end
   end
 
-  @spec create_notifications(Activity.t(), keyword()) :: {:ok, [Notification.t()] | []}
-  def create_notifications(activity, options \\ [])
+  @spec create_notifications(Activity.t()) :: {:ok, [Notification.t()] | []}
+  def create_notifications(activity)
 
-  def create_notifications(%Activity{data: %{"to" => _, "type" => "Create"}} = activity, options) do
+  def create_notifications(%Activity{data: %{"to" => _, "type" => "Create"}} = activity) do
     object = Object.normalize(activity, fetch: false)
 
     if object && object.data["type"] == "Answer" do
       {:ok, []}
     else
-      do_create_notifications(activity, options)
+      do_create_notifications(activity)
     end
   end
 
-  def create_notifications(%Activity{data: %{"type" => type}} = activity, options)
+  def create_notifications(%Activity{data: %{"type" => type}} = activity)
       when type in ["Follow", "Like", "Announce", "Move", "EmojiReact", "Flag", "Update"] do
-    do_create_notifications(activity, options)
+    do_create_notifications(activity)
   end
 
-  def create_notifications(_, _), do: {:ok, []}
+  def create_notifications(_), do: {:ok, []}
 
-  defp do_create_notifications(%Activity{} = activity, options) do
-    do_send = Keyword.get(options, :do_send, true)
+  defp do_create_notifications(%Activity{} = activity) do
+    enabled_receivers = get_notified_from_activity(activity)
 
-    {enabled_receivers, disabled_receivers} = get_notified_from_activity(activity)
-    potential_receivers = enabled_receivers ++ disabled_receivers
+    enabled_subscribers = get_notified_subscribers_from_activity(activity)
 
     notifications =
-      Enum.map(potential_receivers, fn user ->
-        do_send = do_send && user in enabled_receivers
-        create_notification(activity, user, do_send: do_send)
-      end)
+      (Enum.map(enabled_receivers, fn user ->
+         create_notification(activity, user)
+       end) ++
+         Enum.map(enabled_subscribers -- enabled_receivers, fn user ->
+           create_notification(activity, user, type: "status")
+         end))
       |> Enum.reject(&is_nil/1)
 
     {:ok, notifications}
@@ -450,7 +443,6 @@ defmodule Pleroma.Notification do
 
   # TODO move to sql, too.
   def create_notification(%Activity{} = activity, %User{} = user, opts \\ []) do
-    do_send = Keyword.get(opts, :do_send, true)
     type = Keyword.get(opts, :type, type_from_activity(activity))
 
     unless skip?(activity, user, opts) do
@@ -465,11 +457,6 @@ defmodule Pleroma.Notification do
         |> Marker.multi_set_last_read_id(user, "notifications")
         |> Repo.transaction()
 
-      if do_send do
-        Streamer.stream(["user", "user:notification"], notification)
-        Push.send(notification)
-      end
-
       notification
     end
   end
@@ -527,13 +514,28 @@ defmodule Pleroma.Notification do
       |> exclude_relationship_restricted_ap_ids(activity)
       |> exclude_thread_muter_ap_ids(activity)
 
-    notification_enabled_users =
-      Enum.filter(potential_receivers, fn u -> u.ap_id in notification_enabled_ap_ids end)
-
-    {notification_enabled_users, potential_receivers -- notification_enabled_users}
+    Enum.filter(potential_receivers, fn u -> u.ap_id in notification_enabled_ap_ids end)
   end
 
-  def get_notified_from_activity(_, _local_only), do: {[], []}
+  def get_notified_from_activity(_, _local_only), do: []
+
+  def get_notified_subscribers_from_activity(activity, local_only \\ true)
+
+  def get_notified_subscribers_from_activity(
+        %Activity{data: %{"type" => "Create"}} = activity,
+        local_only
+      ) do
+    notification_enabled_ap_ids =
+      []
+      |> Utils.maybe_notify_subscribers(activity)
+
+    potential_receivers =
+      User.get_users_from_set(notification_enabled_ap_ids, local_only: local_only)
+
+    Enum.filter(potential_receivers, fn u -> u.ap_id in notification_enabled_ap_ids end)
+  end
+
+  def get_notified_subscribers_from_activity(_, _), do: []
 
   # For some activities, only notify the author of the object
   def get_potential_receiver_ap_ids(%{data: %{"type" => type, "object" => object_id}})
@@ -576,7 +578,6 @@ defmodule Pleroma.Notification do
     []
     |> Utils.maybe_notify_to_recipients(activity)
     |> Utils.maybe_notify_mentioned_recipients(activity)
-    |> Utils.maybe_notify_subscribers(activity)
     |> Utils.maybe_notify_followers(activity)
     |> Enum.uniq()
   end
@@ -643,6 +644,7 @@ defmodule Pleroma.Notification do
   def skip?(%Activity{} = activity, %User{} = user, opts) do
     [
       :self,
+      :internal,
       :invisible,
       :block_from_strangers,
       :recently_followed,
@@ -662,6 +664,12 @@ defmodule Pleroma.Notification do
     end
   end
 
+  def skip?(:internal, %Activity{} = activity, _user, _opts) do
+    actor = activity.data["actor"]
+    user = User.get_cached_by_ap_id(actor)
+    User.internal?(user)
+  end
+
   def skip?(:invisible, %Activity{} = activity, _user, _opts) do
     actor = activity.data["actor"]
     user = User.get_cached_by_ap_id(actor)
@@ -748,4 +756,12 @@ defmodule Pleroma.Notification do
     )
     |> Repo.update_all(set: [seen: true])
   end
+
+  @spec send(list(Notification.t())) :: :ok
+  def send(notifications) do
+    Enum.each(notifications, fn notification ->
+      Streamer.stream(["user", "user:notification"], notification)
+      Push.send(notification)
+    end)
+  end
 end

lib/pleroma/rule.ex

@@ -0,0 +1,68 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Rule do
+  use Ecto.Schema
+
+  import Ecto.Changeset
+  import Ecto.Query
+
+  alias Pleroma.Repo
+  alias Pleroma.Rule
+
+  schema "rules" do
+    field(:priority, :integer, default: 0)
+    field(:text, :string)
+    field(:hint, :string)
+
+    timestamps()
+  end
+
+  def changeset(%Rule{} = rule, params \\ %{}) do
+    rule
+    |> cast(params, [:priority, :text, :hint])
+    |> validate_required([:text])
+  end
+
+  def query do
+    Rule
+    |> order_by(asc: :priority)
+    |> order_by(asc: :id)
+  end
+
+  def get(ids) when is_list(ids) do
+    from(r in __MODULE__, where: r.id in ^ids)
+    |> Repo.all()
+  end
+
+  def get(id), do: Repo.get(__MODULE__, id)
+
+  def exists?(id) do
+    from(r in __MODULE__, where: r.id == ^id)
+    |> Repo.exists?()
+  end
+
+  def create(params) do
+    {:ok, rule} =
+      %Rule{}
+      |> changeset(params)
+      |> Repo.insert()
+
+    rule
+  end
+
+  def update(params, id) do
+    {:ok, rule} =
+      get(id)
+      |> changeset(params)
+      |> Repo.update()
+
+    rule
+  end
+
+  def delete(id) do
+    get(id)
+    |> Repo.delete()
+  end
+end

lib/pleroma/scheduled_activity.ex

@@ -204,7 +204,7 @@ defmodule Pleroma.ScheduledActivity do
 
   def job_query(scheduled_activity_id) do
     from(j in Oban.Job,
-      where: j.queue == "scheduled_activities",
+      where: j.queue == "federator_outgoing",
       where: fragment("args ->> 'activity_id' = ?::text", ^to_string(scheduled_activity_id))
     )
   end

lib/pleroma/search.ex

@@ -10,8 +10,12 @@ defmodule Pleroma.Search do
   end
 
   def search(query, options) do
-    search_module = Pleroma.Config.get([Pleroma.Search, :module], Pleroma.Activity)
-
+    search_module = Pleroma.Config.get([Pleroma.Search, :module])
     search_module.search(options[:for_user], query, options)
   end
+
+  def healthcheck_endpoints do
+    search_module = Pleroma.Config.get([Pleroma.Search, :module])
+    search_module.healthcheck_endpoints
+  end
 end

lib/pleroma/search/database_search.ex

@@ -48,6 +48,15 @@ defmodule Pleroma.Search.DatabaseSearch do
   @impl true
   def remove_from_index(_object), do: :ok
 
+  @impl true
+  def create_index, do: :ok
+
+  @impl true
+  def drop_index, do: :ok
+
+  @impl true
+  def healthcheck_endpoints, do: nil
+
   def maybe_restrict_author(query, %User{} = author) do
     Activity.Queries.by_author(query, author)
   end

lib/pleroma/search/healthcheck.ex

@@ -0,0 +1,86 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2024 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+defmodule Pleroma.Search.Healthcheck do
+  @doc """
+  Monitors health of search backend to control processing of events based on health and availability.
+  """
+  use GenServer
+  require Logger
+
+  @queue :search_indexing
+  @tick :timer.seconds(5)
+  @timeout :timer.seconds(2)
+
+  def start_link(_) do
+    GenServer.start_link(__MODULE__, [], name: __MODULE__)
+  end
+
+  @impl true
+  def init(_) do
+    state = %{healthy: false}
+    {:ok, state, {:continue, :start}}
+  end
+
+  @impl true
+  def handle_continue(:start, state) do
+    tick()
+    {:noreply, state}
+  end
+
+  @impl true
+  def handle_info(:check, state) do
+    urls = Pleroma.Search.healthcheck_endpoints()
+
+    new_state =
+      if check(urls) do
+        Oban.resume_queue(queue: @queue)
+        Map.put(state, :healthy, true)
+      else
+        Oban.pause_queue(queue: @queue)
+        Map.put(state, :healthy, false)
+      end
+
+    maybe_log_state_change(state, new_state)
+
+    tick()
+    {:noreply, new_state}
+  end
+
+  @impl true
+  def handle_call(:state, _from, state) do
+    {:reply, state, state, :hibernate}
+  end
+
+  def state, do: GenServer.call(__MODULE__, :state)
+
+  def check([]), do: true
+
+  def check(urls) when is_list(urls) do
+    Enum.all?(
+      urls,
+      fn url ->
+        case Pleroma.HTTP.get(url, [], recv_timeout: @timeout) do
+          {:ok, %{status: 200}} -> true
+          _ -> false
+        end
+      end
+    )
+  end
+
+  def check(_), do: true
+
+  defp tick do
+    Process.send_after(self(), :check, @tick)
+  end
+
+  defp maybe_log_state_change(%{healthy: true}, %{healthy: false}) do
+    Logger.error("Pausing Oban queue #{@queue} due to search backend healthcheck failure")
+  end
+
+  defp maybe_log_state_change(%{healthy: false}, %{healthy: true}) do
+    Logger.info("Resuming Oban queue #{@queue} due to search backend healthcheck pass")
+  end
+
+  defp maybe_log_state_change(_, _), do: :ok
+end

lib/pleroma/search/meilisearch.ex

@@ -10,6 +10,12 @@ defmodule Pleroma.Search.Meilisearch do
 
   @behaviour Pleroma.Search.SearchBackend
 
+  @impl true
+  def create_index, do: :ok
+
+  @impl true
+  def drop_index, do: :ok
+
   defp meili_headers do
     private_key = Config.get([Pleroma.Search.Meilisearch, :private_key])
 
@@ -178,4 +184,15 @@ defmodule Pleroma.Search.Meilisearch do
   def remove_from_index(object) do
     meili_delete("/indexes/objects/documents/#{object.id}")
   end
+
+  @impl true
+  def healthcheck_endpoints do
+    endpoint =
+      Config.get([Pleroma.Search.Meilisearch, :url])
+      |> URI.parse()
+      |> Map.put(:path, "/health")
+      |> URI.to_string()
+
+    [endpoint]
+  end
 end

lib/pleroma/search/qdrant_search.ex

@@ -0,0 +1,182 @@
+defmodule Pleroma.Search.QdrantSearch do
+  @behaviour Pleroma.Search.SearchBackend
+  import Ecto.Query
+
+  alias Pleroma.Activity
+  alias Pleroma.Config.Getting, as: Config
+
+  alias __MODULE__.OpenAIClient
+  alias __MODULE__.QdrantClient
+
+  import Pleroma.Search.Meilisearch, only: [object_to_search_data: 1]
+  import Pleroma.Search.DatabaseSearch, only: [maybe_fetch: 3]
+
+  @impl true
+  def create_index do
+    payload = Config.get([Pleroma.Search.QdrantSearch, :qdrant_index_configuration])
+
+    with {:ok, %{status: 200}} <- QdrantClient.put("/collections/posts", payload) do
+      :ok
+    else
+      e -> {:error, e}
+    end
+  end
+
+  @impl true
+  def drop_index do
+    with {:ok, %{status: 200}} <- QdrantClient.delete("/collections/posts") do
+      :ok
+    else
+      e -> {:error, e}
+    end
+  end
+
+  def get_embedding(text) do
+    with {:ok, %{body: %{"data" => [%{"embedding" => embedding}]}}} <-
+           OpenAIClient.post("/v1/embeddings", %{
+             input: text,
+             model: Config.get([Pleroma.Search.QdrantSearch, :openai_model])
+           }) do
+      {:ok, embedding}
+    else
+      _ ->
+        {:error, "Failed to get embedding"}
+    end
+  end
+
+  defp actor_from_activity(%{data: %{"actor" => actor}}) do
+    actor
+  end
+
+  defp actor_from_activity(_), do: nil
+
+  defp build_index_payload(activity, embedding) do
+    actor = actor_from_activity(activity)
+    published_at = activity.data["published"]
+
+    %{
+      points: [
+        %{
+          id: activity.id |> FlakeId.from_string() |> Ecto.UUID.cast!(),
+          vector: embedding,
+          payload: %{actor: actor, published_at: published_at}
+        }
+      ]
+    }
+  end
+
+  defp build_search_payload(embedding, options) do
+    base = %{
+      vector: embedding,
+      limit: options[:limit] || 20,
+      offset: options[:offset] || 0
+    }
+
+    if author = options[:author] do
+      Map.put(base, :filter, %{
+        must: [%{key: "actor", match: %{value: author.ap_id}}]
+      })
+    else
+      base
+    end
+  end
+
+  @impl true
+  def add_to_index(activity) do
+    # This will only index public or unlisted notes
+    maybe_search_data = object_to_search_data(activity.object)
+
+    if activity.data["type"] == "Create" and maybe_search_data do
+      with {:ok, embedding} <- get_embedding(maybe_search_data.content),
+           {:ok, %{status: 200}} <-
+             QdrantClient.put(
+               "/collections/posts/points",
+               build_index_payload(activity, embedding)
+             ) do
+        :ok
+      else
+        e -> {:error, e}
+      end
+    else
+      :ok
+    end
+  end
+
+  @impl true
+  def remove_from_index(object) do
+    activity = Activity.get_by_object_ap_id_with_object(object.data["id"])
+    id = activity.id |> FlakeId.from_string() |> Ecto.UUID.cast!()
+
+    with {:ok, %{status: 200}} <-
+           QdrantClient.post("/collections/posts/points/delete", %{"points" => [id]}) do
+      :ok
+    else
+      e -> {:error, e}
+    end
+  end
+
+  @impl true
+  def search(user, original_query, options) do
+    query = "Represent this sentence for searching relevant passages: #{original_query}"
+
+    with {:ok, embedding} <- get_embedding(query),
+         {:ok, %{body: %{"result" => result}}} <-
+           QdrantClient.post(
+             "/collections/posts/points/search",
+             build_search_payload(embedding, options)
+           ) do
+      ids =
+        Enum.map(result, fn %{"id" => id} ->
+          Ecto.UUID.dump!(id)
+        end)
+
+      from(a in Activity, where: a.id in ^ids)
+      |> Activity.with_preloaded_object()
+      |> Activity.restrict_deactivated_users()
+      |> Ecto.Query.order_by([a], fragment("array_position(?, ?)", ^ids, a.id))
+      |> Pleroma.Repo.all()
+      |> maybe_fetch(user, original_query)
+    else
+      _ ->
+        []
+    end
+  end
+
+  @impl true
+  def healthcheck_endpoints do
+    qdrant_health =
+      Config.get([Pleroma.Search.QdrantSearch, :qdrant_url])
+      |> URI.parse()
+      |> Map.put(:path, "/healthz")
+      |> URI.to_string()
+
+    openai_health = Config.get([Pleroma.Search.QdrantSearch, :openai_healthcheck_url])
+
+    [qdrant_health, openai_health] |> Enum.filter(& &1)
+  end
+end
+
+defmodule Pleroma.Search.QdrantSearch.OpenAIClient do
+  use Tesla
+  alias Pleroma.Config.Getting, as: Config
+
+  plug(Tesla.Middleware.BaseUrl, Config.get([Pleroma.Search.QdrantSearch, :openai_url]))
+  plug(Tesla.Middleware.JSON)
+
+  plug(Tesla.Middleware.Headers, [
+    {"Authorization",
+     "Bearer #{Pleroma.Config.get([Pleroma.Search.QdrantSearch, :openai_api_key])}"}
+  ])
+end
+
+defmodule Pleroma.Search.QdrantSearch.QdrantClient do
+  use Tesla
+  alias Pleroma.Config.Getting, as: Config
+
+  plug(Tesla.Middleware.BaseUrl, Config.get([Pleroma.Search.QdrantSearch, :qdrant_url]))
+  plug(Tesla.Middleware.JSON)
+
+  plug(Tesla.Middleware.Headers, [
+    {"api-key", Pleroma.Config.get([Pleroma.Search.QdrantSearch, :qdrant_api_key])}
+  ])
+end

lib/pleroma/search/search_backend.ex

@@ -21,4 +21,22 @@ defmodule Pleroma.Search.SearchBackend do
   from index.
   """
   @callback remove_from_index(object :: Pleroma.Object.t()) :: :ok | {:error, any()}
+
+  @doc """
+  Create the index
+  """
+  @callback create_index() :: :ok | {:error, any()}
+
+  @doc """
+  Drop the index
+  """
+  @callback drop_index() :: :ok | {:error, any()}
+
+  @doc """
+  Healthcheck endpoints of search backend infrastructure to monitor for controlling
+  processing of jobs in the Oban queue.
+
+  It is expected a 200 response is healthy and other responses are unhealthy.
+  """
+  @callback healthcheck_endpoints :: list() | nil
 end

lib/pleroma/signature.ex

@@ -44,8 +44,7 @@ defmodule Pleroma.Signature do
   defp remove_suffix(uri, []), do: uri
 
   def fetch_public_key(conn) do
-    with %{"keyId" => kid} <- HTTPSignatures.signature_for_conn(conn),
-         {:ok, actor_id} <- key_id_to_actor_id(kid),
+    with {:ok, actor_id} <- get_actor_id(conn),
          {:ok, public_key} <- User.get_public_key_for_ap_id(actor_id) do
       {:ok, public_key}
     else
@@ -55,8 +54,7 @@ defmodule Pleroma.Signature do
   end
 
   def refetch_public_key(conn) do
-    with %{"keyId" => kid} <- HTTPSignatures.signature_for_conn(conn),
-         {:ok, actor_id} <- key_id_to_actor_id(kid),
+    with {:ok, actor_id} <- get_actor_id(conn),
          {:ok, _user} <- ActivityPub.make_user_from_ap_id(actor_id),
          {:ok, public_key} <- User.get_public_key_for_ap_id(actor_id) do
       {:ok, public_key}
@@ -66,6 +64,16 @@ defmodule Pleroma.Signature do
     end
   end
 
+  def get_actor_id(conn) do
+    with %{"keyId" => kid} <- HTTPSignatures.signature_for_conn(conn),
+         {:ok, actor_id} <- key_id_to_actor_id(kid) do
+      {:ok, actor_id}
+    else
+      e ->
+        {:error, e}
+    end
+  end
+
   def sign(%User{keys: keys} = user, headers) do
     with {:ok, private_key, _} <- Keys.keys_from_pem(keys) do
       HTTPSignatures.sign(private_key, user.ap_id <> "#main-key", headers)

lib/pleroma/upload.ex

@@ -239,8 +239,12 @@ defmodule Pleroma.Upload do
           ""
         end
 
-    [base_url, path]
-    |> Path.join()
+    if String.contains?(base_url, Pleroma.Uploaders.IPFS.placeholder()) do
+      String.replace(base_url, Pleroma.Uploaders.IPFS.placeholder(), path)
+    else
+      [base_url, path]
+      |> Path.join()
+    end
   end
 
   defp url_from_spec(_upload, _base_url, {:url, url}), do: url
@@ -277,6 +281,9 @@ defmodule Pleroma.Upload do
           Path.join([upload_base_url, bucket_with_namespace])
         end
 
+      Pleroma.Uploaders.IPFS ->
+        @config_impl.get([Pleroma.Uploaders.IPFS, :get_gateway_url])
+
       _ ->
         public_endpoint || upload_base_url || Pleroma.Web.Endpoint.url() <> "/media/"
     end

lib/pleroma/uploaders/ipfs.ex

@@ -0,0 +1,77 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Uploaders.IPFS do
+  @behaviour Pleroma.Uploaders.Uploader
+  require Logger
+
+  alias Tesla.Multipart
+
+  @config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config)
+
+  defp get_final_url(method) do
+    config = @config_impl.get([__MODULE__])
+    post_base_url = Keyword.get(config, :post_gateway_url)
+
+    Path.join([post_base_url, method])
+  end
+
+  def put_file_endpoint do
+    get_final_url("/api/v0/add")
+  end
+
+  def delete_file_endpoint do
+    get_final_url("/api/v0/files/rm")
+  end
+
+  @placeholder "{CID}"
+  def placeholder, do: @placeholder
+
+  @impl true
+  def get_file(file) do
+    b_url = Pleroma.Upload.base_url()
+
+    if String.contains?(b_url, @placeholder) do
+      {:ok, {:url, String.replace(b_url, @placeholder, URI.decode(file))}}
+    else
+      {:error, "IPFS Get URL doesn't contain 'cid' placeholder"}
+    end
+  end
+
+  @impl true
+  def put_file(%Pleroma.Upload{} = upload) do
+    mp =
+      Multipart.new()
+      |> Multipart.add_content_type_param("charset=utf-8")
+      |> Multipart.add_file(upload.tempfile)
+
+    case Pleroma.HTTP.post(put_file_endpoint(), mp, [], params: ["cid-version": "1"]) do
+      {:ok, ret} ->
+        case Jason.decode(ret.body) do
+          {:ok, ret} ->
+            if Map.has_key?(ret, "Hash") do
+              {:ok, {:file, ret["Hash"]}}
+            else
+              {:error, "JSON doesn't contain Hash key"}
+            end
+
+          error ->
+            Logger.error("#{__MODULE__}: #{inspect(error)}")
+            {:error, "JSON decode failed"}
+        end
+
+      error ->
+        Logger.error("#{__MODULE__}: #{inspect(error)}")
+        {:error, "IPFS Gateway upload failed"}
+    end
+  end
+
+  @impl true
+  def delete_file(file) do
+    case Pleroma.HTTP.post(delete_file_endpoint(), "", [], params: [arg: file]) do
+      {:ok, %{status: 204}} -> :ok
+      error -> {:error, inspect(error)}
+    end
+  end
+end

lib/pleroma/user.ex

@@ -1404,6 +1404,40 @@ defmodule Pleroma.User do
     |> Repo.all()
   end
 
+  @spec get_familiar_followers_query(User.t(), User.t(), pos_integer() | nil) :: Ecto.Query.t()
+  def get_familiar_followers_query(%User{} = user, %User{} = current_user, nil) do
+    friends =
+      get_friends_query(current_user)
+      |> where([u], not u.hide_follows)
+      |> select([u], u.id)
+
+    User.Query.build(%{is_active: true})
+    |> where([u], u.id not in ^[user.id, current_user.id])
+    |> join(:inner, [u], r in FollowingRelationship,
+      as: :followers_relationships,
+      on: r.following_id == ^user.id and r.follower_id == u.id
+    )
+    |> where([followers_relationships: r], r.state == ^:follow_accept)
+    |> where([followers_relationships: r], r.follower_id in subquery(friends))
+  end
+
+  def get_familiar_followers_query(%User{} = user, %User{} = current_user, page) do
+    user
+    |> get_familiar_followers_query(current_user, nil)
+    |> User.Query.paginate(page, 20)
+  end
+
+  @spec get_familiar_followers_query(User.t(), User.t()) :: Ecto.Query.t()
+  def get_familiar_followers_query(%User{} = user, %User{} = current_user),
+    do: get_familiar_followers_query(user, current_user, nil)
+
+  @spec get_familiar_followers(User.t(), User.t(), pos_integer() | nil) :: {:ok, list(User.t())}
+  def get_familiar_followers(%User{} = user, %User{} = current_user, page \\ nil) do
+    user
+    |> get_familiar_followers_query(current_user, page)
+    |> Repo.all()
+  end
+
   def increase_note_count(%User{} = user) do
     User
     |> where(id: ^user.id)

lib/pleroma/web/activity_pub/activity_pub.ex

@@ -200,7 +200,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   end
 
   def notify_and_stream(activity) do
-    Notification.create_notifications(activity)
+    {:ok, notifications} = Notification.create_notifications(activity)
+    Notification.send(notifications)
 
     original_activity =
       case activity do
@@ -978,8 +979,9 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   defp restrict_replies(query, %{exclude_replies: true}) do
     from(
-      [_activity, object] in query,
-      where: fragment("?->>'inReplyTo' is null", object.data)
+      [activity, object] in query,
+      where:
+        fragment("?->>'inReplyTo' is null or ?->>'type' = 'Announce'", object.data, activity.data)
     )
   end
 
@@ -1259,6 +1261,15 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   defp restrict_quote_url(query, _), do: query
 
+  defp restrict_rule(query, %{rule_id: rule_id}) do
+    from(
+      activity in query,
+      where: fragment("(?)->'rules' \\? (?)", activity.data, ^rule_id)
+    )
+  end
+
+  defp restrict_rule(query, _), do: query
+
   defp exclude_poll_votes(query, %{include_poll_votes: true}), do: query
 
   defp exclude_poll_votes(query, _) do
@@ -1421,6 +1432,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
       |> restrict_instance(opts)
       |> restrict_announce_object_actor(opts)
       |> restrict_filtered(opts)
+      |> restrict_rule(opts)
       |> restrict_quote_url(opts)
       |> maybe_restrict_deactivated_users(opts)
       |> exclude_poll_votes(opts)

lib/pleroma/web/activity_pub/activity_pub_controller.ex

@@ -52,6 +52,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
     when action in [:activity, :object]
   )
 
+  plug(:log_inbox_metadata when action in [:inbox])
   plug(:set_requester_reachable when action in [:inbox])
   plug(:relay_active? when action in [:relay])
 
@@ -521,6 +522,13 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
     conn
   end
 
+  defp log_inbox_metadata(%{params: %{"actor" => actor, "type" => type}} = conn, _) do
+    Logger.metadata(actor: actor, type: type)
+    conn
+  end
+
+  defp log_inbox_metadata(conn, _), do: conn
+
   def upload_media(%{assigns: %{user: %User{} = user}} = conn, %{"file" => file} = data) do
     with {:ok, object} <-
            ActivityPub.upload(

lib/pleroma/web/activity_pub/mrf/anti_mention_spam_policy.ex

@@ -0,0 +1,87 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.MRF.AntiMentionSpamPolicy do
+  alias Pleroma.Config
+  alias Pleroma.User
+  require Pleroma.Constants
+
+  @behaviour Pleroma.Web.ActivityPub.MRF.Policy
+
+  defp user_has_posted?(%User{} = u), do: u.note_count > 0
+
+  defp user_has_age?(%User{} = u) do
+    user_age_limit = Config.get([:mrf_antimentionspam, :user_age_limit], 30_000)
+    diff = NaiveDateTime.utc_now() |> NaiveDateTime.diff(u.inserted_at, :millisecond)
+    diff >= user_age_limit
+  end
+
+  defp good_reputation?(%User{} = u) do
+    user_has_age?(u) and user_has_posted?(u)
+  end
+
+  # copied from HellthreadPolicy
+  defp get_recipient_count(message) do
+    recipients = (message["to"] || []) ++ (message["cc"] || [])
+
+    follower_collection =
+      User.get_cached_by_ap_id(message["actor"] || message["attributedTo"]).follower_address
+
+    if Enum.member?(recipients, Pleroma.Constants.as_public()) do
+      recipients =
+        recipients
+        |> List.delete(Pleroma.Constants.as_public())
+        |> List.delete(follower_collection)
+
+      {:public, length(recipients)}
+    else
+      recipients =
+        recipients
+        |> List.delete(follower_collection)
+
+      {:not_public, length(recipients)}
+    end
+  end
+
+  defp object_has_recipients?(%{"object" => object} = activity) do
+    {_, object_count} = get_recipient_count(object)
+    {_, activity_count} = get_recipient_count(activity)
+    object_count + activity_count > 0
+  end
+
+  defp object_has_recipients?(object) do
+    {_, count} = get_recipient_count(object)
+    count > 0
+  end
+
+  @impl true
+  def filter(%{"type" => "Create", "actor" => actor} = activity) do
+    with {:ok, %User{local: false} = u} <- User.get_or_fetch_by_ap_id(actor),
+         {:has_mentions, true} <- {:has_mentions, object_has_recipients?(activity)},
+         {:good_reputation, true} <- {:good_reputation, good_reputation?(u)} do
+      {:ok, activity}
+    else
+      {:ok, %User{local: true}} ->
+        {:ok, activity}
+
+      {:has_mentions, false} ->
+        {:ok, activity}
+
+      {:good_reputation, false} ->
+        {:reject, "[AntiMentionSpamPolicy] User rejected"}
+
+      {:error, _} ->
+        {:reject, "[AntiMentionSpamPolicy] Failed to get or fetch user by ap_id"}
+
+      e ->
+        {:reject, "[AntiMentionSpamPolicy] Unhandled error #{inspect(e)}"}
+    end
+  end
+
+  # in all other cases, pass through
+  def filter(message), do: {:ok, message}
+
+  @impl true
+  def describe, do: {:ok, %{}}
+end

lib/pleroma/web/activity_pub/mrf/dnsrbl_policy.ex

@@ -0,0 +1,142 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2024 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.MRF.DNSRBLPolicy do
+  @moduledoc """
+  Dynamic activity filtering based on an RBL database
+
+  This MRF makes queries to a custom DNS server which will
+  respond with values indicating the classification of the domain
+  the activity originated from. This method has been widely used
+  in the email anti-spam industry for very fast reputation checks.
+
+  e.g., if the DNS response is 127.0.0.1 or empty, the domain is OK
+  Other values such as 127.0.0.2 may be used for specific classifications.
+
+  Information for why the host is blocked can be stored in a corresponding TXT record.
+
+  This method is fail-open so if the queries fail the activites are accepted.
+
+  An example of software meant for this purpsoe is rbldnsd which can be found
+  at http://www.corpit.ru/mjt/rbldnsd.html or mirrored at
+  https://git.pleroma.social/feld/rbldnsd
+
+  It is highly recommended that you run your own copy of rbldnsd and use an
+  external mechanism to sync/share the contents of the zone file. This is
+  important to keep the latency on the queries as low as possible and prevent
+  your DNS server from being attacked so it fails and content is permitted.
+  """
+
+  @behaviour Pleroma.Web.ActivityPub.MRF.Policy
+
+  alias Pleroma.Config
+
+  require Logger
+
+  @query_retries 1
+  @query_timeout 500
+
+  @impl true
+  def filter(%{"actor" => actor} = object) do
+    actor_info = URI.parse(actor)
+
+    with {:ok, object} <- check_rbl(actor_info, object) do
+      {:ok, object}
+    else
+      _ -> {:reject, "[DNSRBLPolicy]"}
+    end
+  end
+
+  @impl true
+  def filter(object), do: {:ok, object}
+
+  @impl true
+  def describe do
+    mrf_dnsrbl =
+      Config.get(:mrf_dnsrbl)
+      |> Enum.into(%{})
+
+    {:ok, %{mrf_dnsrbl: mrf_dnsrbl}}
+  end
+
+  @impl true
+  def config_description do
+    %{
+      key: :mrf_dnsrbl,
+      related_policy: "Pleroma.Web.ActivityPub.MRF.DNSRBLPolicy",
+      label: "MRF DNSRBL",
+      description: "DNS RealTime Blackhole Policy",
+      children: [
+        %{
+          key: :nameserver,
+          type: {:string},
+          description: "DNSRBL Nameserver to Query (IP or hostame)",
+          suggestions: ["127.0.0.1"]
+        },
+        %{
+          key: :port,
+          type: {:string},
+          description: "Nameserver port",
+          suggestions: ["53"]
+        },
+        %{
+          key: :zone,
+          type: {:string},
+          description: "Root zone for querying",
+          suggestions: ["bl.pleroma.com"]
+        }
+      ]
+    }
+  end
+
+  defp check_rbl(%{host: actor_host}, object) do
+    with false <- match?(^actor_host, Pleroma.Web.Endpoint.host()),
+         zone when not is_nil(zone) <- Keyword.get(Config.get([:mrf_dnsrbl]), :zone) do
+      query =
+        Enum.join([actor_host, zone], ".")
+        |> String.to_charlist()
+
+      rbl_response = rblquery(query)
+
+      if Enum.empty?(rbl_response) do
+        {:ok, object}
+      else
+        Task.start(fn ->
+          reason = rblquery(query, :txt) || "undefined"
+
+          Logger.warning(
+            "DNSRBL Rejected activity from #{actor_host} for reason: #{inspect(reason)}"
+          )
+        end)
+
+        :error
+      end
+    else
+      _ -> {:ok, object}
+    end
+  end
+
+  defp get_rblhost_ip(rblhost) do
+    case rblhost |> String.to_charlist() |> :inet_parse.address() do
+      {:ok, _} -> rblhost |> String.to_charlist() |> :inet_parse.address()
+      _ -> {:ok, rblhost |> String.to_charlist() |> :inet_res.lookup(:in, :a) |> Enum.random()}
+    end
+  end
+
+  defp rblquery(query, type \\ :a) do
+    config = Config.get([:mrf_dnsrbl])
+
+    case get_rblhost_ip(config[:nameserver]) do
+      {:ok, rblnsip} ->
+        :inet_res.lookup(query, :in, type,
+          nameservers: [{rblnsip, config[:port]}],
+          timeout: @query_timeout,
+          retry: @query_retries
+        )
+
+      _ ->
+        []
+    end
+  end
+end

lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex

@@ -11,11 +11,6 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do
 
   require Logger
 
-  @adapter_options [
-    pool: :media,
-    recv_timeout: 10_000
-  ]
-
   @impl true
   def history_awareness, do: :auto
 
@@ -27,17 +22,14 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do
 
       Logger.debug("Prefetching #{inspect(url)} as #{inspect(prefetch_url)}")
 
-      if Pleroma.Config.get(:env) == :test do
-        fetch(prefetch_url)
-      else
-        ConcurrentLimiter.limit(__MODULE__, fn ->
-          Task.start(fn -> fetch(prefetch_url) end)
-        end)
-      end
+      fetch(prefetch_url)
     end
   end
 
-  defp fetch(url), do: HTTP.get(url, [], @adapter_options)
+  defp fetch(url) do
+    http_client_opts = Pleroma.Config.get([:media_proxy, :proxy_opts, :http], pool: :media)
+    HTTP.get(url, [], http_client_opts)
+  end
 
   defp preload(%{"object" => %{"attachment" => attachments}} = _message) do
     Enum.each(attachments, fn

lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex

@@ -0,0 +1,265 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.MRF.NsfwApiPolicy do
+  @moduledoc """
+  Hide, delete, or mark sensitive NSFW content with artificial intelligence.
+
+  Requires a NSFW API server, configured like so:
+
+      config :pleroma, Pleroma.Web.ActivityPub.MRF.NsfwMRF,
+        url: "http://127.0.0.1:5000/",
+        threshold: 0.7,
+        mark_sensitive: true,
+        unlist: false,
+        reject: false
+
+  The NSFW API server must implement an HTTP endpoint like this:
+
+      curl http://localhost:5000/?url=https://fedi.com/images/001.jpg
+
+  Returning a response like this:
+
+      {"score", 0.314}
+
+  Where a score is 0-1, with `1` being definitely NSFW.
+
+  A good API server is here: https://github.com/EugenCepoi/nsfw_api
+  You can run it with Docker with a one-liner:
+
+      docker run -it -p 127.0.0.1:5000:5000/tcp --env PORT=5000 eugencepoi/nsfw_api:latest
+
+  Options:
+
+  - `url`: Base URL of the API server. Default: "http://127.0.0.1:5000/"
+  - `threshold`: Lowest score to take action on. Default: `0.7`
+  - `mark_sensitive`: Mark sensitive all detected NSFW content? Default: `true`
+  - `unlist`: Unlist all detected NSFW content? Default: `false`
+  - `reject`: Reject all detected NSFW content (takes precedence)? Default: `false`
+  """
+  alias Pleroma.Config
+  alias Pleroma.Constants
+  alias Pleroma.HTTP
+  alias Pleroma.User
+
+  require Logger
+  require Pleroma.Constants
+
+  @behaviour Pleroma.Web.ActivityPub.MRF.Policy
+  @policy :mrf_nsfw_api
+
+  def build_request_url(url) do
+    Config.get([@policy, :url])
+    |> URI.parse()
+    |> fix_path()
+    |> Map.put(:query, "url=#{url}")
+    |> URI.to_string()
+  end
+
+  def parse_url(url) do
+    request = build_request_url(url)
+
+    with {:ok, %Tesla.Env{body: body}} <- HTTP.get(request) do
+      Jason.decode(body)
+    else
+      error ->
+        Logger.warn("""
+        [NsfwApiPolicy]: The API server failed. Skipping.
+        #{inspect(error)}
+        """)
+
+        error
+    end
+  end
+
+  def check_url_nsfw(url) when is_binary(url) do
+    threshold = Config.get([@policy, :threshold])
+
+    case parse_url(url) do
+      {:ok, %{"score" => score}} when score >= threshold ->
+        {:nsfw, %{url: url, score: score, threshold: threshold}}
+
+      {:ok, %{"score" => score}} ->
+        {:sfw, %{url: url, score: score, threshold: threshold}}
+
+      _ ->
+        {:sfw, %{url: url, score: nil, threshold: threshold}}
+    end
+  end
+
+  def check_url_nsfw(%{"href" => url}) when is_binary(url) do
+    check_url_nsfw(url)
+  end
+
+  def check_url_nsfw(url) do
+    threshold = Config.get([@policy, :threshold])
+    {:sfw, %{url: url, score: nil, threshold: threshold}}
+  end
+
+  def check_attachment_nsfw(%{"url" => urls} = attachment) when is_list(urls) do
+    if Enum.all?(urls, &match?({:sfw, _}, check_url_nsfw(&1))) do
+      {:sfw, attachment}
+    else
+      {:nsfw, attachment}
+    end
+  end
+
+  def check_attachment_nsfw(%{"url" => url} = attachment) when is_binary(url) do
+    case check_url_nsfw(url) do
+      {:sfw, _} -> {:sfw, attachment}
+      {:nsfw, _} -> {:nsfw, attachment}
+    end
+  end
+
+  def check_attachment_nsfw(attachment), do: {:sfw, attachment}
+
+  def check_object_nsfw(%{"attachment" => attachments} = object) when is_list(attachments) do
+    if Enum.all?(attachments, &match?({:sfw, _}, check_attachment_nsfw(&1))) do
+      {:sfw, object}
+    else
+      {:nsfw, object}
+    end
+  end
+
+  def check_object_nsfw(%{"object" => %{} = child_object} = object) do
+    case check_object_nsfw(child_object) do
+      {:sfw, _} -> {:sfw, object}
+      {:nsfw, _} -> {:nsfw, object}
+    end
+  end
+
+  def check_object_nsfw(object), do: {:sfw, object}
+
+  @impl true
+  def filter(object) do
+    with {:sfw, object} <- check_object_nsfw(object) do
+      {:ok, object}
+    else
+      {:nsfw, _data} -> handle_nsfw(object)
+      _ -> {:reject, "NSFW: Attachment rejected"}
+    end
+  end
+
+  defp handle_nsfw(object) do
+    if Config.get([@policy, :reject]) do
+      {:reject, object}
+    else
+      {:ok,
+       object
+       |> maybe_unlist()
+       |> maybe_mark_sensitive()}
+    end
+  end
+
+  defp maybe_unlist(object) do
+    if Config.get([@policy, :unlist]) do
+      unlist(object)
+    else
+      object
+    end
+  end
+
+  defp maybe_mark_sensitive(object) do
+    if Config.get([@policy, :mark_sensitive]) do
+      mark_sensitive(object)
+    else
+      object
+    end
+  end
+
+  def unlist(%{"to" => to, "cc" => cc, "actor" => actor} = object) do
+    with %User{} = user <- User.get_cached_by_ap_id(actor) do
+      to =
+        [user.follower_address | to]
+        |> List.delete(Constants.as_public())
+        |> Enum.uniq()
+
+      cc =
+        [Constants.as_public() | cc]
+        |> List.delete(user.follower_address)
+        |> Enum.uniq()
+
+      object
+      |> Map.put("to", to)
+      |> Map.put("cc", cc)
+    else
+      _ -> raise "[NsfwApiPolicy]: Could not find user #{actor}"
+    end
+  end
+
+  def mark_sensitive(%{"object" => child_object} = object) when is_map(child_object) do
+    Map.put(object, "object", mark_sensitive(child_object))
+  end
+
+  def mark_sensitive(object) when is_map(object) do
+    tags = (object["tag"] || []) ++ ["nsfw"]
+
+    object
+    |> Map.put("tag", tags)
+    |> Map.put("sensitive", true)
+  end
+
+  # Hackney needs a trailing slash
+  defp fix_path(%URI{path: path} = uri) when is_binary(path) do
+    path = String.trim_trailing(path, "/") <> "/"
+    Map.put(uri, :path, path)
+  end
+
+  defp fix_path(%URI{path: nil} = uri), do: Map.put(uri, :path, "/")
+
+  @impl true
+  def describe do
+    options = %{
+      threshold: Config.get([@policy, :threshold]),
+      mark_sensitive: Config.get([@policy, :mark_sensitive]),
+      unlist: Config.get([@policy, :unlist]),
+      reject: Config.get([@policy, :reject])
+    }
+
+    {:ok, %{@policy => options}}
+  end
+
+  @impl true
+  def config_description do
+    %{
+      key: @policy,
+      related_policy: to_string(__MODULE__),
+      label: "NSFW API Policy",
+      description:
+        "Hide, delete, or mark sensitive NSFW content with artificial intelligence. Requires running an external API server.",
+      children: [
+        %{
+          key: :url,
+          type: :string,
+          description: "Base URL of the API server.",
+          suggestions: ["http://127.0.0.1:5000/"]
+        },
+        %{
+          key: :threshold,
+          type: :float,
+          description: "Lowest score to take action on. Between 0 and 1.",
+          suggestions: [0.7]
+        },
+        %{
+          key: :mark_sensitive,
+          type: :boolean,
+          description: "Mark sensitive all detected NSFW content?",
+          suggestions: [true]
+        },
+        %{
+          key: :unlist,
+          type: :boolean,
+          description: "Unlist sensitive all detected NSFW content?",
+          suggestions: [false]
+        },
+        %{
+          key: :reject,
+          type: :boolean,
+          description: "Reject sensitive all detected NSFW content (takes precedence)?",
+          suggestions: [false]
+        }
+      ]
+    }
+  end
+end

lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex

@@ -15,6 +15,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator do
     field(:type, :string, default: "Link")
     field(:mediaType, ObjectValidators.MIME, default: "application/octet-stream")
     field(:name, :string)
+    field(:summary, :string)
     field(:blurhash, :string)
 
     embeds_many :url, UrlObjectValidator, primary_key: false do
@@ -44,7 +45,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator do
       |> fix_url()
 
     struct
-    |> cast(data, [:id, :type, :mediaType, :name, :blurhash])
+    |> cast(data, [:id, :type, :mediaType, :name, :summary, :blurhash])
     |> cast_embed(:url, with: &url_changeset/2, required: true)
     |> validate_inclusion(:type, ~w[Link Document Audio Image Video])
     |> validate_required([:type, :mediaType])

lib/pleroma/web/activity_pub/side_effects.ex

@@ -21,7 +21,6 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
   alias Pleroma.Web.ActivityPub.Builder
   alias Pleroma.Web.ActivityPub.Pipeline
   alias Pleroma.Web.ActivityPub.Utils
-  alias Pleroma.Web.Push
   alias Pleroma.Web.Streamer
   alias Pleroma.Workers.PollWorker
 
@@ -125,7 +124,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
         nil
     end
 
-    {:ok, notifications} = Notification.create_notifications(object, do_send: false)
+    {:ok, notifications} = Notification.create_notifications(object)
 
     meta =
       meta
@@ -184,7 +183,11 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
     liked_object = Object.get_by_ap_id(object.data["object"])
     Utils.add_like_to_object(object, liked_object)
 
-    Notification.create_notifications(object)
+    {:ok, notifications} = Notification.create_notifications(object)
+
+    meta =
+      meta
+      |> add_notifications(notifications)
 
     {:ok, object, meta}
   end
@@ -202,7 +205,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
   def handle(%{data: %{"type" => "Create"}} = activity, meta) do
     with {:ok, object, meta} <- handle_object_creation(meta[:object_data], activity, meta),
          %User{} = user <- User.get_cached_by_ap_id(activity.data["actor"]) do
-      {:ok, notifications} = Notification.create_notifications(activity, do_send: false)
+      {:ok, notifications} = Notification.create_notifications(activity)
       {:ok, _user} = ActivityPub.increase_note_count_if_public(user, object)
       {:ok, _user} = ActivityPub.update_last_status_at_if_public(user, object)
 
@@ -256,11 +259,13 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
 
     Utils.add_announce_to_object(object, announced_object)
 
-    if !User.internal?(user) do
-      Notification.create_notifications(object)
+    {:ok, notifications} = Notification.create_notifications(object)
 
-      ap_streamer().stream_out(object)
-    end
+    if !User.internal?(user), do: ap_streamer().stream_out(object)
+
+    meta =
+      meta
+      |> add_notifications(notifications)
 
     {:ok, object, meta}
   end
@@ -281,7 +286,11 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
     reacted_object = Object.get_by_ap_id(object.data["object"])
     Utils.add_emoji_reaction_to_object(object, reacted_object)
 
-    Notification.create_notifications(object)
+    {:ok, notifications} = Notification.create_notifications(object)
+
+    meta =
+      meta
+      |> add_notifications(notifications)
 
     {:ok, object, meta}
   end
@@ -585,10 +594,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
 
   defp send_notifications(meta) do
     Keyword.get(meta, :notifications, [])
-    |> Enum.each(fn notification ->
-      Streamer.stream(["user", "user:notification"], notification)
-      Push.send(notification)
-    end)
+    |> Notification.send()
 
     meta
   end

lib/pleroma/web/activity_pub/utils.ex

@@ -721,14 +721,18 @@ defmodule Pleroma.Web.ActivityPub.Utils do
 
   #### Flag-related helpers
   @spec make_flag_data(map(), map()) :: map()
-  def make_flag_data(%{actor: actor, context: context, content: content} = params, additional) do
+  def make_flag_data(
+        %{actor: actor, context: context, content: content} = params,
+        additional
+      ) do
     %{
       "type" => "Flag",
       "actor" => actor.ap_id,
       "content" => content,
       "object" => build_flag_object(params),
       "context" => context,
-      "state" => "open"
+      "state" => "open",
+      "rules" => Map.get(params, :rules, nil)
     }
     |> Map.merge(additional)
   end

lib/pleroma/web/admin_api/controllers/rule_controller.ex

@@ -0,0 +1,62 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.AdminAPI.RuleController do
+  use Pleroma.Web, :controller
+
+  alias Pleroma.Repo
+  alias Pleroma.Rule
+  alias Pleroma.Web.Plugs.OAuthScopesPlug
+
+  import Pleroma.Web.ControllerHelper,
+    only: [
+      json_response: 3
+    ]
+
+  plug(Pleroma.Web.ApiSpec.CastAndValidate)
+
+  plug(
+    OAuthScopesPlug,
+    %{scopes: ["admin:write"]}
+    when action in [:create, :update, :delete]
+  )
+
+  plug(OAuthScopesPlug, %{scopes: ["admin:read"]} when action == :index)
+
+  action_fallback(AdminAPI.FallbackController)
+
+  defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.Admin.RuleOperation
+
+  def index(conn, _) do
+    rules =
+      Rule.query()
+      |> Repo.all()
+
+    render(conn, "index.json", rules: rules)
+  end
+
+  def create(%{body_params: params} = conn, _) do
+    rule =
+      params
+      |> Rule.create()
+
+    render(conn, "show.json", rule: rule)
+  end
+
+  def update(%{body_params: params} = conn, %{id: id}) do
+    rule =
+      params
+      |> Rule.update(id)
+
+    render(conn, "show.json", rule: rule)
+  end
+
+  def delete(conn, %{id: id}) do
+    with {:ok, _} <- Rule.delete(id) do
+      json(conn, %{})
+    else
+      _ -> json_response(conn, :bad_request, "")
+    end
+  end
+end

lib/pleroma/web/admin_api/views/report_view.ex

@@ -6,9 +6,11 @@ defmodule Pleroma.Web.AdminAPI.ReportView do
   use Pleroma.Web, :view
 
   alias Pleroma.HTML
+  alias Pleroma.Rule
   alias Pleroma.User
   alias Pleroma.Web.AdminAPI
   alias Pleroma.Web.AdminAPI.Report
+  alias Pleroma.Web.AdminAPI.RuleView
   alias Pleroma.Web.CommonAPI.Utils
   alias Pleroma.Web.MastodonAPI.StatusView
 
@@ -46,7 +48,8 @@ defmodule Pleroma.Web.AdminAPI.ReportView do
           as: :activity
         }),
       state: report.data["state"],
-      notes: render(__MODULE__, "index_notes.json", %{notes: report.report_notes})
+      notes: render(__MODULE__, "index_notes.json", %{notes: report.report_notes}),
+      rules: rules(Map.get(report.data, "rules", nil))
     }
   end
 
@@ -71,4 +74,16 @@ defmodule Pleroma.Web.AdminAPI.ReportView do
       created_at: Utils.to_masto_date(inserted_at)
     }
   end
+
+  defp rules(nil) do
+    []
+  end
+
+  defp rules(rule_ids) do
+    rules =
+      rule_ids
+      |> Rule.get()
+
+    render(RuleView, "index.json", rules: rules)
+  end
 end

lib/pleroma/web/admin_api/views/rule_view.ex

@@ -0,0 +1,22 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.AdminAPI.RuleView do
+  use Pleroma.Web, :view
+
+  require Pleroma.Constants
+
+  def render("index.json", %{rules: rules} = _opts) do
+    render_many(rules, __MODULE__, "show.json")
+  end
+
+  def render("show.json", %{rule: rule} = _opts) do
+    %{
+      id: to_string(rule.id),
+      priority: rule.priority,
+      text: rule.text,
+      hint: rule.hint
+    }
+  end
+end

lib/pleroma/web/api_spec.ex

@@ -97,6 +97,7 @@ defmodule Pleroma.Web.ApiSpec do
               "Frontend management",
               "Instance configuration",
               "Instance documents",
+              "Instance rule managment",
               "Invites",
               "MediaProxy cache",
               "OAuth application management",

lib/pleroma/web/api_spec/operations/account_operation.ex

@@ -11,6 +11,7 @@ defmodule Pleroma.Web.ApiSpec.AccountOperation do
   alias Pleroma.Web.ApiSpec.Schemas.ActorType
   alias Pleroma.Web.ApiSpec.Schemas.ApiError
   alias Pleroma.Web.ApiSpec.Schemas.BooleanLike
+  alias Pleroma.Web.ApiSpec.Schemas.FlakeID
   alias Pleroma.Web.ApiSpec.Schemas.List
   alias Pleroma.Web.ApiSpec.Schemas.Status
   alias Pleroma.Web.ApiSpec.Schemas.VisibilityScope
@@ -513,6 +514,48 @@ defmodule Pleroma.Web.ApiSpec.AccountOperation do
     }
   end
 
+  def familiar_followers_operation do
+    %Operation{
+      tags: ["Retrieve account information"],
+      summary: "Followers that you follow",
+      operationId: "AccountController.familiar_followers",
+      description:
+        "Obtain a list of all accounts that follow a given account, filtered for accounts you follow.",
+      security: [%{"oAuth" => ["read:follows"]}],
+      parameters: [
+        Operation.parameter(
+          :id,
+          :query,
+          %Schema{
+            oneOf: [%Schema{type: :array, items: %Schema{type: :string}}, %Schema{type: :string}]
+          },
+          "Account IDs",
+          example: "123"
+        )
+      ],
+      responses: %{
+        200 =>
+          Operation.response("Accounts", "application/json", %Schema{
+            title: "ArrayOfAccounts",
+            type: :array,
+            items: %Schema{
+              title: "Account",
+              type: :object,
+              properties: %{
+                id: FlakeID,
+                accounts: %Schema{
+                  title: "ArrayOfAccounts",
+                  type: :array,
+                  items: Account,
+                  example: [Account.schema().example]
+                }
+              }
+            }
+          })
+      }
+    }
+  end
+
   defp create_request do
     %Schema{
       title: "AccountCreateRequest",

lib/pleroma/web/api_spec/operations/admin/report_operation.ex

@@ -30,6 +30,12 @@ defmodule Pleroma.Web.ApiSpec.Admin.ReportOperation do
           report_state(),
           "Filter by report state"
         ),
+        Operation.parameter(
+          :rule_id,
+          :query,
+          %Schema{type: :string},
+          "Filter by selected rule id"
+        ),
         Operation.parameter(
           :limit,
           :query,
@@ -169,6 +175,17 @@ defmodule Pleroma.Web.ApiSpec.Admin.ReportOperation do
               inserted_at: %Schema{type: :string, format: :"date-time"}
             }
           }
+        },
+        rules: %Schema{
+          type: :array,
+          items: %Schema{
+            type: :object,
+            properties: %{
+              id: %Schema{type: :string},
+              text: %Schema{type: :string},
+              hint: %Schema{type: :string, nullable: true}
+            }
+          }
         }
       }
     }

lib/pleroma/web/api_spec/operations/admin/rule_operation.ex

@@ -0,0 +1,115 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ApiSpec.Admin.RuleOperation do
+  alias OpenApiSpex.Operation
+  alias OpenApiSpex.Schema
+  alias Pleroma.Web.ApiSpec.Schemas.ApiError
+
+  import Pleroma.Web.ApiSpec.Helpers
+
+  def open_api_operation(action) do
+    operation = String.to_existing_atom("#{action}_operation")
+    apply(__MODULE__, operation, [])
+  end
+
+  def index_operation do
+    %Operation{
+      tags: ["Instance rule managment"],
+      summary: "Retrieve list of instance rules",
+      operationId: "AdminAPI.RuleController.index",
+      security: [%{"oAuth" => ["admin:read"]}],
+      responses: %{
+        200 =>
+          Operation.response("Response", "application/json", %Schema{
+            type: :array,
+            items: rule()
+          }),
+        403 => Operation.response("Forbidden", "application/json", ApiError)
+      }
+    }
+  end
+
+  def create_operation do
+    %Operation{
+      tags: ["Instance rule managment"],
+      summary: "Create new rule",
+      operationId: "AdminAPI.RuleController.create",
+      security: [%{"oAuth" => ["admin:write"]}],
+      parameters: admin_api_params(),
+      requestBody: request_body("Parameters", create_request(), required: true),
+      responses: %{
+        200 => Operation.response("Response", "application/json", rule()),
+        400 => Operation.response("Bad Request", "application/json", ApiError),
+        403 => Operation.response("Forbidden", "application/json", ApiError)
+      }
+    }
+  end
+
+  def update_operation do
+    %Operation{
+      tags: ["Instance rule managment"],
+      summary: "Modify existing rule",
+      operationId: "AdminAPI.RuleController.update",
+      security: [%{"oAuth" => ["admin:write"]}],
+      parameters: [Operation.parameter(:id, :path, :string, "Rule ID")],
+      requestBody: request_body("Parameters", update_request(), required: true),
+      responses: %{
+        200 => Operation.response("Response", "application/json", rule()),
+        400 => Operation.response("Bad Request", "application/json", ApiError),
+        403 => Operation.response("Forbidden", "application/json", ApiError)
+      }
+    }
+  end
+
+  def delete_operation do
+    %Operation{
+      tags: ["Instance rule managment"],
+      summary: "Delete rule",
+      operationId: "AdminAPI.RuleController.delete",
+      parameters: [Operation.parameter(:id, :path, :string, "Rule ID")],
+      security: [%{"oAuth" => ["admin:write"]}],
+      responses: %{
+        200 => empty_object_response(),
+        404 => Operation.response("Not Found", "application/json", ApiError),
+        403 => Operation.response("Forbidden", "application/json", ApiError)
+      }
+    }
+  end
+
+  defp create_request do
+    %Schema{
+      type: :object,
+      required: [:text],
+      properties: %{
+        priority: %Schema{type: :integer},
+        text: %Schema{type: :string},
+        hint: %Schema{type: :string}
+      }
+    }
+  end
+
+  defp update_request do
+    %Schema{
+      type: :object,
+      properties: %{
+        priority: %Schema{type: :integer},
+        text: %Schema{type: :string},
+        hint: %Schema{type: :string}
+      }
+    }
+  end
+
+  defp rule do
+    %Schema{
+      type: :object,
+      properties: %{
+        id: %Schema{type: :string},
+        priority: %Schema{type: :integer},
+        text: %Schema{type: :string},
+        hint: %Schema{type: :string, nullable: true}
+      }
+    }
+  end
+end

lib/pleroma/web/api_spec/operations/instance_operation.ex

@@ -46,10 +46,30 @@ defmodule Pleroma.Web.ApiSpec.InstanceOperation do
     }
   end
 
+  def rules_operation do
+    %Operation{
+      tags: ["Instance misc"],
+      summary: "Retrieve list of instance rules",
+      operationId: "InstanceController.rules",
+      responses: %{
+        200 => Operation.response("Array of domains", "application/json", array_of_rules())
+      }
+    }
+  end
+
   defp instance do
     %Schema{
       type: :object,
       properties: %{
+        accounts: %Schema{
+          type: :object,
+          properties: %{
+            max_featured_tags: %Schema{
+              type: :integer,
+              description: "The maximum number of featured tags allowed for each account."
+            }
+          }
+        },
         uri: %Schema{type: :string, description: "The domain name of the instance"},
         title: %Schema{type: :string, description: "The title of the website"},
         description: %Schema{
@@ -172,7 +192,8 @@ defmodule Pleroma.Web.ApiSpec.InstanceOperation do
         "urls" => %{
           "streaming_api" => "wss://lain.com"
         },
-        "version" => "2.7.2 (compatible; Pleroma 2.0.50-536-g25eec6d7-develop)"
+        "version" => "2.7.2 (compatible; Pleroma 2.0.50-536-g25eec6d7-develop)",
+        "rules" => array_of_rules()
       }
     }
   end
@@ -272,6 +293,19 @@ defmodule Pleroma.Web.ApiSpec.InstanceOperation do
           type: :object,
           description: "Instance configuration",
           properties: %{
+            accounts: %Schema{
+              type: :object,
+              properties: %{
+                max_featured_tags: %Schema{
+                  type: :integer,
+                  description: "The maximum number of featured tags allowed for each account."
+                },
+                max_pinned_statuses: %Schema{
+                  type: :integer,
+                  description: "The maximum number of pinned statuses for each account."
+                }
+              }
+            },
             urls: %Schema{
               type: :object,
               properties: %{
@@ -285,6 +319,11 @@ defmodule Pleroma.Web.ApiSpec.InstanceOperation do
               type: :object,
               description: "A map with poll limits for local statuses",
               properties: %{
+                characters_reserved_per_url: %Schema{
+                  type: :integer,
+                  description:
+                    "Each URL in a status will be assumed to be exactly this many characters."
+                },
                 max_characters: %Schema{
                   type: :integer,
                   description: "Posts character limit (CW/Subject included in the counter)"
@@ -344,4 +383,18 @@ defmodule Pleroma.Web.ApiSpec.InstanceOperation do
       example: ["pleroma.site", "lain.com", "bikeshed.party"]
     }
   end
+
+  defp array_of_rules do
+    %Schema{
+      type: :array,
+      items: %Schema{
+        type: :object,
+        properties: %{
+          id: %Schema{type: :string},
+          text: %Schema{type: :string},
+          hint: %Schema{type: :string}
+        }
+      }
+    }
+  end
 end

lib/pleroma/web/api_spec/operations/notification_operation.ex

@@ -202,7 +202,8 @@ defmodule Pleroma.Web.ApiSpec.NotificationOperation do
         "pleroma:report",
         "move",
         "follow_request",
-        "poll"
+        "poll",
+        "status"
       ],
       description: """
       The type of event that resulted in the notification.
@@ -216,6 +217,7 @@ defmodule Pleroma.Web.ApiSpec.NotificationOperation do
       - `pleroma:emoji_reaction` - Someone reacted with emoji to your status
       - `pleroma:chat_mention` - Someone mentioned you in a chat message
       - `pleroma:report` - Someone was reported
+      - `status` - Someone you are subscribed to created a status
       """
     }
   end

lib/pleroma/web/api_spec/operations/pleroma_notification_operation.ex

@@ -5,7 +5,6 @@
 defmodule Pleroma.Web.ApiSpec.PleromaNotificationOperation do
   alias OpenApiSpex.Operation
   alias OpenApiSpex.Schema
-  alias Pleroma.Web.ApiSpec.NotificationOperation
   alias Pleroma.Web.ApiSpec.Schemas.ApiError
 
   import Pleroma.Web.ApiSpec.Helpers
@@ -35,12 +34,7 @@ defmodule Pleroma.Web.ApiSpec.PleromaNotificationOperation do
           Operation.response(
             "A Notification or array of Notifications",
             "application/json",
-            %Schema{
-              anyOf: [
-                %Schema{type: :array, items: NotificationOperation.notification()},
-                NotificationOperation.notification()
-              ]
-            }
+            %Schema{type: :string}
           ),
         400 => Operation.response("Bad Request", "application/json", ApiError)
       }

lib/pleroma/web/api_spec/operations/report_operation.ex

@@ -53,6 +53,12 @@ defmodule Pleroma.Web.ApiSpec.ReportOperation do
           default: false,
           description:
             "If the account is remote, should the report be forwarded to the remote admin?"
+        },
+        rule_ids: %Schema{
+          type: :array,
+          nullable: true,
+          items: %Schema{type: :string},
+          description: "Array of rules"
         }
       },
       required: [:account_id],
@@ -60,7 +66,8 @@ defmodule Pleroma.Web.ApiSpec.ReportOperation do
         "account_id" => "123",
         "status_ids" => ["1337"],
         "comment" => "bad status!",
-        "forward" => "false"
+        "forward" => "false",
+        "rule_ids" => ["3"]
       }
     }
   end

lib/pleroma/web/api_spec/schemas/attachment.ex

@@ -50,7 +50,11 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Attachment do
       pleroma: %Schema{
         type: :object,
         properties: %{
-          mime_type: %Schema{type: :string, description: "mime type of the attachment"}
+          mime_type: %Schema{type: :string, description: "mime type of the attachment"},
+          name: %Schema{
+            type: :string,
+            description: "Name of the attachment, typically the filename"
+          }
         }
       }
     },

lib/pleroma/web/common_api.ex

@@ -8,6 +8,7 @@ defmodule Pleroma.Web.CommonAPI do
   alias Pleroma.Formatter
   alias Pleroma.ModerationLog
   alias Pleroma.Object
+  alias Pleroma.Rule
   alias Pleroma.ThreadMute
   alias Pleroma.User
   alias Pleroma.UserRelationship
@@ -568,14 +569,16 @@ defmodule Pleroma.Web.CommonAPI do
   def report(user, data) do
     with {:ok, account} <- get_reported_account(data.account_id),
          {:ok, {content_html, _, _}} <- make_report_content_html(data[:comment]),
-         {:ok, statuses} <- get_report_statuses(account, data) do
+         {:ok, statuses} <- get_report_statuses(account, data),
+         rules <- get_report_rules(Map.get(data, :rule_ids, nil)) do
       ActivityPub.flag(%{
         context: Utils.generate_context_id(),
         actor: user,
         account: account,
         statuses: statuses,
         content: content_html,
-        forward: Map.get(data, :forward, false)
+        forward: Map.get(data, :forward, false),
+        rules: rules
       })
     end
   end
@@ -587,6 +590,15 @@ defmodule Pleroma.Web.CommonAPI do
     end
   end
 
+  defp get_report_rules(nil) do
+    nil
+  end
+
+  defp get_report_rules(rule_ids) do
+    rule_ids
+    |> Enum.filter(&Rule.exists?/1)
+  end
+
   def update_report_state(activity_ids, state) when is_list(activity_ids) do
     case Utils.update_report_state(activity_ids, state) do
       :ok -> {:ok, activity_ids}

lib/pleroma/web/common_api/activity_draft.ex

@@ -129,8 +129,22 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do
 
   defp in_reply_to(%{params: %{in_reply_to_status_id: ""}} = draft), do: draft
 
-  defp in_reply_to(%{params: %{in_reply_to_status_id: id}} = draft) when is_binary(id) do
-    %__MODULE__{draft | in_reply_to: Activity.get_by_id(id)}
+  defp in_reply_to(%{params: %{in_reply_to_status_id: :deleted}} = draft) do
+    add_error(draft, dgettext("errors", "Cannot reply to a deleted status"))
+  end
+
+  defp in_reply_to(%{params: %{in_reply_to_status_id: id} = params} = draft) when is_binary(id) do
+    activity = Activity.get_by_id(id)
+
+    params =
+      if is_nil(activity) do
+        # Deleted activities are returned as nil
+        Map.put(params, :in_reply_to_status_id, :deleted)
+      else
+        Map.put(params, :in_reply_to_status_id, activity)
+      end
+
+    in_reply_to(%{draft | params: params})
   end
 
   defp in_reply_to(%{params: %{in_reply_to_status_id: %Activity{} = in_reply_to}} = draft) do

lib/pleroma/web/endpoint.ex

@@ -38,6 +38,8 @@ defmodule Pleroma.Web.Endpoint do
 
   plug(Plug.Telemetry, event_prefix: [:phoenix, :endpoint])
 
+  plug(Pleroma.Web.Plugs.LoggerMetadataPath)
+
   plug(Pleroma.Web.Plugs.SetLocalePlug)
   plug(CORSPlug)
   plug(Pleroma.Web.Plugs.HTTPSecurityPlug)

lib/pleroma/web/federator.ex

@@ -44,7 +44,7 @@ defmodule Pleroma.Web.Federator do
   end
 
   def incoming_ap_doc(%{"type" => "Delete"} = params) do
-    ReceiverWorker.enqueue("incoming_ap_doc", %{"params" => params}, priority: 3)
+    ReceiverWorker.enqueue("incoming_ap_doc", %{"params" => params}, priority: 3, queue: :slow)
   end
 
   def incoming_ap_doc(params) do

lib/pleroma/web/mastodon_api/controllers/account_controller.ex

@@ -72,7 +72,10 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do
     %{scopes: ["follow", "write:blocks"]} when action in [:block, :unblock]
   )
 
-  plug(OAuthScopesPlug, %{scopes: ["read:follows"]} when action == :relationships)
+  plug(
+    OAuthScopesPlug,
+    %{scopes: ["read:follows"]} when action in [:relationships, :familiar_followers]
+  )
 
   plug(
     OAuthScopesPlug,
@@ -629,6 +632,35 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do
     )
   end
 
+  @doc "GET /api/v1/accounts/familiar_followers"
+  def familiar_followers(
+        %{assigns: %{user: user}, private: %{open_api_spex: %{params: %{id: id}}}} = conn,
+        _id
+      ) do
+    users =
+      User.get_all_by_ids(List.wrap(id))
+      |> Enum.map(&%{id: &1.id, accounts: get_familiar_followers(&1, user)})
+
+    conn
+    |> render("familiar_followers.json",
+      for: user,
+      users: users,
+      as: :user
+    )
+  end
+
+  defp get_familiar_followers(%{id: id} = user, %{id: id}) do
+    User.get_familiar_followers(user, user)
+  end
+
+  defp get_familiar_followers(%{hide_followers: true}, _current_user) do
+    []
+  end
+
+  defp get_familiar_followers(user, current_user) do
+    User.get_familiar_followers(user, current_user)
+  end
+
   @doc "GET /api/v1/identity_proofs"
   def identity_proofs(conn, params), do: MastodonAPIController.empty_array(conn, params)
 end

lib/pleroma/web/mastodon_api/controllers/instance_controller.ex

@@ -25,4 +25,9 @@ defmodule Pleroma.Web.MastodonAPI.InstanceController do
   def peers(conn, _params) do
     json(conn, Pleroma.Stats.get_peers())
   end
+
+  @doc "GET /api/v1/instance/rules"
+  def rules(conn, _params) do
+    render(conn, "rules.json")
+  end
 end

lib/pleroma/web/mastodon_api/controllers/notification_controller.ex

@@ -34,6 +34,7 @@ defmodule Pleroma.Web.MastodonAPI.NotificationController do
       pleroma:emoji_reaction
       poll
       update
+      status
     }
 
   # GET /api/v1/notifications

lib/pleroma/web/mastodon_api/views/account_view.ex

@@ -193,6 +193,25 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do
     render_many(targets, AccountView, "relationship.json", render_opts)
   end
 
+  def render("familiar_followers.json", %{users: users} = opts) do
+    opts =
+      opts
+      |> Map.merge(%{as: :user})
+      |> Map.delete(:users)
+
+    users
+    |> render_many(AccountView, "familiar_followers.json", opts)
+  end
+
+  def render("familiar_followers.json", %{user: %{id: id, accounts: accounts}} = opts) do
+    accounts =
+      accounts
+      |> render_many(AccountView, "show.json", opts)
+      |> Enum.filter(&Enum.any?/1)
+
+    %{id: id, accounts: accounts}
+  end
+
   defp do_render("show.json", %{user: user} = opts) do
     self = opts[:for] == user
 

lib/pleroma/web/mastodon_api/views/instance_view.ex

@@ -76,12 +76,26 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do
     })
   end
 
+  def render("rules.json", _) do
+    Pleroma.Rule.query()
+    |> Pleroma.Repo.all()
+    |> render_many(__MODULE__, "rule.json", as: :rule)
+  end
+
+  def render("rule.json", %{rule: rule}) do
+    %{
+      id: to_string(rule.id),
+      text: rule.text,
+      hint: rule.hint || ""
+    }
+  end
+
   defp common_information(instance) do
     %{
-      title: Keyword.get(instance, :name),
-      version: "#{@mastodon_api_level} (compatible; #{Pleroma.Application.named_version()})",
       languages: Keyword.get(instance, :languages, ["en"]),
-      rules: []
+      rules: render(__MODULE__, "rules.json"),
+      title: Keyword.get(instance, :name),
+      version: "#{@mastodon_api_level} (compatible; #{Pleroma.Application.named_version()})"
     }
   end
 
@@ -138,6 +152,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do
 
   def federation do
     quarantined = Config.get([:instance, :quarantined_instances], [])
+    rejected = Config.get([:instance, :rejected_instances], [])
 
     if Config.get([:mrf, :transparency]) do
       {:ok, data} = MRF.describe()
@@ -157,6 +172,12 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do
           |> Enum.map(fn {instance, reason} -> {instance, %{"reason" => reason}} end)
           |> Map.new()
       })
+      |> Map.put(
+        :rejected_instances,
+        rejected
+        |> Enum.map(fn {instance, reason} -> {instance, %{"reason" => reason}} end)
+        |> Map.new()
+      )
     else
       %{}
     end
@@ -213,6 +234,8 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do
 
   defp configuration2 do
     configuration()
+    |> put_in([:accounts, :max_pinned_statuses], Config.get([:instance, :max_pinned_statuses], 0))
+    |> put_in([:statuses, :characters_reserved_per_url], 0)
     |> Map.merge(%{
       urls: %{
         streaming: Pleroma.Web.Endpoint.websocket_url(),

lib/pleroma/web/mastodon_api/views/notification_view.ex

@@ -108,6 +108,9 @@ defmodule Pleroma.Web.MastodonAPI.NotificationView do
       "mention" ->
         put_status(response, activity, reading_user, status_render_opts)
 
+      "status" ->
+        put_status(response, activity, reading_user, status_render_opts)
+
       "favourite" ->
         put_status(response, parent_activity_fn.(), reading_user, status_render_opts)
 

lib/pleroma/web/mastodon_api/views/status_view.ex

@@ -624,6 +624,19 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
           to_string(attachment["id"] || hash_id)
       end
 
+    description =
+      if attachment["summary"] do
+        HTML.strip_tags(attachment["summary"])
+      else
+        attachment["name"]
+      end
+
+    name = if attachment["summary"], do: attachment["name"]
+
+    pleroma =
+      %{mime_type: media_type}
+      |> Maps.put_if_present(:name, name)
+
     %{
       id: attachment_id,
       url: href,
@@ -631,8 +644,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
       preview_url: href_preview,
       text_url: href,
       type: type,
-      description: attachment["name"],
-      pleroma: %{mime_type: media_type},
+      description: description,
+      pleroma: pleroma,
       blurhash: attachment["blurhash"]
     }
     |> Maps.put_if_present(:meta, meta)

lib/pleroma/web/media_proxy/media_proxy_controller.ex

@@ -54,9 +54,10 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
 
   defp handle_preview(conn, url) do
     media_proxy_url = MediaProxy.url(url)
+    http_client_opts = Pleroma.Config.get([:media_proxy, :proxy_opts, :http], pool: :media)
 
     with {:ok, %{status: status} = head_response} when status in 200..299 <-
-           Pleroma.HTTP.request(:head, media_proxy_url, "", [], pool: :media) do
+           Pleroma.HTTP.request(:head, media_proxy_url, "", [], http_client_opts) do
       content_type = Tesla.get_header(head_response, "content-type")
       content_length = Tesla.get_header(head_response, "content-length")
       content_length = content_length && String.to_integer(content_length)

lib/pleroma/web/pleroma_api/controllers/notification_controller.ex

@@ -23,8 +23,9 @@ defmodule Pleroma.Web.PleromaAPI.NotificationController do
         } = conn,
         _
       ) do
-    with {:ok, notification} <- Notification.read_one(user, notification_id) do
-      render(conn, "show.json", notification: notification, for: user)
+    with {:ok, _} <- Notification.read_one(user, notification_id) do
+      conn
+      |> json("ok")
     else
       {:error, message} ->
         conn
@@ -38,11 +39,14 @@ defmodule Pleroma.Web.PleromaAPI.NotificationController do
           conn,
         _
       ) do
-    notifications =
-      user
-      |> Notification.set_read_up_to(max_id)
-      |> Enum.take(80)
-
-    render(conn, "index.json", notifications: notifications, for: user)
+    with {:ok, _} <- Notification.set_read_up_to(user, max_id) do
+      conn
+      |> json("ok")
+    else
+      {:error, message} ->
+        conn
+        |> put_status(:bad_request)
+        |> json(%{"error" => message})
+    end
   end
 end

lib/pleroma/web/plugs/http_security_plug.ex

@@ -3,26 +3,27 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do
-  alias Pleroma.Config
   import Plug.Conn
 
   require Logger
 
+  @config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config)
+
   def init(opts), do: opts
 
   def call(conn, _options) do
-    if Config.get([:http_security, :enabled]) do
+    if @config_impl.get([:http_security, :enabled]) do
       conn
       |> merge_resp_headers(headers())
-      |> maybe_send_sts_header(Config.get([:http_security, :sts]))
+      |> maybe_send_sts_header(@config_impl.get([:http_security, :sts]))
     else
       conn
     end
   end
 
   def primary_frontend do
-    with %{"name" => frontend} <- Config.get([:frontends, :primary]),
-         available <- Config.get([:frontends, :available]),
+    with %{"name" => frontend} <- @config_impl.get([:frontends, :primary]),
+         available <- @config_impl.get([:frontends, :available]),
          %{} = primary_frontend <- Map.get(available, frontend) do
       {:ok, primary_frontend}
     end
@@ -37,8 +38,8 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do
   end
 
   def headers do
-    referrer_policy = Config.get([:http_security, :referrer_policy])
-    report_uri = Config.get([:http_security, :report_uri])
+    referrer_policy = @config_impl.get([:http_security, :referrer_policy])
+    report_uri = @config_impl.get([:http_security, :report_uri])
     custom_http_frontend_headers = custom_http_frontend_headers()
 
     headers = [
@@ -86,10 +87,10 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do
   @csp_start [Enum.join(static_csp_rules, ";") <> ";"]
 
   defp csp_string do
-    scheme = Config.get([Pleroma.Web.Endpoint, :url])[:scheme]
+    scheme = @config_impl.get([Pleroma.Web.Endpoint, :url])[:scheme]
     static_url = Pleroma.Web.Endpoint.static_url()
     websocket_url = Pleroma.Web.Endpoint.websocket_url()
-    report_uri = Config.get([:http_security, :report_uri])
+    report_uri = @config_impl.get([:http_security, :report_uri])
 
     img_src = "img-src 'self' data: blob:"
     media_src = "media-src 'self'"
@@ -97,8 +98,8 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do
 
     # Strict multimedia CSP enforcement only when MediaProxy is enabled
     {img_src, media_src, connect_src} =
-      if Config.get([:media_proxy, :enabled]) &&
-           !Config.get([:media_proxy, :proxy_opts, :redirect_on_failure]) do
+      if @config_impl.get([:media_proxy, :enabled]) &&
+           !@config_impl.get([:media_proxy, :proxy_opts, :redirect_on_failure]) do
         sources = build_csp_multimedia_source_list()
 
         {
@@ -115,17 +116,21 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do
       end
 
     connect_src =
-      if Config.get(:env) == :dev do
+      if @config_impl.get([:env]) == :dev do
         [connect_src, " http://localhost:3035/"]
       else
         connect_src
       end
 
     script_src =
-      if Config.get(:env) == :dev do
-        "script-src 'self' 'unsafe-eval'"
+      if @config_impl.get([:http_security, :allow_unsafe_eval]) do
+        if @config_impl.get([:env]) == :dev do
+          "script-src 'self' 'unsafe-eval'"
+        else
+          "script-src 'self' 'wasm-unsafe-eval'"
+        end
       else
-        "script-src 'self' 'wasm-unsafe-eval'"
+        "script-src 'self'"
       end
 
     report = if report_uri, do: ["report-uri ", report_uri, ";report-to csp-endpoint"]
@@ -161,11 +166,11 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do
   defp build_csp_multimedia_source_list do
     media_proxy_whitelist =
       [:media_proxy, :whitelist]
-      |> Config.get()
+      |> @config_impl.get()
       |> build_csp_from_whitelist([])
 
-    captcha_method = Config.get([Pleroma.Captcha, :method])
-    captcha_endpoint = Config.get([captcha_method, :endpoint])
+    captcha_method = @config_impl.get([Pleroma.Captcha, :method])
+    captcha_endpoint = @config_impl.get([captcha_method, :endpoint])
 
     base_endpoints =
       [
@@ -173,7 +178,7 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do
         [Pleroma.Upload, :base_url],
         [Pleroma.Uploaders.S3, :public_endpoint]
       ]
-      |> Enum.map(&Config.get/1)
+      |> Enum.map(&@config_impl.get/1)
 
     [captcha_endpoint | base_endpoints]
     |> Enum.map(&build_csp_param/1)
@@ -200,7 +205,7 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do
   end
 
   def warn_if_disabled do
-    unless Config.get([:http_security, :enabled]) do
+    unless Pleroma.Config.get([:http_security, :enabled]) do
       Logger.warning("
                                  .i;;;;i.
                                iYcviii;vXY:
@@ -245,8 +250,8 @@ your instance and your users via malicious posts:
   end
 
   defp maybe_send_sts_header(conn, true) do
-    max_age_sts = Config.get([:http_security, :sts_max_age])
-    max_age_ct = Config.get([:http_security, :ct_max_age])
+    max_age_sts = @config_impl.get([:http_security, :sts_max_age])
+    max_age_ct = @config_impl.get([:http_security, :ct_max_age])
 
     merge_resp_headers(conn, [
       {"strict-transport-security", "max-age=#{max_age_sts}; includeSubDomains"},

lib/pleroma/web/plugs/http_signature_plug.ex

@@ -3,10 +3,22 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.Plugs.HTTPSignaturePlug do
+  alias Pleroma.Helpers.InetHelper
+
   import Plug.Conn
   import Phoenix.Controller, only: [get_format: 1, text: 2]
+
+  alias Pleroma.Web.ActivityPub.MRF
+
   require Logger
 
+  @config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config)
+  @http_signatures_impl Application.compile_env(
+                          :pleroma,
+                          [__MODULE__, :http_signatures_impl],
+                          HTTPSignatures
+                        )
+
   def init(options) do
     options
   end
@@ -19,7 +31,9 @@ defmodule Pleroma.Web.Plugs.HTTPSignaturePlug do
     if get_format(conn) in ["json", "activity+json"] do
       conn
       |> maybe_assign_valid_signature()
+      |> maybe_assign_actor_id()
       |> maybe_require_signature()
+      |> maybe_filter_requests()
     else
       conn
     end
@@ -33,7 +47,7 @@ defmodule Pleroma.Web.Plugs.HTTPSignaturePlug do
       |> put_req_header("(request-target)", request_target)
       |> put_req_header("@request-target", request_target)
 
-    HTTPSignatures.validate_conn(conn)
+    @http_signatures_impl.validate_conn(conn)
   end
 
   defp validate_signature(conn) do
@@ -83,20 +97,63 @@ defmodule Pleroma.Web.Plugs.HTTPSignaturePlug do
     end
   end
 
+  defp maybe_assign_actor_id(%{assigns: %{valid_signature: true}} = conn) do
+    adapter = Application.get_env(:http_signatures, :adapter)
+
+    {:ok, actor_id} = adapter.get_actor_id(conn)
+
+    assign(conn, :actor_id, actor_id)
+  end
+
+  defp maybe_assign_actor_id(conn), do: conn
+
   defp has_signature_header?(conn) do
     conn |> get_req_header("signature") |> Enum.at(0, false)
   end
 
   defp maybe_require_signature(%{assigns: %{valid_signature: true}} = conn), do: conn
 
-  defp maybe_require_signature(conn) do
-    if Pleroma.Config.get([:activitypub, :authorized_fetch_mode], false) do
-      conn
-      |> put_status(:unauthorized)
-      |> text("Request not signed")
-      |> halt()
+  defp maybe_require_signature(%{remote_ip: remote_ip} = conn) do
+    if @config_impl.get([:activitypub, :authorized_fetch_mode], false) do
+      exceptions =
+        @config_impl.get([:activitypub, :authorized_fetch_mode_exceptions], [])
+        |> Enum.map(&InetHelper.parse_cidr/1)
+
+      if Enum.any?(exceptions, fn x -> InetCidr.contains?(x, remote_ip) end) do
+        conn
+      else
+        conn
+        |> put_status(:unauthorized)
+        |> text("Request not signed")
+        |> halt()
+      end
     else
       conn
     end
   end
+
+  defp maybe_filter_requests(%{halted: true} = conn), do: conn
+
+  defp maybe_filter_requests(conn) do
+    if @config_impl.get([:activitypub, :authorized_fetch_mode], false) and
+         conn.assigns[:actor_id] do
+      %{host: host} = URI.parse(conn.assigns.actor_id)
+
+      if MRF.subdomain_match?(rejected_domains(), host) do
+        conn
+        |> put_status(:unauthorized)
+        |> halt()
+      else
+        conn
+      end
+    else
+      conn
+    end
+  end
+
+  defp rejected_domains do
+    @config_impl.get([:instance, :rejected_instances])
+    |> Pleroma.Web.ActivityPub.MRF.instance_list_from_tuples()
+    |> Pleroma.Web.ActivityPub.MRF.subdomains_regex()
+  end
 end

lib/pleroma/web/plugs/logger_metadata_path.ex

@@ -0,0 +1,12 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.Plugs.LoggerMetadataPath do
+  def init(opts), do: opts
+
+  def call(conn, _) do
+    Logger.metadata(path: conn.request_path)
+    conn
+  end
+end

lib/pleroma/web/plugs/logger_metadata_user.ex

@@ -0,0 +1,18 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.Plugs.LoggerMetadataUser do
+  alias Pleroma.User
+
+  def init(opts), do: opts
+
+  def call(%{assigns: %{user: user = %User{}}} = conn, _) do
+    Logger.metadata(user: user.nickname)
+    conn
+  end
+
+  def call(conn, _) do
+    conn
+  end
+end

lib/pleroma/web/plugs/remote_ip.ex

@@ -8,6 +8,7 @@ defmodule Pleroma.Web.Plugs.RemoteIp do
   """
 
   alias Pleroma.Config
+  alias Pleroma.Helpers.InetHelper
   import Plug.Conn
 
   @behaviour Plug
@@ -30,19 +31,8 @@ defmodule Pleroma.Web.Plugs.RemoteIp do
     proxies =
       Config.get([__MODULE__, :proxies], [])
       |> Enum.concat(reserved)
-      |> Enum.map(&maybe_add_cidr/1)
+      |> Enum.map(&InetHelper.parse_cidr/1)
 
     {headers, proxies}
   end
-
-  defp maybe_add_cidr(proxy) when is_binary(proxy) do
-    proxy =
-      cond do
-        "/" in String.codepoints(proxy) -> proxy
-        InetCidr.v4?(InetCidr.parse_address!(proxy)) -> proxy <> "/32"
-        InetCidr.v6?(InetCidr.parse_address!(proxy)) -> proxy <> "/128"
-      end
-
-    InetCidr.parse_cidr!(proxy, true)
-  end
 end

lib/pleroma/web/push/impl.ex

@@ -192,6 +192,7 @@ defmodule Pleroma.Web.Push.Impl do
   def format_title(%{type: type}, mastodon_type) do
     case mastodon_type || type do
       "mention" -> "New Mention"
+      "status" -> "New Status"
       "follow" -> "New Follower"
       "follow_request" -> "New Follow Request"
       "reblog" -> "New Repeat"

lib/pleroma/web/rich_media/helpers.ex

@@ -58,7 +58,7 @@ defmodule Pleroma.Web.RichMedia.Helpers do
 
   defp http_options do
     [
-      pool: :media,
+      pool: :rich_media,
       max_body: Config.get([:rich_media, :max_body], 5_000_000)
     ]
   end

lib/pleroma/web/rich_media/parser/ttl/aws_signed_url.ex

@@ -23,7 +23,7 @@ defmodule Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl do
     %URI{host: host, query: query} = URI.parse(image)
 
     is_binary(host) and String.contains?(host, "amazonaws.com") and
-      String.contains?(query, "X-Amz-Expires")
+      is_binary(query) and String.contains?(query, "X-Amz-Expires")
   end
 
   defp aws_signed_url?(_), do: nil

lib/pleroma/web/router.ex

@@ -29,6 +29,7 @@ defmodule Pleroma.Web.Router do
   pipeline :browser do
     plug(:accepts, ["html"])
     plug(:fetch_session)
+    plug(Pleroma.Web.Plugs.LoggerMetadataUser)
   end
 
   pipeline :oauth do
@@ -67,12 +68,14 @@ defmodule Pleroma.Web.Router do
     plug(:fetch_session)
     plug(:authenticate)
     plug(OpenApiSpex.Plug.PutApiSpec, module: Pleroma.Web.ApiSpec)
+    plug(Pleroma.Web.Plugs.LoggerMetadataUser)
   end
 
   pipeline :no_auth_or_privacy_expectations_api do
     plug(:base_api)
     plug(:after_auth)
     plug(Pleroma.Web.Plugs.IdempotencyPlug)
+    plug(Pleroma.Web.Plugs.LoggerMetadataUser)
   end
 
   # Pipeline for app-related endpoints (no user auth checks — app-bound tokens must be supported)
@@ -83,12 +86,14 @@ defmodule Pleroma.Web.Router do
   pipeline :api do
     plug(:expect_public_instance_or_user_authentication)
     plug(:no_auth_or_privacy_expectations_api)
+    plug(Pleroma.Web.Plugs.LoggerMetadataUser)
   end
 
   pipeline :authenticated_api do
     plug(:expect_user_authentication)
     plug(:no_auth_or_privacy_expectations_api)
     plug(Pleroma.Web.Plugs.EnsureAuthenticatedPlug)
+    plug(Pleroma.Web.Plugs.LoggerMetadataUser)
   end
 
   pipeline :admin_api do
@@ -99,6 +104,7 @@ defmodule Pleroma.Web.Router do
     plug(Pleroma.Web.Plugs.EnsureAuthenticatedPlug)
     plug(Pleroma.Web.Plugs.UserIsStaffPlug)
     plug(Pleroma.Web.Plugs.IdempotencyPlug)
+    plug(Pleroma.Web.Plugs.LoggerMetadataUser)
   end
 
   pipeline :require_admin do
@@ -179,6 +185,7 @@ defmodule Pleroma.Web.Router do
     plug(:browser)
     plug(:authenticate)
     plug(Pleroma.Web.Plugs.EnsureUserTokenAssignsPlug)
+    plug(Pleroma.Web.Plugs.LoggerMetadataUser)
   end
 
   pipeline :well_known do
@@ -193,6 +200,7 @@ defmodule Pleroma.Web.Router do
   pipeline :pleroma_api do
     plug(:accepts, ["html", "json"])
     plug(OpenApiSpex.Plug.PutApiSpec, module: Pleroma.Web.ApiSpec)
+    plug(Pleroma.Web.Plugs.LoggerMetadataUser)
   end
 
   pipeline :mailbox_preview do
@@ -292,6 +300,11 @@ defmodule Pleroma.Web.Router do
     post("/frontends/install", FrontendController, :install)
 
     post("/backups", AdminAPIController, :create_backup)
+
+    get("/rules", RuleController, :index)
+    post("/rules", RuleController, :create)
+    patch("/rules/:id", RuleController, :update)
+    delete("/rules/:id", RuleController, :delete)
   end
 
   # AdminAPI: admins and mods (staff) can perform these actions (if privileged by role)
@@ -633,6 +646,7 @@ defmodule Pleroma.Web.Router do
     patch("/accounts/update_credentials", AccountController, :update_credentials)
 
     get("/accounts/relationships", AccountController, :relationships)
+    get("/accounts/familiar_followers", AccountController, :familiar_followers)
     get("/accounts/:id/lists", AccountController, :lists)
     get("/accounts/:id/identity_proofs", AccountController, :identity_proofs)
     get("/endorsements", AccountController, :endorsements)
@@ -764,6 +778,7 @@ defmodule Pleroma.Web.Router do
 
     get("/instance", InstanceController, :show)
     get("/instance/peers", InstanceController, :peers)
+    get("/instance/rules", InstanceController, :rules)
 
     get("/statuses", StatusController, :index)
     get("/statuses/:id", StatusController, :show)

lib/pleroma/web/web_finger.ex

@@ -155,7 +155,16 @@ defmodule Pleroma.Web.WebFinger do
     end
   end
 
+  @cachex Pleroma.Config.get([:cachex, :provider], Cachex)
   def find_lrdd_template(domain) do
+    @cachex.fetch!(:host_meta_cache, domain, fn _ ->
+      {:commit, fetch_lrdd_template(domain)}
+    end)
+  rescue
+    e -> {:error, "Cachex error: #{inspect(e)}"}
+  end
+
+  defp fetch_lrdd_template(domain) do
     # WebFinger is restricted to HTTPS - https://tools.ietf.org/html/rfc7033#section-9.1
     meta_url = "https://#{domain}/.well-known/host-meta"
 
@@ -168,7 +177,7 @@ defmodule Pleroma.Web.WebFinger do
     end
   end
 
-  defp get_address_from_domain(domain, encoded_account) when is_binary(domain) do
+  defp get_address_from_domain(domain, "acct:" <> _ = encoded_account) when is_binary(domain) do
     case find_lrdd_template(domain) do
       {:ok, template} ->
         String.replace(template, "{uri}", encoded_account)
@@ -178,6 +187,11 @@ defmodule Pleroma.Web.WebFinger do
     end
   end
 
+  defp get_address_from_domain(domain, account) when is_binary(domain) do
+    encoded_account = URI.encode("acct:#{account}")
+    get_address_from_domain(domain, encoded_account)
+  end
+
   defp get_address_from_domain(_, _), do: {:error, :webfinger_no_domain}
 
   @spec finger(String.t()) :: {:ok, map()} | {:error, any()}
@@ -192,9 +206,7 @@ defmodule Pleroma.Web.WebFinger do
           URI.parse(account).host
       end
 
-    encoded_account = URI.encode("acct:#{account}")
-
-    with address when is_binary(address) <- get_address_from_domain(domain, encoded_account),
+    with address when is_binary(address) <- get_address_from_domain(domain, account),
          {:ok, %{status: status, body: body, headers: headers}} when status in 200..299 <-
            HTTP.get(
              address,
@@ -216,10 +228,28 @@ defmodule Pleroma.Web.WebFinger do
         _ ->
           {:error, {:content_type, nil}}
       end
+      |> case do
+        {:ok, data} -> validate_webfinger(address, data)
+        error -> error
+      end
     else
       error ->
         Logger.debug("Couldn't finger #{account}: #{inspect(error)}")
         error
     end
   end
+
+  defp validate_webfinger(request_url, %{"subject" => "acct:" <> acct = subject} = data) do
+    with [_name, acct_host] <- String.split(acct, "@"),
+         {_, url} <- {:address, get_address_from_domain(acct_host, subject)},
+         %URI{host: request_host} <- URI.parse(request_url),
+         %URI{host: acct_host} <- URI.parse(url),
+         {_, true} <- {:hosts_match, acct_host == request_host} do
+      {:ok, data}
+    else
+      _ -> {:error, {:webfinger_invalid, request_url, data}}
+    end
+  end
+
+  defp validate_webfinger(url, data), do: {:error, {:webfinger_invalid, url, data}}
 end

lib/pleroma/workers/attachments_cleanup_worker.ex

@@ -8,7 +8,7 @@ defmodule Pleroma.Workers.AttachmentsCleanupWorker do
   alias Pleroma.Object
   alias Pleroma.Repo
 
-  use Pleroma.Workers.WorkerHelper, queue: "attachments_cleanup"
+  use Pleroma.Workers.WorkerHelper, queue: "slow"
 
   @impl Oban.Worker
   def perform(%Job{

lib/pleroma/workers/backup_worker.ex

@@ -3,7 +3,7 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Workers.BackupWorker do
-  use Oban.Worker, queue: :backup, max_attempts: 1
+  use Oban.Worker, queue: :slow, max_attempts: 1
 
   alias Oban.Job
   alias Pleroma.User.Backup

lib/pleroma/workers/cron/new_users_digest_worker.ex

@@ -9,7 +9,7 @@ defmodule Pleroma.Workers.Cron.NewUsersDigestWorker do
 
   import Ecto.Query
 
-  use Pleroma.Workers.WorkerHelper, queue: "mailer"
+  use Pleroma.Workers.WorkerHelper, queue: "background"
 
   @impl Oban.Worker
   def perform(_job) do

lib/pleroma/workers/mailer_worker.ex

@@ -3,7 +3,7 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Workers.MailerWorker do
-  use Pleroma.Workers.WorkerHelper, queue: "mailer"
+  use Pleroma.Workers.WorkerHelper, queue: "background"
 
   @impl Oban.Worker
   def perform(%Job{args: %{"op" => "email", "encoded_email" => encoded_email, "config" => config}}) do

lib/pleroma/workers/mute_expire_worker.ex

@@ -3,7 +3,7 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Workers.MuteExpireWorker do
-  use Pleroma.Workers.WorkerHelper, queue: "mute_expire"
+  use Pleroma.Workers.WorkerHelper, queue: "background"
 
   @impl Oban.Worker
   def perform(%Job{args: %{"op" => "unmute_user", "muter_id" => muter_id, "mutee_id" => mutee_id}}) do

lib/pleroma/workers/poll_worker.ex

@@ -6,7 +6,7 @@ defmodule Pleroma.Workers.PollWorker do
   @moduledoc """
   Generates notifications when a poll ends.
   """
-  use Pleroma.Workers.WorkerHelper, queue: "poll_notifications"
+  use Pleroma.Workers.WorkerHelper, queue: "background"
 
   alias Pleroma.Activity
   alias Pleroma.Notification

lib/pleroma/workers/purge_expired_activity.ex

@@ -7,7 +7,7 @@ defmodule Pleroma.Workers.PurgeExpiredActivity do
   Worker which purges expired activity.
   """
 
-  use Oban.Worker, queue: :activity_expiration, max_attempts: 1, unique: [period: :infinity]
+  use Oban.Worker, queue: :slow, max_attempts: 1, unique: [period: :infinity]
 
   import Ecto.Query
 
@@ -59,7 +59,7 @@ defmodule Pleroma.Workers.PurgeExpiredActivity do
   def get_expiration(id) do
     from(j in Oban.Job,
       where: j.state == "scheduled",
-      where: j.queue == "activity_expiration",
+      where: j.queue == "slow",
       where: fragment("?->>'activity_id' = ?", j.args, ^id)
     )
     |> Pleroma.Repo.one()

lib/pleroma/workers/purge_expired_filter.ex

@@ -7,7 +7,7 @@ defmodule Pleroma.Workers.PurgeExpiredFilter do
   Worker which purges expired filters
   """
 
-  use Oban.Worker, queue: :filter_expiration, max_attempts: 1, unique: [period: :infinity]
+  use Oban.Worker, queue: :background, max_attempts: 1, unique: [period: :infinity]
 
   import Ecto.Query
 
@@ -38,7 +38,7 @@ defmodule Pleroma.Workers.PurgeExpiredFilter do
   def get_expiration(id) do
     from(j in Job,
       where: j.state == "scheduled",
-      where: j.queue == "filter_expiration",
+      where: j.queue == "background",
       where: fragment("?->'filter_id' = ?", j.args, ^id)
     )
     |> Repo.one()

lib/pleroma/workers/purge_expired_token.ex

@@ -7,7 +7,7 @@ defmodule Pleroma.Workers.PurgeExpiredToken do
   Worker which purges expired OAuth tokens
   """
 
-  use Oban.Worker, queue: :token_expiration, max_attempts: 1
+  use Oban.Worker, queue: :background, max_attempts: 1
 
   @spec enqueue(%{token_id: integer(), valid_until: DateTime.t(), mod: module()}) ::
           {:ok, Oban.Job.t()} | {:error, Ecto.Changeset.t()}

lib/pleroma/workers/remote_fetcher_worker.ex

@@ -5,7 +5,7 @@
 defmodule Pleroma.Workers.RemoteFetcherWorker do
   alias Pleroma.Object.Fetcher
 
-  use Pleroma.Workers.WorkerHelper, queue: "remote_fetcher"
+  use Pleroma.Workers.WorkerHelper, queue: "background"
 
   @impl Oban.Worker
   def perform(%Job{args: %{"op" => "fetch_remote", "id" => id} = args}) do

lib/pleroma/workers/rich_media_expiration_worker.ex

@@ -6,7 +6,7 @@ defmodule Pleroma.Workers.RichMediaExpirationWorker do
   alias Pleroma.Web.RichMedia.Card
 
   use Oban.Worker,
-    queue: :rich_media_expiration
+    queue: :background
 
   @impl Oban.Worker
   def perform(%Job{args: %{"url" => url} = _args}) do

lib/pleroma/workers/scheduled_activity_worker.ex

@@ -7,7 +7,7 @@ defmodule Pleroma.Workers.ScheduledActivityWorker do
   The worker to post scheduled activity.
   """
 
-  use Pleroma.Workers.WorkerHelper, queue: "scheduled_activities"
+  use Pleroma.Workers.WorkerHelper, queue: "federator_outgoing"
 
   alias Pleroma.Repo
   alias Pleroma.ScheduledActivity