From 718e8e1edb537aca984216be39b3be5c8af4e6da Mon Sep 17 00:00:00 2001
From: Alex Gleason <alex@alexgleason.me>
Date: Sun, 16 May 2021 21:39:58 -0500
Subject: [PATCH 001/192] Create NsfwApiPolicy

---
 config/config.exs                             |   7 +
 .../web/activity_pub/mrf/nsfw_api_policy.ex   | 185 ++++++++++++++++++
 2 files changed, 192 insertions(+)
 create mode 100644 lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex

diff --git a/config/config.exs b/config/config.exs
index 66aee3264..d96dc1646 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -404,6 +404,13 @@ config :pleroma, :mrf_object_age,
   threshold: 604_800,
   actions: [:delist, :strip_followers]
 
+config :pleroma, :mrf_nsfw_api,
+  url: "http://127.0.0.1:5000/",
+  threshold: 0.7,
+  mark_sensitive: true,
+  unlist: false,
+  reject: false
+
 config :pleroma, :rich_media,
   enabled: true,
   ignore_hosts: [],
diff --git a/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex b/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex
new file mode 100644
index 000000000..9ad175b1b
--- /dev/null
+++ b/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex
@@ -0,0 +1,185 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.MRF.NsfwApiPolicy do
+  @moduledoc """
+  Hide, delete, or mark sensitive NSFW content with artificial intelligence.
+
+  Requires a NSFW API server, configured like so:
+
+      config :pleroma, Pleroma.Web.ActivityPub.MRF.NsfwMRF,
+        url: "http://127.0.0.1:5000/",
+        threshold: 0.8,
+        mark_sensitive: true,
+        unlist: false,
+        reject: false
+
+  The NSFW API server must implement an HTTP endpoint like this:
+
+      curl http://localhost:5000/?url=https://fedi.com/images/001.jpg
+
+  Returning a response like this:
+
+      {"score", 0.314}
+
+  Where a score is 0-1, with `1` being definitely NSFW.
+
+  A good API server is here: https://github.com/EugenCepoi/nsfw_api
+  You can run it with Docker with a one-liner:
+
+      docker run -it -p 127.0.0.1:5000:5000/tcp --env PORT=5000 eugencepoi/nsfw_api:latest
+
+  Options:
+
+  - `url`: Base URL of the API server. Default: "http://127.0.0.1:5000/"
+  - `threshold`: Lowest score to take action on. Default: `0.7`
+  - `mark_sensitive`: Mark sensitive all detected NSFW content? Default: `true`
+  - `unlist`: Unlist all detected NSFW content? Default: `false`
+  - `reject`: Reject all detected NSFW content (takes precedence)? Default: `false`
+  """
+  alias Pleroma.Config
+  alias Pleroma.Constants
+  alias Pleroma.HTTP
+  alias Pleroma.User
+
+  require Logger
+  require Pleroma.Constants
+
+  @behaviour Pleroma.Web.ActivityPub.MRF
+  @policy :mrf_nsfw_api
+
+  defp build_request_url(url) do
+    Config.get([@policy, :url])
+    |> URI.parse()
+    |> Map.put(:query, "url=#{url}")
+    |> URI.to_string()
+  end
+
+  defp parse_url(url) do
+    request = build_request_url(url)
+
+    with {:ok, %Tesla.Env{body: body}} <- HTTP.get(request) do
+      Jason.decode(body)
+    else
+      error ->
+        Logger.warn("""
+        [NsfwApiPolicy]: The API server failed. Skipping.
+        #{inspect(error)}
+        """)
+
+        error
+    end
+  end
+
+  defp check_url_nsfw(url) when is_binary(url) do
+    threshold = Config.get([@policy, :threshold])
+
+    case parse_url(url) do
+      {:ok, %{"score" => score}} when score >= threshold ->
+        {:nsfw, %{url: url, score: score, threshold: threshold}}
+
+      _ ->
+        {:sfw, url}
+    end
+  end
+
+  defp check_url_nsfw(%{"href" => url}) when is_binary(url) do
+    check_url_nsfw(url)
+  end
+
+  defp check_attachment_nsfw(%{"url" => urls} = attachment) when is_list(urls) do
+    if Enum.all?(urls, &match?({:sfw, _}, check_url_nsfw(&1))) do
+      {:sfw, attachment}
+    else
+      {:nsfw, attachment}
+    end
+  end
+
+  defp check_object_nsfw(%{"attachment" => attachments} = object) when is_list(attachments) do
+    if Enum.all?(attachments, &match?({:sfw, _}, check_attachment_nsfw(&1))) do
+      {:sfw, object}
+    else
+      {:nsfw, object}
+    end
+  end
+
+  defp check_object_nsfw(%{"object" => %{} = child_object} = object) do
+    case check_object_nsfw(child_object) do
+      {:sfw, _} -> {:sfw, object}
+      {:nsfw, _} -> {:nsfw, object}
+    end
+  end
+
+  defp check_object_nsfw(object), do: {:sfw, object}
+
+  @impl true
+  def filter(object) do
+    with {:sfw, object} <- check_object_nsfw(object) do
+      {:ok, object}
+    else
+      {:nsfw, _data} -> handle_nsfw(object)
+      _ -> {:reject, "NSFW: Attachment rejected"}
+    end
+  end
+
+  defp handle_nsfw(object) do
+    if Config.get([@policy, :reject]) do
+      {:reject, object}
+    else
+      {:ok,
+       object
+       |> maybe_unlist()
+       |> maybe_mark_sensitive()}
+    end
+  end
+
+  defp maybe_unlist(object) do
+    if Config.get([@policy, :unlist]) do
+      unlist(object)
+    else
+      object
+    end
+  end
+
+  defp maybe_mark_sensitive(object) do
+    if Config.get([@policy, :mark_sensitive]) do
+      mark_sensitive(object)
+    else
+      object
+    end
+  end
+
+  defp unlist(%{"to" => to, "cc" => cc, "actor" => actor} = object) do
+    with %User{} = user <- User.get_cached_by_ap_id(actor) do
+      to =
+        [user.follower_address | to]
+        |> List.delete(Constants.as_public())
+        |> Enum.uniq()
+
+      cc =
+        [Constants.as_public() | cc]
+        |> List.delete(user.follower_address)
+        |> Enum.uniq()
+
+      object
+      |> Map.put("to", to)
+      |> Map.put("cc", cc)
+    end
+  end
+
+  defp mark_sensitive(%{"object" => child_object} = object) when is_map(child_object) do
+    Map.put(object, "object", mark_sensitive(child_object))
+  end
+
+  defp mark_sensitive(object) when is_map(object) do
+    tags = (object["tag"] || []) ++ ["nsfw"]
+
+    object
+    |> Map.put("tag", tags)
+    |> Map.put("sensitive", true)
+  end
+
+  @impl true
+  def describe, do: {:ok, %{}}
+end

From f15d419062b5f9aba2a2e84257dc2379b44f92e8 Mon Sep 17 00:00:00 2001
From: Alex Gleason <alex@alexgleason.me>
Date: Wed, 16 Jun 2021 22:30:18 -0500
Subject: [PATCH 002/192] NsfwApiPolicy: raise if can't fetch user

---
 lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex b/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex
index 9ad175b1b..63e6af0a0 100644
--- a/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex
@@ -165,6 +165,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.NsfwApiPolicy do
       object
       |> Map.put("to", to)
       |> Map.put("cc", cc)
+    else
+      _ -> raise "[NsfwApiPolicy]: Could not fetch user #{actor}"
     end
   end
 

From 2b3dfbb42f7ec0c5604876276a81d55a05955416 Mon Sep 17 00:00:00 2001
From: Alex Gleason <alex@alexgleason.me>
Date: Thu, 17 Jun 2021 14:36:51 -0500
Subject: [PATCH 003/192] NsfwApiPolicy: add tests

---
 .../web/activity_pub/mrf/nsfw_api_policy.ex   |  45 ++-
 .../activity_pub/mrf/nsfw_api_policy_test.exs | 267 ++++++++++++++++++
 2 files changed, 299 insertions(+), 13 deletions(-)
 create mode 100644 test/pleroma/web/activity_pub/mrf/nsfw_api_policy_test.exs

diff --git a/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex b/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex
index 63e6af0a0..9dcdf560e 100644
--- a/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex
@@ -49,14 +49,15 @@ defmodule Pleroma.Web.ActivityPub.MRF.NsfwApiPolicy do
   @behaviour Pleroma.Web.ActivityPub.MRF
   @policy :mrf_nsfw_api
 
-  defp build_request_url(url) do
+  def build_request_url(url) do
     Config.get([@policy, :url])
     |> URI.parse()
+    |> fix_path()
     |> Map.put(:query, "url=#{url}")
     |> URI.to_string()
   end
 
-  defp parse_url(url) do
+  def parse_url(url) do
     request = build_request_url(url)
 
     with {:ok, %Tesla.Env{body: body}} <- HTTP.get(request) do
@@ -72,23 +73,26 @@ defmodule Pleroma.Web.ActivityPub.MRF.NsfwApiPolicy do
     end
   end
 
-  defp check_url_nsfw(url) when is_binary(url) do
+  def check_url_nsfw(url) when is_binary(url) do
     threshold = Config.get([@policy, :threshold])
 
     case parse_url(url) do
       {:ok, %{"score" => score}} when score >= threshold ->
         {:nsfw, %{url: url, score: score, threshold: threshold}}
 
+      {:ok, %{"score" => score}} ->
+        {:sfw, %{url: url, score: score, threshold: threshold}}
+
       _ ->
-        {:sfw, url}
+        {:sfw, %{url: url, score: nil, threshold: threshold}}
     end
   end
 
-  defp check_url_nsfw(%{"href" => url}) when is_binary(url) do
+  def check_url_nsfw(%{"href" => url}) when is_binary(url) do
     check_url_nsfw(url)
   end
 
-  defp check_attachment_nsfw(%{"url" => urls} = attachment) when is_list(urls) do
+  def check_attachment_nsfw(%{"url" => urls} = attachment) when is_list(urls) do
     if Enum.all?(urls, &match?({:sfw, _}, check_url_nsfw(&1))) do
       {:sfw, attachment}
     else
@@ -96,7 +100,14 @@ defmodule Pleroma.Web.ActivityPub.MRF.NsfwApiPolicy do
     end
   end
 
-  defp check_object_nsfw(%{"attachment" => attachments} = object) when is_list(attachments) do
+  def check_attachment_nsfw(%{"url" => url} = attachment) when is_binary(url) do
+    case check_url_nsfw(url) do
+      {:sfw, _} -> {:sfw, attachment}
+      {:nsfw, _} -> {:nsfw, attachment}
+    end
+  end
+
+  def check_object_nsfw(%{"attachment" => attachments} = object) when is_list(attachments) do
     if Enum.all?(attachments, &match?({:sfw, _}, check_attachment_nsfw(&1))) do
       {:sfw, object}
     else
@@ -104,14 +115,14 @@ defmodule Pleroma.Web.ActivityPub.MRF.NsfwApiPolicy do
     end
   end
 
-  defp check_object_nsfw(%{"object" => %{} = child_object} = object) do
+  def check_object_nsfw(%{"object" => %{} = child_object} = object) do
     case check_object_nsfw(child_object) do
       {:sfw, _} -> {:sfw, object}
       {:nsfw, _} -> {:nsfw, object}
     end
   end
 
-  defp check_object_nsfw(object), do: {:sfw, object}
+  def check_object_nsfw(object), do: {:sfw, object}
 
   @impl true
   def filter(object) do
@@ -150,7 +161,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.NsfwApiPolicy do
     end
   end
 
-  defp unlist(%{"to" => to, "cc" => cc, "actor" => actor} = object) do
+  def unlist(%{"to" => to, "cc" => cc, "actor" => actor} = object) do
     with %User{} = user <- User.get_cached_by_ap_id(actor) do
       to =
         [user.follower_address | to]
@@ -166,15 +177,15 @@ defmodule Pleroma.Web.ActivityPub.MRF.NsfwApiPolicy do
       |> Map.put("to", to)
       |> Map.put("cc", cc)
     else
-      _ -> raise "[NsfwApiPolicy]: Could not fetch user #{actor}"
+      _ -> raise "[NsfwApiPolicy]: Could not find user #{actor}"
     end
   end
 
-  defp mark_sensitive(%{"object" => child_object} = object) when is_map(child_object) do
+  def mark_sensitive(%{"object" => child_object} = object) when is_map(child_object) do
     Map.put(object, "object", mark_sensitive(child_object))
   end
 
-  defp mark_sensitive(object) when is_map(object) do
+  def mark_sensitive(object) when is_map(object) do
     tags = (object["tag"] || []) ++ ["nsfw"]
 
     object
@@ -182,6 +193,14 @@ defmodule Pleroma.Web.ActivityPub.MRF.NsfwApiPolicy do
     |> Map.put("sensitive", true)
   end
 
+  # Hackney needs a trailing slash
+  defp fix_path(%URI{path: path} = uri) when is_binary(path) do
+    path = String.trim_trailing(path, "/") <> "/"
+    Map.put(uri, :path, path)
+  end
+
+  defp fix_path(%URI{path: nil} = uri), do: Map.put(uri, :path, "/")
+
   @impl true
   def describe, do: {:ok, %{}}
 end
diff --git a/test/pleroma/web/activity_pub/mrf/nsfw_api_policy_test.exs b/test/pleroma/web/activity_pub/mrf/nsfw_api_policy_test.exs
new file mode 100644
index 000000000..0beb9c2cb
--- /dev/null
+++ b/test/pleroma/web/activity_pub/mrf/nsfw_api_policy_test.exs
@@ -0,0 +1,267 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.MRF.NsfwApiPolicyTest do
+  use Pleroma.DataCase
+
+  import ExUnit.CaptureLog
+  import Pleroma.Factory
+
+  alias Pleroma.Constants
+  alias Pleroma.Web.ActivityPub.MRF.NsfwApiPolicy
+
+  require Pleroma.Constants
+
+  @policy :mrf_nsfw_api
+
+  @sfw_url "https://kittens.co/kitty.gif"
+  @nsfw_url "https://b00bies.com/nsfw.jpg"
+  @timeout_url "http://time.out/i.jpg"
+
+  setup_all do
+    clear_config(@policy,
+      url: "http://127.0.0.1:5000/",
+      threshold: 0.7,
+      mark_sensitive: true,
+      unlist: false,
+      reject: false
+    )
+  end
+
+  setup do
+    Tesla.Mock.mock(fn
+      # NSFW URL
+      %{method: :get, url: "http://127.0.0.1:5000/?url=#{@nsfw_url}"} ->
+        %Tesla.Env{status: 200, body: ~s({"score":0.99772077798843384,"url":"#{@nsfw_url}"})}
+
+      # SFW URL
+      %{method: :get, url: "http://127.0.0.1:5000/?url=#{@sfw_url}"} ->
+        %Tesla.Env{status: 200, body: ~s({"score":0.00011714912398019806,"url":"#{@sfw_url}"})}
+
+      # Timeout URL
+      %{method: :get, url: "http://127.0.0.1:5000/?url=#{@timeout_url}"} ->
+        {:error, :timeout}
+
+      # Fallback URL
+      %{method: :get, url: "http://127.0.0.1:5000/?url=" <> url} ->
+        body =
+          ~s({"error_code":500,"error_reason":"[Errno -2] Name or service not known","url":"#{url}"})
+
+        %Tesla.Env{status: 500, body: body}
+    end)
+
+    :ok
+  end
+
+  describe "build_request_url/1" do
+    test "it works" do
+      expected = "http://127.0.0.1:5000/?url=https://b00bies.com/nsfw.jpg"
+      assert NsfwApiPolicy.build_request_url(@nsfw_url) == expected
+    end
+
+    test "it adds a trailing slash" do
+      clear_config([@policy, :url], "http://localhost:5000")
+
+      expected = "http://localhost:5000/?url=https://b00bies.com/nsfw.jpg"
+      assert NsfwApiPolicy.build_request_url(@nsfw_url) == expected
+    end
+
+    test "it adds a trailing slash preserving the path" do
+      clear_config([@policy, :url], "http://localhost:5000/nsfw_api")
+
+      expected = "http://localhost:5000/nsfw_api/?url=https://b00bies.com/nsfw.jpg"
+      assert NsfwApiPolicy.build_request_url(@nsfw_url) == expected
+    end
+  end
+
+  describe "parse_url/1" do
+    test "returns decoded JSON from the API server" do
+      expected = %{"score" => 0.99772077798843384, "url" => @nsfw_url}
+      assert NsfwApiPolicy.parse_url(@nsfw_url) == {:ok, expected}
+    end
+
+    test "warns when the API server fails" do
+      expected = "[NsfwApiPolicy]: The API server failed. Skipping."
+      assert capture_log(fn -> NsfwApiPolicy.parse_url(@timeout_url) end) =~ expected
+    end
+
+    test "returns {:error, _} tuple when the API server fails" do
+      capture_log(fn ->
+        assert {:error, _} = NsfwApiPolicy.parse_url(@timeout_url)
+      end)
+    end
+  end
+
+  describe "check_url_nsfw/1" do
+    test "returns {:nsfw, _} tuple" do
+      expected = {:nsfw, %{url: @nsfw_url, score: 0.99772077798843384, threshold: 0.7}}
+      assert NsfwApiPolicy.check_url_nsfw(@nsfw_url) == expected
+    end
+
+    test "returns {:sfw, _} tuple" do
+      expected = {:sfw, %{url: @sfw_url, score: 0.00011714912398019806, threshold: 0.7}}
+      assert NsfwApiPolicy.check_url_nsfw(@sfw_url) == expected
+    end
+
+    test "returns {:sfw, _} on failure" do
+      expected = {:sfw, %{url: @timeout_url, score: nil, threshold: 0.7}}
+
+      capture_log(fn ->
+        assert NsfwApiPolicy.check_url_nsfw(@timeout_url) == expected
+      end)
+    end
+
+    test "works with map URL" do
+      expected = {:nsfw, %{url: @nsfw_url, score: 0.99772077798843384, threshold: 0.7}}
+      assert NsfwApiPolicy.check_url_nsfw(%{"href" => @nsfw_url}) == expected
+    end
+  end
+
+  describe "check_attachment_nsfw/1" do
+    test "returns {:nsfw, _} if any items are NSFW" do
+      attachment = %{"url" => [%{"href" => @nsfw_url}, @nsfw_url, @sfw_url]}
+      assert NsfwApiPolicy.check_attachment_nsfw(attachment) == {:nsfw, attachment}
+    end
+
+    test "returns {:sfw, _} if all items are SFW" do
+      attachment = %{"url" => [%{"href" => @sfw_url}, @sfw_url, @sfw_url]}
+      assert NsfwApiPolicy.check_attachment_nsfw(attachment) == {:sfw, attachment}
+    end
+
+    test "works with binary URL" do
+      attachment = %{"url" => @nsfw_url}
+      assert NsfwApiPolicy.check_attachment_nsfw(attachment) == {:nsfw, attachment}
+    end
+  end
+
+  describe "check_object_nsfw/1" do
+    test "returns {:nsfw, _} if any items are NSFW" do
+      object = %{"attachment" => [%{"url" => [%{"href" => @nsfw_url}, @sfw_url]}]}
+      assert NsfwApiPolicy.check_object_nsfw(object) == {:nsfw, object}
+    end
+
+    test "returns {:sfw, _} if all items are SFW" do
+      object = %{"attachment" => [%{"url" => [%{"href" => @sfw_url}, @sfw_url]}]}
+      assert NsfwApiPolicy.check_object_nsfw(object) == {:sfw, object}
+    end
+
+    test "works with embedded object" do
+      object = %{"object" => %{"attachment" => [%{"url" => [%{"href" => @nsfw_url}, @sfw_url]}]}}
+      assert NsfwApiPolicy.check_object_nsfw(object) == {:nsfw, object}
+    end
+  end
+
+  describe "unlist/1" do
+    test "unlist addressing" do
+      user = insert(:user)
+
+      object = %{
+        "to" => [Constants.as_public()],
+        "cc" => [user.follower_address, "https://hello.world/users/alex"],
+        "actor" => user.ap_id
+      }
+
+      expected = %{
+        "to" => [user.follower_address],
+        "cc" => [Constants.as_public(), "https://hello.world/users/alex"],
+        "actor" => user.ap_id
+      }
+
+      assert NsfwApiPolicy.unlist(object) == expected
+    end
+
+    test "raise if user isn't found" do
+      object = %{
+        "to" => [Constants.as_public()],
+        "cc" => [],
+        "actor" => "https://hello.world/users/alex"
+      }
+
+      assert_raise(RuntimeError, fn ->
+        NsfwApiPolicy.unlist(object)
+      end)
+    end
+  end
+
+  describe "mark_sensitive/1" do
+    test "adds nsfw tag and marks sensitive" do
+      object = %{"tag" => ["yolo"]}
+      expected = %{"tag" => ["yolo", "nsfw"], "sensitive" => true}
+      assert NsfwApiPolicy.mark_sensitive(object) == expected
+    end
+
+    test "works with embedded object" do
+      object = %{"object" => %{"tag" => ["yolo"]}}
+      expected = %{"object" => %{"tag" => ["yolo", "nsfw"], "sensitive" => true}}
+      assert NsfwApiPolicy.mark_sensitive(object) == expected
+    end
+  end
+
+  describe "filter/1" do
+    setup do
+      user = insert(:user)
+
+      nsfw_object = %{
+        "to" => [Constants.as_public()],
+        "cc" => [user.follower_address],
+        "actor" => user.ap_id,
+        "attachment" => [%{"url" => @nsfw_url}]
+      }
+
+      sfw_object = %{
+        "to" => [Constants.as_public()],
+        "cc" => [user.follower_address],
+        "actor" => user.ap_id,
+        "attachment" => [%{"url" => @sfw_url}]
+      }
+
+      %{user: user, nsfw_object: nsfw_object, sfw_object: sfw_object}
+    end
+
+    test "passes SFW object through", %{sfw_object: object} do
+      {:ok, _} = NsfwApiPolicy.filter(object)
+    end
+
+    test "passes NSFW object through when actions are disabled", %{nsfw_object: object} do
+      clear_config([@policy, :mark_sensitive], false)
+      clear_config([@policy, :unlist], false)
+      clear_config([@policy, :reject], false)
+      {:ok, _} = NsfwApiPolicy.filter(object)
+    end
+
+    test "passes NSFW object through when :threshold is 1", %{nsfw_object: object} do
+      clear_config([@policy, :reject], true)
+      clear_config([@policy, :threshold], 1)
+      {:ok, _} = NsfwApiPolicy.filter(object)
+    end
+
+    test "rejects SFW object through when :threshold is 0", %{sfw_object: object} do
+      clear_config([@policy, :reject], true)
+      clear_config([@policy, :threshold], 0)
+      {:reject, _} = NsfwApiPolicy.filter(object)
+    end
+
+    test "rejects NSFW when :reject is enabled", %{nsfw_object: object} do
+      clear_config([@policy, :reject], true)
+      {:reject, _} = NsfwApiPolicy.filter(object)
+    end
+
+    test "passes NSFW through when :reject is disabled", %{nsfw_object: object} do
+      clear_config([@policy, :reject], false)
+      {:ok, _} = NsfwApiPolicy.filter(object)
+    end
+
+    test "unlists NSFW when :unlist is enabled", %{user: user, nsfw_object: object} do
+      clear_config([@policy, :unlist], true)
+      {:ok, object} = NsfwApiPolicy.filter(object)
+      assert object["to"] == [user.follower_address]
+    end
+
+    test "passes NSFW through when :unlist is disabled", %{nsfw_object: object} do
+      clear_config([@policy, :unlist], false)
+      {:ok, object} = NsfwApiPolicy.filter(object)
+      assert object["to"] == [Constants.as_public()]
+    end
+  end
+end

From b293c14a1b01398029dfa80aea306946efc2f284 Mon Sep 17 00:00:00 2001
From: Alex Gleason <alex@alexgleason.me>
Date: Thu, 17 Jun 2021 14:52:07 -0500
Subject: [PATCH 004/192] NsfwApiPolicy: add describe/0 and
 config_description/0

---
 .../web/activity_pub/mrf/nsfw_api_policy.ex   | 56 ++++++++++++++++++-
 1 file changed, 54 insertions(+), 2 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex b/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex
index 9dcdf560e..a1560c584 100644
--- a/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex
@@ -10,7 +10,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.NsfwApiPolicy do
 
       config :pleroma, Pleroma.Web.ActivityPub.MRF.NsfwMRF,
         url: "http://127.0.0.1:5000/",
-        threshold: 0.8,
+        threshold: 0.7,
         mark_sensitive: true,
         unlist: false,
         reject: false
@@ -202,5 +202,57 @@ defmodule Pleroma.Web.ActivityPub.MRF.NsfwApiPolicy do
   defp fix_path(%URI{path: nil} = uri), do: Map.put(uri, :path, "/")
 
   @impl true
-  def describe, do: {:ok, %{}}
+  def describe do
+    options = %{
+      threshold: Config.get([@policy, :threshold]),
+      mark_sensitive: Config.get([@policy, :mark_sensitive]),
+      unlist: Config.get([@policy, :unlist]),
+      reject: Config.get([@policy, :reject])
+    }
+
+    {:ok, %{@policy => options}}
+  end
+
+  @impl true
+  def config_description do
+    %{
+      key: @policy,
+      related_policy: to_string(__MODULE__),
+      label: "NSFW API Policy",
+      description:
+        "Hide, delete, or mark sensitive NSFW content with artificial intelligence. Requires running an external API server.",
+      children: [
+        %{
+          key: :url,
+          type: :string,
+          description: "Base URL of the API server.",
+          suggestions: ["http://127.0.0.1:5000/"]
+        },
+        %{
+          key: :threshold,
+          type: :float,
+          description: "Lowest score to take action on. Between 0 and 1.",
+          suggestions: [0.7]
+        },
+        %{
+          key: :mark_sensitive,
+          type: :boolean,
+          description: "Mark sensitive all detected NSFW content?",
+          suggestions: [true]
+        },
+        %{
+          key: :unlist,
+          type: :boolean,
+          description: "Unlist sensitive all detected NSFW content?",
+          suggestions: [false]
+        },
+        %{
+          key: :reject,
+          type: :boolean,
+          description: "Reject sensitive all detected NSFW content (takes precedence)?",
+          suggestions: [false]
+        }
+      ]
+    }
+  end
 end

From c802c3055ef6c1f763d5df68f9e5308093f7d565 Mon Sep 17 00:00:00 2001
From: Alex Gleason <alex@alexgleason.me>
Date: Thu, 17 Jun 2021 15:04:40 -0500
Subject: [PATCH 005/192] NsfwApiPolicy: add systemd example file

---
 installation/nsfw-api.service | 15 +++++++++++++++
 1 file changed, 15 insertions(+)
 create mode 100644 installation/nsfw-api.service

diff --git a/installation/nsfw-api.service b/installation/nsfw-api.service
new file mode 100644
index 000000000..ec629df67
--- /dev/null
+++ b/installation/nsfw-api.service
@@ -0,0 +1,15 @@
+[Unit]
+Description=NSFW API
+After=docker.service
+Requires=docker.service
+
+[Service]
+TimeoutStartSec=0
+Restart=always
+ExecStartPre=-/usr/bin/docker stop %n
+ExecStartPre=-/usr/bin/docker rm %n
+ExecStartPre=/usr/bin/docker pull eugencepoi/nsfw_api:latest
+ExecStart=/usr/bin/docker run --rm -p 127.0.0.1:5000:5000/tcp --env PORT=5000 --name %n eugencepoi/nsfw_api:latest
+
+[Install]
+WantedBy=multi-user.target

From a704d5499c03cb5609ea38a5f2ef06095ced3ef3 Mon Sep 17 00:00:00 2001
From: Alex Gleason <alex@alexgleason.me>
Date: Thu, 17 Jun 2021 15:32:42 -0500
Subject: [PATCH 006/192] NsfwApiPolicy: Fall back more generously when
 functions don't match

---
 lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex b/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex
index a1560c584..920821f38 100644
--- a/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex
@@ -92,6 +92,11 @@ defmodule Pleroma.Web.ActivityPub.MRF.NsfwApiPolicy do
     check_url_nsfw(url)
   end
 
+  def check_url_nsfw(url) do
+    threshold = Config.get([@policy, :threshold])
+    {:sfw, %{url: url, score: nil, threshold: threshold}}
+  end
+
   def check_attachment_nsfw(%{"url" => urls} = attachment) when is_list(urls) do
     if Enum.all?(urls, &match?({:sfw, _}, check_url_nsfw(&1))) do
       {:sfw, attachment}
@@ -107,6 +112,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.NsfwApiPolicy do
     end
   end
 
+  def check_attachment_nsfw(attachment), do: {:sfw, attachment}
+
   def check_object_nsfw(%{"attachment" => attachments} = object) when is_list(attachments) do
     if Enum.all?(attachments, &match?({:sfw, _}, check_attachment_nsfw(&1))) do
       {:sfw, object}

From 9423052e9217aa1358950d37c5c96b11d554b37a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= <git@mkljczk.pl>
Date: Mon, 25 Apr 2022 12:39:36 +0200
Subject: [PATCH 007/192] Add "status" notification type
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: marcin mikołajczak <git@mkljczk.pl>
---
 lib/pleroma/notification.ex                   | 38 ++++++++++++--
 .../operations/notification_operation.ex      |  4 +-
 .../controllers/notification_controller.ex    |  1 +
 .../mastodon_api/views/notification_view.ex   |  3 ++
 lib/pleroma/web/push/impl.ex                  |  1 +
 ...00000_add_status_to_notifications_enum.exs | 50 +++++++++++++++++++
 test/pleroma/notification_test.exs            |  1 +
 7 files changed, 92 insertions(+), 6 deletions(-)
 create mode 100644 priv/repo/migrations/20220319000000_add_status_to_notifications_enum.exs

diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex
index 52fd2656b..d142baa8b 100644
--- a/lib/pleroma/notification.ex
+++ b/lib/pleroma/notification.ex
@@ -73,6 +73,7 @@ defmodule Pleroma.Notification do
     pleroma:report
     reblog
     poll
+    status
   }
 
   def changeset(%Notification{} = notification, attrs) do
@@ -397,11 +398,18 @@ defmodule Pleroma.Notification do
     {enabled_receivers, disabled_receivers} = get_notified_from_activity(activity)
     potential_receivers = enabled_receivers ++ disabled_receivers
 
+    {enabled_subscribers, disabled_subscribers} = get_notified_subscribers_from_activity(activity)
+    potential_subscribers = (enabled_subscribers ++ disabled_subscribers) -- potential_receivers
+
     notifications =
-      Enum.map(potential_receivers, fn user ->
-        do_send = do_send && user in enabled_receivers
-        create_notification(activity, user, do_send: do_send)
-      end)
+      (Enum.map(potential_receivers, fn user ->
+         do_send = do_send && user in enabled_receivers
+         create_notification(activity, user, do_send: do_send)
+       end) ++
+         Enum.map(potential_subscribers, fn user ->
+           do_send = do_send && user in enabled_subscribers
+           create_notification(activity, user, do_send: do_send, type: "status")
+         end))
       |> Enum.reject(&is_nil/1)
 
     {:ok, notifications}
@@ -533,6 +541,27 @@ defmodule Pleroma.Notification do
 
   def get_notified_from_activity(_, _local_only), do: {[], []}
 
+  def get_notified_subscribers_from_activity(activity, local_only \\ true)
+
+  def get_notified_subscribers_from_activity(
+        %Activity{data: %{"type" => "Create"}} = activity,
+        local_only
+      ) do
+    notification_enabled_ap_ids =
+      []
+      |> Utils.maybe_notify_subscribers(activity)
+
+    potential_receivers =
+      User.get_users_from_set(notification_enabled_ap_ids, local_only: local_only)
+
+    notification_enabled_users =
+      Enum.filter(potential_receivers, fn u -> u.ap_id in notification_enabled_ap_ids end)
+
+    {notification_enabled_users, potential_receivers -- notification_enabled_users}
+  end
+
+  def get_notified_subscribers_from_activity(_, _), do: {[], []}
+
   # For some activities, only notify the author of the object
   def get_potential_receiver_ap_ids(%{data: %{"type" => type, "object" => object_id}})
       when type in ~w{Like Announce EmojiReact} do
@@ -557,7 +586,6 @@ defmodule Pleroma.Notification do
     []
     |> Utils.maybe_notify_to_recipients(activity)
     |> Utils.maybe_notify_mentioned_recipients(activity)
-    |> Utils.maybe_notify_subscribers(activity)
     |> Utils.maybe_notify_followers(activity)
     |> Enum.uniq()
   end
diff --git a/lib/pleroma/web/api_spec/operations/notification_operation.ex b/lib/pleroma/web/api_spec/operations/notification_operation.ex
index 7f2336ff6..aa965fabb 100644
--- a/lib/pleroma/web/api_spec/operations/notification_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/notification_operation.ex
@@ -196,7 +196,8 @@ defmodule Pleroma.Web.ApiSpec.NotificationOperation do
         "pleroma:report",
         "move",
         "follow_request",
-        "poll"
+        "poll",
+        "status"
       ],
       description: """
       The type of event that resulted in the notification.
@@ -210,6 +211,7 @@ defmodule Pleroma.Web.ApiSpec.NotificationOperation do
       - `pleroma:emoji_reaction` - Someone reacted with emoji to your status
       - `pleroma:chat_mention` - Someone mentioned you in a chat message
       - `pleroma:report` - Someone was reported
+      - `status` - Someone you are subscribed to created a status
       """
     }
   end
diff --git a/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex b/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex
index 932bc6423..9209e8ebd 100644
--- a/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex
+++ b/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex
@@ -51,6 +51,7 @@ defmodule Pleroma.Web.MastodonAPI.NotificationController do
     move
     pleroma:emoji_reaction
     poll
+    status
   }
   def index(%{assigns: %{user: user}} = conn, params) do
     params =
diff --git a/lib/pleroma/web/mastodon_api/views/notification_view.ex b/lib/pleroma/web/mastodon_api/views/notification_view.ex
index 0dc7f3beb..b10b0893c 100644
--- a/lib/pleroma/web/mastodon_api/views/notification_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/notification_view.ex
@@ -103,6 +103,9 @@ defmodule Pleroma.Web.MastodonAPI.NotificationView do
       "mention" ->
         put_status(response, activity, reading_user, status_render_opts)
 
+      "status" ->
+        put_status(response, activity, reading_user, status_render_opts)
+
       "favourite" ->
         put_status(response, parent_activity_fn.(), reading_user, status_render_opts)
 
diff --git a/lib/pleroma/web/push/impl.ex b/lib/pleroma/web/push/impl.ex
index daf3eeb9e..77bc2941d 100644
--- a/lib/pleroma/web/push/impl.ex
+++ b/lib/pleroma/web/push/impl.ex
@@ -183,6 +183,7 @@ defmodule Pleroma.Web.Push.Impl do
   def format_title(%{type: type}, mastodon_type) do
     case mastodon_type || type do
       "mention" -> "New Mention"
+      "status" -> "New Status"
       "follow" -> "New Follower"
       "follow_request" -> "New Follow Request"
       "reblog" -> "New Repeat"
diff --git a/priv/repo/migrations/20220319000000_add_status_to_notifications_enum.exs b/priv/repo/migrations/20220319000000_add_status_to_notifications_enum.exs
new file mode 100644
index 000000000..62c0afb63
--- /dev/null
+++ b/priv/repo/migrations/20220319000000_add_status_to_notifications_enum.exs
@@ -0,0 +1,50 @@
+defmodule Pleroma.Repo.Migrations.AddStatusToNotificationsEnum do
+  use Ecto.Migration
+
+  @disable_ddl_transaction true
+
+  def up do
+    """
+    alter type notification_type add value 'status'
+    """
+    |> execute()
+  end
+
+  def down do
+    alter table(:notifications) do
+      modify(:type, :string)
+    end
+
+    """
+    delete from notifications where type = 'status'
+    """
+    |> execute()
+
+    """
+    drop type if exists notification_type
+    """
+    |> execute()
+
+    """
+    create type notification_type as enum (
+      'follow',
+      'follow_request',
+      'mention',
+      'move',
+      'pleroma:emoji_reaction',
+      'pleroma:chat_mention',
+      'reblog',
+      'favourite',
+      'pleroma:report',
+      'poll
+    )
+    """
+    |> execute()
+
+    """
+    alter table notifications
+    alter column type type notification_type using (type::notification_type)
+    """
+    |> execute()
+  end
+end
diff --git a/test/pleroma/notification_test.exs b/test/pleroma/notification_test.exs
index 805764ea4..eea2fcb67 100644
--- a/test/pleroma/notification_test.exs
+++ b/test/pleroma/notification_test.exs
@@ -104,6 +104,7 @@ defmodule Pleroma.NotificationTest do
       {:ok, [notification]} = Notification.create_notifications(status)
 
       assert notification.user_id == subscriber.id
+      assert notification.type == "status"
     end
 
     test "does not create a notification for subscribed users if status is a reply" do

From fa2a6d5d6b24657ddbda4ef11d2e6dbcb59545d3 Mon Sep 17 00:00:00 2001
From: Claudio Maradonna <penguyman@stronzi.org>
Date: Thu, 7 Apr 2022 18:25:02 +0200
Subject: [PATCH 008/192] feat: simple, but not stupid, uploader for IPFS

fix: format fix with credo
---
 config/config.exs             |  4 +++
 config/description.exs        | 24 +++++++++++++
 config/dev.exs                |  4 +++
 lib/pleroma/upload.ex         | 13 +++++--
 lib/pleroma/uploaders/ipfs.ex | 64 +++++++++++++++++++++++++++++++++++
 5 files changed, 107 insertions(+), 2 deletions(-)
 create mode 100644 lib/pleroma/uploaders/ipfs.ex

diff --git a/config/config.exs b/config/config.exs
index 6a5acda09..7efad0061 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -82,6 +82,10 @@ config :ex_aws, :s3,
   # region: "us-east-1", # may be required for Amazon AWS
   scheme: "https://"
 
+config :pleroma, Pleroma.Uploaders.IPFS,
+  post_gateway_url: nil,
+  get_gateway_url: nil
+
 config :pleroma, :emoji,
   shortcode_globs: ["/emoji/custom/**/*.png"],
   pack_extensions: [".png", ".gif"],
diff --git a/config/description.exs b/config/description.exs
index 7caad18b4..d87bbb9b8 100644
--- a/config/description.exs
+++ b/config/description.exs
@@ -136,6 +136,30 @@ config :pleroma, :config_description, [
       }
     ]
   },
+  %{
+    group: :pleroma,
+    key: Pleroma.Uploaders.IPFS,
+    type: :group,
+    description: "IPFS uploader-related settings",
+    children: [
+      %{
+        key: :get_gateway_url,
+        type: :string,
+        description: "GET Gateway URL",
+        suggestions: [
+          "get_gateway_url"
+        ]
+      },
+      %{
+        key: :post_gateway_url,
+        type: :string,
+        description: "POST Gateway URL",
+        suggestions: [
+          "post_gateway_url"
+        ]
+      }
+    ]
+  },
   %{
     group: :pleroma,
     key: Pleroma.Uploaders.S3,
diff --git a/config/dev.exs b/config/dev.exs
index ab3e83c12..89a84bf05 100644
--- a/config/dev.exs
+++ b/config/dev.exs
@@ -58,6 +58,10 @@ config :pleroma, Pleroma.Web.ApiSpec.CastAndValidate, strict: true
 # https://dashbit.co/blog/speeding-up-re-compilation-of-elixir-projects
 config :phoenix, :plug_init_mode, :runtime
 
+config :pleroma, Pleroma.Uploaders.IPFS,
+  post_gateway_url: nil,
+  get_gateway_url: nil
+
 if File.exists?("./config/dev.secret.exs") do
   import_config "dev.secret.exs"
 else
diff --git a/lib/pleroma/upload.ex b/lib/pleroma/upload.ex
index db2909276..de39bcd6c 100644
--- a/lib/pleroma/upload.ex
+++ b/lib/pleroma/upload.ex
@@ -235,8 +235,14 @@ defmodule Pleroma.Upload do
           ""
         end
 
-    [base_url, path]
-    |> Path.join()
+    uploader = Config.get([Pleroma.Upload, :uploader])
+
+    if uploader == Pleroma.Uploaders.IPFS && String.contains?(base_url, "{CID}") do
+      String.replace(base_url, "{CID}", path)
+    else
+      [base_url, path]
+      |> Path.join()
+    end
   end
 
   defp url_from_spec(_upload, _base_url, {:url, url}), do: url
@@ -273,6 +279,9 @@ defmodule Pleroma.Upload do
           Path.join([upload_base_url, bucket_with_namespace])
         end
 
+      Pleroma.Uploaders.IPFS ->
+        Config.get([Pleroma.Uploaders.IPFS, :get_gateway_url])
+
       _ ->
         public_endpoint || upload_base_url || Pleroma.Web.Endpoint.url() <> "/media/"
     end
diff --git a/lib/pleroma/uploaders/ipfs.ex b/lib/pleroma/uploaders/ipfs.ex
new file mode 100644
index 000000000..b46e9322e
--- /dev/null
+++ b/lib/pleroma/uploaders/ipfs.ex
@@ -0,0 +1,64 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Uploaders.IPFS do
+  @behaviour Pleroma.Uploaders.Uploader
+  require Logger
+
+  alias Pleroma.Config
+  alias Tesla.Multipart
+
+  @impl true
+  def get_file(file) do
+    b_url = Pleroma.Upload.base_url()
+
+    if String.contains?(b_url, "{CID}") do
+      {:ok, {:url, String.replace(b_url, "{CID}", URI.decode(file))}}
+    else
+      {:error, "IPFS Get URL doesn't contain '{CID}' placeholder"}
+    end
+  end
+
+  @impl true
+  def put_file(%Pleroma.Upload{} = upload) do
+    config = Config.get([__MODULE__])
+    post_base_url = Keyword.get(config, :post_gateway_url)
+
+    mp =
+      Multipart.new()
+      |> Multipart.add_content_type_param("charset=utf-8")
+      |> Multipart.add_file(upload.tempfile)
+
+    final_url = Path.join([post_base_url, "/api/v0/add"])
+
+    case Pleroma.HTTP.post(final_url, mp, [], params: ["cid-version": "1"]) do
+      {:ok, ret} ->
+        case Jason.decode(ret.body) do
+          {:ok, ret} ->
+            {:ok, {:file, ret["Hash"]}}
+
+          error ->
+            Logger.error("#{__MODULE__}: #{inspect(error)}")
+            {:error, "JSON decode failed"}
+        end
+
+      error ->
+        Logger.error("#{__MODULE__}: #{inspect(error)}")
+        {:error, "IPFS Gateway Upload failed"}
+    end
+  end
+
+  @impl true
+  def delete_file(file) do
+    config = Config.get([__MODULE__])
+    post_base_url = Keyword.get(config, :post_gateway_url)
+
+    final_url = Path.join([post_base_url, "/api/v0/files/rm"])
+
+    case Pleroma.HTTP.post(final_url, "", [], params: [arg: file]) do
+      {:ok, %{status_code: 204}} -> :ok
+      error -> {:error, inspect(error)}
+    end
+  end
+end

From 43dfa58ebda407a0813d398bee8d0ae3e5c9fd5b Mon Sep 17 00:00:00 2001
From: Claudio Maradonna <penguyman@stronzi.org>
Date: Mon, 11 Apr 2022 15:10:01 +0200
Subject: [PATCH 009/192] added tests for ipfs uploader. adapted changelog.md
 accordingly. improved ipfs uploader with external suggestions

fix lint description.exs
---
 CHANGELOG.md                         |  1 +
 config/description.exs               |  5 +-
 config/dev.exs                       |  4 --
 docs/configuration/cheatsheet.md     | 13 ++++
 lib/pleroma/upload.ex                |  6 +-
 lib/pleroma/uploaders/ipfs.ex        | 14 +++--
 test/pleroma/uploaders/ipfs_test.exs | 88 ++++++++++++++++++++++++++++
 7 files changed, 116 insertions(+), 15 deletions(-)
 create mode 100644 test/pleroma/uploaders/ipfs_test.exs

diff --git a/CHANGELOG.md b/CHANGELOG.md
index f1beb0cd0..b2185b1ba 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -32,6 +32,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - PleromaAPI: Add `GET /api/v1/pleroma/birthdays` API endpoint
 - Make backend-rendered pages translatable. This includes emails. Pages returned as a HTTP response are translated using the language specified in the `userLanguage` cookie, or the `Accept-Language` header. Emails are translated using the `language` field when registering. This language can be changed by `PATCH /api/v1/accounts/update_credentials` with the `language` field.
 - Uploadfilter `Pleroma.Upload.Filter.Exiftool.ReadDescription` returns description values to the FE so they can pre fill the image description field
+- Uploader: Add support for uploading attachments using IPFS
 
 ### Fixed
 - Subscription(Bell) Notifications: Don't create from Pipeline Ingested replies
diff --git a/config/description.exs b/config/description.exs
index d87bbb9b8..1fe5f01f0 100644
--- a/config/description.exs
+++ b/config/description.exs
@@ -147,7 +147,8 @@ config :pleroma, :config_description, [
         type: :string,
         description: "GET Gateway URL",
         suggestions: [
-          "get_gateway_url"
+          "https://ipfs.mydomain.com/<%= cid %>",
+          "https://<%= cid %>.ipfs.mydomain.com/"
         ]
       },
       %{
@@ -155,7 +156,7 @@ config :pleroma, :config_description, [
         type: :string,
         description: "POST Gateway URL",
         suggestions: [
-          "post_gateway_url"
+          "http://localhost:5001/"
         ]
       }
     ]
diff --git a/config/dev.exs b/config/dev.exs
index 89a84bf05..ab3e83c12 100644
--- a/config/dev.exs
+++ b/config/dev.exs
@@ -58,10 +58,6 @@ config :pleroma, Pleroma.Web.ApiSpec.CastAndValidate, strict: true
 # https://dashbit.co/blog/speeding-up-re-compilation-of-elixir-projects
 config :phoenix, :plug_init_mode, :runtime
 
-config :pleroma, Pleroma.Uploaders.IPFS,
-  post_gateway_url: nil,
-  get_gateway_url: nil
-
 if File.exists?("./config/dev.secret.exs") do
   import_config "dev.secret.exs"
 else
diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md
index 74642397b..7e1f9c934 100644
--- a/docs/configuration/cheatsheet.md
+++ b/docs/configuration/cheatsheet.md
@@ -614,6 +614,19 @@ config :ex_aws, :s3,
   host: "s3.eu-central-1.amazonaws.com"
 ```
 
+#### Pleroma.Uploaders.IPFS
+
+* `post_gateway_url`: URL with port of POST Gateway (unauthenticated)
+* `get_gateway_url`: URL of public GET Gateway
+
+Example:
+
+```elixir
+config :pleroma, Pleroma.Uploaders.IPFS,
+  post_gateway_url: "http://localhost:5001",
+  get_gateway_url: "http://<%= cid %>.ipfs.mydomain.com"
+```
+
 ### Upload filters
 
 #### Pleroma.Upload.Filter.AnonymizeFilename
diff --git a/lib/pleroma/upload.ex b/lib/pleroma/upload.ex
index de39bcd6c..b51d23f9e 100644
--- a/lib/pleroma/upload.ex
+++ b/lib/pleroma/upload.ex
@@ -235,10 +235,8 @@ defmodule Pleroma.Upload do
           ""
         end
 
-    uploader = Config.get([Pleroma.Upload, :uploader])
-
-    if uploader == Pleroma.Uploaders.IPFS && String.contains?(base_url, "{CID}") do
-      String.replace(base_url, "{CID}", path)
+    if String.contains?(base_url, "<%= cid %>") do
+      EEx.eval_string(base_url, cid: path)
     else
       [base_url, path]
       |> Path.join()
diff --git a/lib/pleroma/uploaders/ipfs.ex b/lib/pleroma/uploaders/ipfs.ex
index b46e9322e..722c68fa1 100644
--- a/lib/pleroma/uploaders/ipfs.ex
+++ b/lib/pleroma/uploaders/ipfs.ex
@@ -13,10 +13,10 @@ defmodule Pleroma.Uploaders.IPFS do
   def get_file(file) do
     b_url = Pleroma.Upload.base_url()
 
-    if String.contains?(b_url, "{CID}") do
-      {:ok, {:url, String.replace(b_url, "{CID}", URI.decode(file))}}
+    if String.contains?(b_url, "<%= cid %>") do
+      {:ok, {:url, EEx.eval_string(b_url, cid: URI.decode(file))}}
     else
-      {:error, "IPFS Get URL doesn't contain '{CID}' placeholder"}
+      {:error, "IPFS Get URL doesn't contain 'cid' placeholder"}
     end
   end
 
@@ -36,7 +36,11 @@ defmodule Pleroma.Uploaders.IPFS do
       {:ok, ret} ->
         case Jason.decode(ret.body) do
           {:ok, ret} ->
-            {:ok, {:file, ret["Hash"]}}
+            if Map.has_key?(ret, "Hash") do
+              {:ok, {:file, ret["Hash"]}}
+            else
+              {:error, "JSON doesn't contain Hash value"}
+            end
 
           error ->
             Logger.error("#{__MODULE__}: #{inspect(error)}")
@@ -45,7 +49,7 @@ defmodule Pleroma.Uploaders.IPFS do
 
       error ->
         Logger.error("#{__MODULE__}: #{inspect(error)}")
-        {:error, "IPFS Gateway Upload failed"}
+        {:error, "IPFS Gateway upload failed"}
     end
   end
 
diff --git a/test/pleroma/uploaders/ipfs_test.exs b/test/pleroma/uploaders/ipfs_test.exs
new file mode 100644
index 000000000..f9ae046cf
--- /dev/null
+++ b/test/pleroma/uploaders/ipfs_test.exs
@@ -0,0 +1,88 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Uploaders.IPFSTest do
+  use Pleroma.DataCase
+
+  alias Pleroma.Uploaders.IPFS
+  alias Tesla.Multipart
+
+  import Mock
+  import ExUnit.CaptureLog
+
+  setup do
+    clear_config([Pleroma.Upload, :uploader], Pleroma.Uploaders.IPFS)
+    clear_config([Pleroma.Uploaders.IPFS])
+
+    clear_config(
+      [Pleroma.Uploaders.IPFS, :get_gateway_url],
+      "https://<%= cid %>.ipfs.mydomain.com"
+    )
+
+    clear_config([Pleroma.Uploaders.IPFS, :post_gateway_url], "http://localhost:5001")
+  end
+
+  describe "get_file/1" do
+    test "it returns path to ipfs file with cid as subdomain" do
+      assert IPFS.get_file("testcid") == {
+               :ok,
+               {:url, "https://testcid.ipfs.mydomain.com"}
+             }
+    end
+
+    test "it returns path to ipfs file with cid as path" do
+      clear_config(
+        [Pleroma.Uploaders.IPFS, :get_gateway_url],
+        "https://ipfs.mydomain.com/ipfs/<%= cid %>"
+      )
+
+      assert IPFS.get_file("testcid") == {
+               :ok,
+               {:url, "https://ipfs.mydomain.com/ipfs/testcid"}
+             }
+    end
+  end
+
+  describe "put_file/1" do
+    setup do
+      file_upload = %Pleroma.Upload{
+        name: "image-tet.jpg",
+        content_type: "image/jpeg",
+        path: "test_folder/image-tet.jpg",
+        tempfile: Path.absname("test/instance_static/add/shortcode.png")
+      }
+
+      [file_upload: file_upload]
+    end
+
+    test "save file", %{file_upload: file_upload} do
+      with_mock Pleroma.HTTP,
+        post: fn _, _, _, _ ->
+          {:ok,
+           %Tesla.Env{
+             status: 200,
+             body: "{\"Hash\":\"bafybeicrh7ltzx52yxcwrvxxckfmwhqdgsb6qym6dxqm2a4ymsakeshwoi\"}"
+           }}
+        end do
+        assert IPFS.put_file(file_upload) ==
+                 {:ok, {:file, "bafybeicrh7ltzx52yxcwrvxxckfmwhqdgsb6qym6dxqm2a4ymsakeshwoi"}}
+      end
+    end
+
+    test "returns error", %{file_upload: file_upload} do
+      with_mock Pleroma.HTTP, post: fn _, _, _, _ -> {:error, "IPFS Gateway upload failed"} end do
+        assert capture_log(fn ->
+                 assert IPFS.put_file(file_upload) == {:error, "IPFS Gateway upload failed"}
+               end) =~ "Elixir.Pleroma.Uploaders.IPFS: {:error, \"IPFS Gateway upload failed\"}"
+      end
+    end
+  end
+
+  describe "delete_file/1" do
+    test_with_mock "deletes file", Pleroma.HTTP,
+      post: fn _, _, _, _ -> {:ok, %{status_code: 204}} end do
+      assert :ok = IPFS.delete_file("image.jpg")
+    end
+  end
+end

From 44659ecd65fb2251f9130fcecf1732b8931104c1 Mon Sep 17 00:00:00 2001
From: Claudio Maradonna <penguyman@stronzi.org>
Date: Sat, 16 Apr 2022 09:38:49 +0200
Subject: [PATCH 010/192] ipfs: revert to String.replace for cid placeholder

ipfs: fix lint
---
 config/description.exs               | 4 ++--
 docs/configuration/cheatsheet.md     | 2 +-
 lib/pleroma/upload.ex                | 4 ++--
 lib/pleroma/uploaders/ipfs.ex        | 7 +++++--
 test/pleroma/uploaders/ipfs_test.exs | 4 ++--
 5 files changed, 12 insertions(+), 9 deletions(-)

diff --git a/config/description.exs b/config/description.exs
index 1fe5f01f0..b180e7308 100644
--- a/config/description.exs
+++ b/config/description.exs
@@ -147,8 +147,8 @@ config :pleroma, :config_description, [
         type: :string,
         description: "GET Gateway URL",
         suggestions: [
-          "https://ipfs.mydomain.com/<%= cid %>",
-          "https://<%= cid %>.ipfs.mydomain.com/"
+          "https://ipfs.mydomain.com/{CID}",
+          "https://{CID}.ipfs.mydomain.com/"
         ]
       },
       %{
diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md
index 7e1f9c934..d35b33574 100644
--- a/docs/configuration/cheatsheet.md
+++ b/docs/configuration/cheatsheet.md
@@ -624,7 +624,7 @@ Example:
 ```elixir
 config :pleroma, Pleroma.Uploaders.IPFS,
   post_gateway_url: "http://localhost:5001",
-  get_gateway_url: "http://<%= cid %>.ipfs.mydomain.com"
+  get_gateway_url: "http://{CID}.ipfs.mydomain.com"
 ```
 
 ### Upload filters
diff --git a/lib/pleroma/upload.ex b/lib/pleroma/upload.ex
index b51d23f9e..8a01cf613 100644
--- a/lib/pleroma/upload.ex
+++ b/lib/pleroma/upload.ex
@@ -235,8 +235,8 @@ defmodule Pleroma.Upload do
           ""
         end
 
-    if String.contains?(base_url, "<%= cid %>") do
-      EEx.eval_string(base_url, cid: path)
+    if String.contains?(base_url, Pleroma.Uploaders.IPFS.placeholder()) do
+      String.replace(base_url, Pleroma.Uploaders.IPFS.placeholder(), path)
     else
       [base_url, path]
       |> Path.join()
diff --git a/lib/pleroma/uploaders/ipfs.ex b/lib/pleroma/uploaders/ipfs.ex
index 722c68fa1..dde520d8e 100644
--- a/lib/pleroma/uploaders/ipfs.ex
+++ b/lib/pleroma/uploaders/ipfs.ex
@@ -9,12 +9,15 @@ defmodule Pleroma.Uploaders.IPFS do
   alias Pleroma.Config
   alias Tesla.Multipart
 
+  @placeholder "{CID}"
+  def placeholder, do: @placeholder
+
   @impl true
   def get_file(file) do
     b_url = Pleroma.Upload.base_url()
 
-    if String.contains?(b_url, "<%= cid %>") do
-      {:ok, {:url, EEx.eval_string(b_url, cid: URI.decode(file))}}
+    if String.contains?(b_url, @placeholder) do
+      {:ok, {:url, String.replace(b_url, @placeholder, URI.decode(file))}}
     else
       {:error, "IPFS Get URL doesn't contain 'cid' placeholder"}
     end
diff --git a/test/pleroma/uploaders/ipfs_test.exs b/test/pleroma/uploaders/ipfs_test.exs
index f9ae046cf..fc87fa378 100644
--- a/test/pleroma/uploaders/ipfs_test.exs
+++ b/test/pleroma/uploaders/ipfs_test.exs
@@ -17,7 +17,7 @@ defmodule Pleroma.Uploaders.IPFSTest do
 
     clear_config(
       [Pleroma.Uploaders.IPFS, :get_gateway_url],
-      "https://<%= cid %>.ipfs.mydomain.com"
+      "https://{CID}.ipfs.mydomain.com"
     )
 
     clear_config([Pleroma.Uploaders.IPFS, :post_gateway_url], "http://localhost:5001")
@@ -34,7 +34,7 @@ defmodule Pleroma.Uploaders.IPFSTest do
     test "it returns path to ipfs file with cid as path" do
       clear_config(
         [Pleroma.Uploaders.IPFS, :get_gateway_url],
-        "https://ipfs.mydomain.com/ipfs/<%= cid %>"
+        "https://ipfs.mydomain.com/ipfs/{CID}"
       )
 
       assert IPFS.get_file("testcid") == {

From 7c1af86f979ecebcd38995e5278fe2d59a36eda5 Mon Sep 17 00:00:00 2001
From: Claudio Maradonna <penguyman@stronzi.org>
Date: Mon, 9 May 2022 12:15:40 +0200
Subject: [PATCH 011/192] ipfs: refactor final_url generation. add tests for
 final_url

fix lint
---
 lib/pleroma/uploaders/ipfs.ex        | 19 ++++++++++---------
 test/pleroma/uploaders/ipfs_test.exs | 13 ++++++++++++-
 2 files changed, 22 insertions(+), 10 deletions(-)

diff --git a/lib/pleroma/uploaders/ipfs.ex b/lib/pleroma/uploaders/ipfs.ex
index dde520d8e..7a7481d81 100644
--- a/lib/pleroma/uploaders/ipfs.ex
+++ b/lib/pleroma/uploaders/ipfs.ex
@@ -12,6 +12,13 @@ defmodule Pleroma.Uploaders.IPFS do
   @placeholder "{CID}"
   def placeholder, do: @placeholder
 
+  def get_final_url(method) do
+    config = Config.get([__MODULE__])
+    post_base_url = Keyword.get(config, :post_gateway_url)
+
+    Path.join([post_base_url, method])
+  end
+
   @impl true
   def get_file(file) do
     b_url = Pleroma.Upload.base_url()
@@ -25,15 +32,12 @@ defmodule Pleroma.Uploaders.IPFS do
 
   @impl true
   def put_file(%Pleroma.Upload{} = upload) do
-    config = Config.get([__MODULE__])
-    post_base_url = Keyword.get(config, :post_gateway_url)
-
     mp =
       Multipart.new()
       |> Multipart.add_content_type_param("charset=utf-8")
       |> Multipart.add_file(upload.tempfile)
 
-    final_url = Path.join([post_base_url, "/api/v0/add"])
+    final_url = get_final_url("/api/v0/add")
 
     case Pleroma.HTTP.post(final_url, mp, [], params: ["cid-version": "1"]) do
       {:ok, ret} ->
@@ -42,7 +46,7 @@ defmodule Pleroma.Uploaders.IPFS do
             if Map.has_key?(ret, "Hash") do
               {:ok, {:file, ret["Hash"]}}
             else
-              {:error, "JSON doesn't contain Hash value"}
+              {:error, "JSON doesn't contain Hash key"}
             end
 
           error ->
@@ -58,10 +62,7 @@ defmodule Pleroma.Uploaders.IPFS do
 
   @impl true
   def delete_file(file) do
-    config = Config.get([__MODULE__])
-    post_base_url = Keyword.get(config, :post_gateway_url)
-
-    final_url = Path.join([post_base_url, "/api/v0/files/rm"])
+    final_url = get_final_url("/api/v0/files/rm")
 
     case Pleroma.HTTP.post(final_url, "", [], params: [arg: file]) do
       {:ok, %{status_code: 204}} -> :ok
diff --git a/test/pleroma/uploaders/ipfs_test.exs b/test/pleroma/uploaders/ipfs_test.exs
index fc87fa378..d567272d2 100644
--- a/test/pleroma/uploaders/ipfs_test.exs
+++ b/test/pleroma/uploaders/ipfs_test.exs
@@ -23,6 +23,16 @@ defmodule Pleroma.Uploaders.IPFSTest do
     clear_config([Pleroma.Uploaders.IPFS, :post_gateway_url], "http://localhost:5001")
   end
 
+  describe "get_final_url" do
+    test "it returns the final url for put_file" do
+      assert IPFS.get_final_url("/api/v0/add") == "http://localhost:5001/api/v0/add"
+    end
+
+    test "it returns the final url for delete_file" do
+      assert IPFS.get_final_url("/api/v0/files/rm") == "http://localhost:5001/api/v0/files/rm"
+    end
+  end
+
   describe "get_file/1" do
     test "it returns path to ipfs file with cid as subdomain" do
       assert IPFS.get_file("testcid") == {
@@ -62,7 +72,8 @@ defmodule Pleroma.Uploaders.IPFSTest do
           {:ok,
            %Tesla.Env{
              status: 200,
-             body: "{\"Hash\":\"bafybeicrh7ltzx52yxcwrvxxckfmwhqdgsb6qym6dxqm2a4ymsakeshwoi\"}"
+             body:
+               "{\"Name\":\"image-tet.jpg\",\"Size\":\"5000\", \"Hash\":\"bafybeicrh7ltzx52yxcwrvxxckfmwhqdgsb6qym6dxqm2a4ymsakeshwoi\"}"
            }}
         end do
         assert IPFS.put_file(file_upload) ==

From 98f268e5ecc5bab98c98270a582f8b3f0e3be4e8 Mon Sep 17 00:00:00 2001
From: Claudio Maradonna <penguyman@stronzi.org>
Date: Thu, 9 Jun 2022 19:24:13 +0200
Subject: [PATCH 012/192] ipfs: small refactor and more tests

---
 lib/pleroma/uploaders/ipfs.ex        | 24 ++++++++++++++----------
 test/pleroma/uploaders/ipfs_test.exs | 24 ++++++++++++++++++++++--
 2 files changed, 36 insertions(+), 12 deletions(-)

diff --git a/lib/pleroma/uploaders/ipfs.ex b/lib/pleroma/uploaders/ipfs.ex
index 7a7481d81..9f6f26e2e 100644
--- a/lib/pleroma/uploaders/ipfs.ex
+++ b/lib/pleroma/uploaders/ipfs.ex
@@ -9,16 +9,24 @@ defmodule Pleroma.Uploaders.IPFS do
   alias Pleroma.Config
   alias Tesla.Multipart
 
-  @placeholder "{CID}"
-  def placeholder, do: @placeholder
-
-  def get_final_url(method) do
+  defp get_final_url(method) do
     config = Config.get([__MODULE__])
     post_base_url = Keyword.get(config, :post_gateway_url)
 
     Path.join([post_base_url, method])
   end
 
+  def put_file_endpoint() do
+    get_final_url("/api/v0/add")
+  end
+
+  def delete_file_endpoint() do
+    get_final_url("/api/v0/files/rm")
+  end
+
+  @placeholder "{CID}"
+  def placeholder, do: @placeholder
+
   @impl true
   def get_file(file) do
     b_url = Pleroma.Upload.base_url()
@@ -37,9 +45,7 @@ defmodule Pleroma.Uploaders.IPFS do
       |> Multipart.add_content_type_param("charset=utf-8")
       |> Multipart.add_file(upload.tempfile)
 
-    final_url = get_final_url("/api/v0/add")
-
-    case Pleroma.HTTP.post(final_url, mp, [], params: ["cid-version": "1"]) do
+    case Pleroma.HTTP.post(put_file_endpoint(), mp, [], params: ["cid-version": "1"]) do
       {:ok, ret} ->
         case Jason.decode(ret.body) do
           {:ok, ret} ->
@@ -62,9 +68,7 @@ defmodule Pleroma.Uploaders.IPFS do
 
   @impl true
   def delete_file(file) do
-    final_url = get_final_url("/api/v0/files/rm")
-
-    case Pleroma.HTTP.post(final_url, "", [], params: [arg: file]) do
+    case Pleroma.HTTP.post(delete_file_endpoint(), "", [], params: [arg: file]) do
       {:ok, %{status_code: 204}} -> :ok
       error -> {:error, inspect(error)}
     end
diff --git a/test/pleroma/uploaders/ipfs_test.exs b/test/pleroma/uploaders/ipfs_test.exs
index d567272d2..f2b880d9f 100644
--- a/test/pleroma/uploaders/ipfs_test.exs
+++ b/test/pleroma/uploaders/ipfs_test.exs
@@ -25,11 +25,11 @@ defmodule Pleroma.Uploaders.IPFSTest do
 
   describe "get_final_url" do
     test "it returns the final url for put_file" do
-      assert IPFS.get_final_url("/api/v0/add") == "http://localhost:5001/api/v0/add"
+      assert IPFS.put_file_endpoint() == "http://localhost:5001/api/v0/add"
     end
 
     test "it returns the final url for delete_file" do
-      assert IPFS.get_final_url("/api/v0/files/rm") == "http://localhost:5001/api/v0/files/rm"
+      assert IPFS.delete_file_endpoint() == "http://localhost:5001/api/v0/files/rm"
     end
   end
 
@@ -88,6 +88,26 @@ defmodule Pleroma.Uploaders.IPFSTest do
                end) =~ "Elixir.Pleroma.Uploaders.IPFS: {:error, \"IPFS Gateway upload failed\"}"
       end
     end
+
+    test "returns error if JSON decode fails", %{file_upload: file_upload} do
+      with_mocks([
+        {Pleroma.HTTP, [], [post: fn _, _, _, _ -> {:ok, %Tesla.Env{status: 200, body: ''}} end]},
+        {Jason, [], [decode: fn _ -> {:error, "JSON decode failed"} end]}
+      ]) do
+        assert capture_log(fn ->
+                 assert IPFS.put_file(file_upload) == {:error, "JSON decode failed"}
+               end) =~ "Elixir.Pleroma.Uploaders.IPFS: {:error, \"JSON decode failed\"}"
+      end
+    end
+
+    test "returns error if JSON body doesn't contain Hash key", %{file_upload: file_upload} do
+      with_mocks([
+        {Pleroma.HTTP, [], [post: fn _, _, _, _ -> {:ok, %Tesla.Env{status: 200, body: ''}} end]},
+        {Jason, [], [decode: fn _ -> {:ok, %{}} end]}
+      ]) do
+        assert IPFS.put_file(file_upload) == {:error, "JSON doesn't contain Hash key"}
+      end
+    end
   end
 
   describe "delete_file/1" do

From 254f2ea85400ebd692fc4a45f5ac22fedd49ec09 Mon Sep 17 00:00:00 2001
From: Claudio Maradonna <penguyman@stronzi.org>
Date: Thu, 9 Jun 2022 23:38:50 +0200
Subject: [PATCH 013/192] ipfs: remove unused alias

fix analysis job
---
 lib/pleroma/uploaders/ipfs.ex        | 4 ++--
 test/pleroma/uploaders/ipfs_test.exs | 1 -
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/lib/pleroma/uploaders/ipfs.ex b/lib/pleroma/uploaders/ipfs.ex
index 9f6f26e2e..32e06c5cf 100644
--- a/lib/pleroma/uploaders/ipfs.ex
+++ b/lib/pleroma/uploaders/ipfs.ex
@@ -16,11 +16,11 @@ defmodule Pleroma.Uploaders.IPFS do
     Path.join([post_base_url, method])
   end
 
-  def put_file_endpoint() do
+  def put_file_endpoint do
     get_final_url("/api/v0/add")
   end
 
-  def delete_file_endpoint() do
+  def delete_file_endpoint do
     get_final_url("/api/v0/files/rm")
   end
 
diff --git a/test/pleroma/uploaders/ipfs_test.exs b/test/pleroma/uploaders/ipfs_test.exs
index f2b880d9f..5edb6266b 100644
--- a/test/pleroma/uploaders/ipfs_test.exs
+++ b/test/pleroma/uploaders/ipfs_test.exs
@@ -6,7 +6,6 @@ defmodule Pleroma.Uploaders.IPFSTest do
   use Pleroma.DataCase
 
   alias Pleroma.Uploaders.IPFS
-  alias Tesla.Multipart
 
   import Mock
   import ExUnit.CaptureLog

From 5e097eb91def0efd3cd0008309fd524fcfd88e15 Mon Sep 17 00:00:00 2001
From: Claudio Maradonna <penguyman@stronzi.org>
Date: Tue, 28 Jun 2022 17:53:44 +0200
Subject: [PATCH 014/192] ipfs: better tests with @ilja suggestions

---
 test/pleroma/uploaders/ipfs_test.exs | 38 ++++++++++++++++++----------
 1 file changed, 25 insertions(+), 13 deletions(-)

diff --git a/test/pleroma/uploaders/ipfs_test.exs b/test/pleroma/uploaders/ipfs_test.exs
index 5edb6266b..9df3f239c 100644
--- a/test/pleroma/uploaders/ipfs_test.exs
+++ b/test/pleroma/uploaders/ipfs_test.exs
@@ -6,6 +6,7 @@ defmodule Pleroma.Uploaders.IPFSTest do
   use Pleroma.DataCase
 
   alias Pleroma.Uploaders.IPFS
+  alias Tesla.Multipart
 
   import Mock
   import ExUnit.CaptureLog
@@ -62,12 +63,17 @@ defmodule Pleroma.Uploaders.IPFSTest do
         tempfile: Path.absname("test/instance_static/add/shortcode.png")
       }
 
-      [file_upload: file_upload]
+      mp =
+        Multipart.new()
+        |> Multipart.add_content_type_param("charset=utf-8")
+        |> Multipart.add_file(file_upload.tempfile)
+
+      [file_upload: file_upload, mp: mp]
     end
 
     test "save file", %{file_upload: file_upload} do
       with_mock Pleroma.HTTP,
-        post: fn _, _, _, _ ->
+        post: fn "http://localhost:5001/api/v0/add", mp, [], params: ["cid-version": "1"] ->
           {:ok,
            %Tesla.Env{
              status: 200,
@@ -81,7 +87,10 @@ defmodule Pleroma.Uploaders.IPFSTest do
     end
 
     test "returns error", %{file_upload: file_upload} do
-      with_mock Pleroma.HTTP, post: fn _, _, _, _ -> {:error, "IPFS Gateway upload failed"} end do
+      with_mock Pleroma.HTTP,
+        post: fn "http://localhost:5001/api/v0/add", mp, [], params: ["cid-version": "1"] ->
+          {:error, "IPFS Gateway upload failed"}
+        end do
         assert capture_log(fn ->
                  assert IPFS.put_file(file_upload) == {:error, "IPFS Gateway upload failed"}
                end) =~ "Elixir.Pleroma.Uploaders.IPFS: {:error, \"IPFS Gateway upload failed\"}"
@@ -89,21 +98,22 @@ defmodule Pleroma.Uploaders.IPFSTest do
     end
 
     test "returns error if JSON decode fails", %{file_upload: file_upload} do
-      with_mocks([
-        {Pleroma.HTTP, [], [post: fn _, _, _, _ -> {:ok, %Tesla.Env{status: 200, body: ''}} end]},
-        {Jason, [], [decode: fn _ -> {:error, "JSON decode failed"} end]}
-      ]) do
+      with_mock Pleroma.HTTP, [],
+        post: fn "http://localhost:5001/api/v0/add", mp, [], params: ["cid-version": "1"] ->
+          {:ok, %Tesla.Env{status: 200, body: 'invalid'}}
+        end do
         assert capture_log(fn ->
                  assert IPFS.put_file(file_upload) == {:error, "JSON decode failed"}
-               end) =~ "Elixir.Pleroma.Uploaders.IPFS: {:error, \"JSON decode failed\"}"
+               end) =~
+                 "Elixir.Pleroma.Uploaders.IPFS: {:error, %Jason.DecodeError{data: \"invalid\", position: 0, token: nil}}"
       end
     end
 
     test "returns error if JSON body doesn't contain Hash key", %{file_upload: file_upload} do
-      with_mocks([
-        {Pleroma.HTTP, [], [post: fn _, _, _, _ -> {:ok, %Tesla.Env{status: 200, body: ''}} end]},
-        {Jason, [], [decode: fn _ -> {:ok, %{}} end]}
-      ]) do
+      with_mock Pleroma.HTTP, [],
+        post: fn "http://localhost:5001/api/v0/add", mp, [], params: ["cid-version": "1"] ->
+          {:ok, %Tesla.Env{status: 200, body: '{"key": "value"}'}}
+        end do
         assert IPFS.put_file(file_upload) == {:error, "JSON doesn't contain Hash key"}
       end
     end
@@ -111,7 +121,9 @@ defmodule Pleroma.Uploaders.IPFSTest do
 
   describe "delete_file/1" do
     test_with_mock "deletes file", Pleroma.HTTP,
-      post: fn _, _, _, _ -> {:ok, %{status_code: 204}} end do
+      post: fn "http://localhost:5001/api/v0/files/rm", "", [], params: [arg: "image.jpg"] ->
+        {:ok, %{status_code: 204}}
+      end do
       assert :ok = IPFS.delete_file("image.jpg")
     end
   end

From 21d9091f5e422493ff69fe59db9c965e0d511369 Mon Sep 17 00:00:00 2001
From: Claudio Maradonna <penguyman@stronzi.org>
Date: Fri, 8 Jul 2022 10:06:46 +0200
Subject: [PATCH 015/192] ipfs: replacing single quotes with double quotes

---
 test/pleroma/uploaders/ipfs_test.exs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/pleroma/uploaders/ipfs_test.exs b/test/pleroma/uploaders/ipfs_test.exs
index 9df3f239c..853d185e5 100644
--- a/test/pleroma/uploaders/ipfs_test.exs
+++ b/test/pleroma/uploaders/ipfs_test.exs
@@ -100,7 +100,7 @@ defmodule Pleroma.Uploaders.IPFSTest do
     test "returns error if JSON decode fails", %{file_upload: file_upload} do
       with_mock Pleroma.HTTP, [],
         post: fn "http://localhost:5001/api/v0/add", mp, [], params: ["cid-version": "1"] ->
-          {:ok, %Tesla.Env{status: 200, body: 'invalid'}}
+          {:ok, %Tesla.Env{status: 200, body: "invalid"}}
         end do
         assert capture_log(fn ->
                  assert IPFS.put_file(file_upload) == {:error, "JSON decode failed"}
@@ -112,7 +112,7 @@ defmodule Pleroma.Uploaders.IPFSTest do
     test "returns error if JSON body doesn't contain Hash key", %{file_upload: file_upload} do
       with_mock Pleroma.HTTP, [],
         post: fn "http://localhost:5001/api/v0/add", mp, [], params: ["cid-version": "1"] ->
-          {:ok, %Tesla.Env{status: 200, body: '{"key": "value"}'}}
+          {:ok, %Tesla.Env{status: 200, body: "{\"key\": \"value\"}"}}
         end do
         assert IPFS.put_file(file_upload) == {:error, "JSON doesn't contain Hash key"}
       end

From 3ed39e310939d90ddbad7bd7ffa1ebd8aca6e74c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= <git@mkljczk.pl>
Date: Fri, 8 Jul 2022 21:28:23 +0200
Subject: [PATCH 016/192] Add test
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: marcin mikołajczak <git@mkljczk.pl>
---
 test/pleroma/notification_test.exs | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/test/pleroma/notification_test.exs b/test/pleroma/notification_test.exs
index eea2fcb67..c43502eb5 100644
--- a/test/pleroma/notification_test.exs
+++ b/test/pleroma/notification_test.exs
@@ -128,6 +128,21 @@ defmodule Pleroma.NotificationTest do
       subscriber_notifications = Notification.for_user(subscriber)
       assert Enum.empty?(subscriber_notifications)
     end
+
+    test "does not create subscriber notification if mentioned" do
+      user = insert(:user)
+      subscriber = insert(:user)
+
+      User.subscribe(subscriber, user)
+
+      {:ok, status} = CommonAPI.post(user, %{status: "mentioning @#{subscriber.nickname}"})
+      {:ok, [notification] = notifications} = Notification.create_notifications(status)
+
+      assert length(notifications) == 1
+
+      assert notification.user_id == subscriber.id
+      assert notification.type == "mention"
+    end
   end
 
   test "create_poll_notifications/1" do

From c899af1d6acad1895240a0247e9b91eca5db08df Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= <git@mkljczk.pl>
Date: Thu, 14 Apr 2022 20:09:43 +0200
Subject: [PATCH 017/192] Reject requests from specified instances if
 `authorized_fetch_mode` is enabled
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: marcin mikołajczak <git@mkljczk.pl>
---
 config/config.exs                             |  1 +
 config/description.exs                        | 12 ++++
 docs/configuration/cheatsheet.md              |  1 +
 lib/pleroma/signature.ex                      | 16 +++--
 .../web/mastodon_api/views/instance_view.ex   |  7 +++
 lib/pleroma/web/plugs/http_signature_plug.ex  | 40 ++++++++++++
 test/pleroma/signature_test.exs               |  8 +++
 .../web/plugs/http_signature_plug_test.exs    | 63 +++++++++++++++++--
 8 files changed, 140 insertions(+), 8 deletions(-)

diff --git a/config/config.exs b/config/config.exs
index 0fc959807..cfa16f766 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -216,6 +216,7 @@ config :pleroma, :instance,
   allow_relay: true,
   public: true,
   quarantined_instances: [],
+  rejected_instances: [],
   static_dir: "instance/static/",
   allowed_post_formats: [
     "text/plain",
diff --git a/config/description.exs b/config/description.exs
index b29348edf..a75f40929 100644
--- a/config/description.exs
+++ b/config/description.exs
@@ -714,6 +714,18 @@ config :pleroma, :config_description, [
           {"*.quarantined.com", "Reason"}
         ]
       },
+      %{
+        key: :rejected_instances,
+        type: {:list, :tuple},
+        key_placeholder: "instance",
+        value_placeholder: "reason",
+        description:
+          "List of ActivityPub instances to reject requests from if authorized_fetch_mode is enabled",
+        suggestions: [
+          {"rejected.com", "Reason"},
+          {"*.rejected.com", "Reason"}
+        ]
+      },
       %{
         key: :static_dir,
         type: :string,
diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md
index 6e13b9622..84a5bdb98 100644
--- a/docs/configuration/cheatsheet.md
+++ b/docs/configuration/cheatsheet.md
@@ -41,6 +41,7 @@ To add configuration to your config file, you can copy it from the base config.
 * `allow_relay`: Permits remote instances to subscribe to all public posts of your instance. This may increase the visibility of your instance.
 * `public`: Makes the client API in authenticated mode-only except for user-profiles. Useful for disabling the Local Timeline and The Whole Known Network. Note that there is a dependent setting restricting or allowing unauthenticated access to specific resources, see `restrict_unauthenticated` for more details.
 * `quarantined_instances`: ActivityPub instances where private (DMs, followers-only) activities will not be send.
+* `rejected_instances`: ActivityPub instances to reject requests from if authorized_fetch_mode is enabled.
 * `allowed_post_formats`: MIME-type list of formats allowed to be posted (transformed into HTML).
 * `extended_nickname_format`: Set to `true` to use extended local nicknames format (allows underscores/dashes). This will break federation with
     older software for theses nicknames.
diff --git a/lib/pleroma/signature.ex b/lib/pleroma/signature.ex
index dbe6fd209..d5ba5c4fb 100644
--- a/lib/pleroma/signature.ex
+++ b/lib/pleroma/signature.ex
@@ -37,8 +37,7 @@ defmodule Pleroma.Signature do
   end
 
   def fetch_public_key(conn) do
-    with %{"keyId" => kid} <- HTTPSignatures.signature_for_conn(conn),
-         {:ok, actor_id} <- key_id_to_actor_id(kid),
+    with {:ok, actor_id} <- get_actor_id(conn),
          {:ok, public_key} <- User.get_public_key_for_ap_id(actor_id) do
       {:ok, public_key}
     else
@@ -48,8 +47,7 @@ defmodule Pleroma.Signature do
   end
 
   def refetch_public_key(conn) do
-    with %{"keyId" => kid} <- HTTPSignatures.signature_for_conn(conn),
-         {:ok, actor_id} <- key_id_to_actor_id(kid),
+    with {:ok, actor_id} <- get_actor_id(conn),
          {:ok, _user} <- ActivityPub.make_user_from_ap_id(actor_id),
          {:ok, public_key} <- User.get_public_key_for_ap_id(actor_id) do
       {:ok, public_key}
@@ -59,6 +57,16 @@ defmodule Pleroma.Signature do
     end
   end
 
+  def get_actor_id(conn) do
+    with %{"keyId" => kid} <- HTTPSignatures.signature_for_conn(conn),
+         {:ok, actor_id} <- key_id_to_actor_id(kid) do
+      {:ok, actor_id}
+    else
+      e ->
+        {:error, e}
+    end
+  end
+
   def sign(%User{} = user, headers) do
     with {:ok, %{keys: keys}} <- User.ensure_keys_present(user),
          {:ok, private_key, _} <- Keys.keys_from_pem(keys) do
diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex
index 62931bd41..017bd62e2 100644
--- a/lib/pleroma/web/mastodon_api/views/instance_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex
@@ -105,6 +105,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do
 
   def federation do
     quarantined = Config.get([:instance, :quarantined_instances], [])
+    rejected = Config.get([:instance, :rejected_instances], [])
 
     if Config.get([:mrf, :transparency]) do
       {:ok, data} = MRF.describe()
@@ -124,6 +125,12 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do
           |> Enum.map(fn {instance, reason} -> {instance, %{"reason" => reason}} end)
           |> Map.new()
       })
+      |> Map.put(
+        :rejected_instances,
+        rejected
+        |> Enum.map(fn {instance, reason} -> {instance, %{"reason" => reason}} end)
+        |> Map.new()
+      )
     else
       %{}
     end
diff --git a/lib/pleroma/web/plugs/http_signature_plug.ex b/lib/pleroma/web/plugs/http_signature_plug.ex
index d023754a6..cf80b9b14 100644
--- a/lib/pleroma/web/plugs/http_signature_plug.ex
+++ b/lib/pleroma/web/plugs/http_signature_plug.ex
@@ -5,6 +5,10 @@
 defmodule Pleroma.Web.Plugs.HTTPSignaturePlug do
   import Plug.Conn
   import Phoenix.Controller, only: [get_format: 1, text: 2]
+
+  alias Pleroma.Config
+  alias Pleroma.Web.ActivityPub.MRF
+
   require Logger
 
   def init(options) do
@@ -19,7 +23,9 @@ defmodule Pleroma.Web.Plugs.HTTPSignaturePlug do
     if get_format(conn) == "activity+json" do
       conn
       |> maybe_assign_valid_signature()
+      |> maybe_assign_actor_id()
       |> maybe_require_signature()
+      |> maybe_filter_requests()
     else
       conn
     end
@@ -46,6 +52,16 @@ defmodule Pleroma.Web.Plugs.HTTPSignaturePlug do
     end
   end
 
+  defp maybe_assign_actor_id(%{assigns: %{valid_signature: true}} = conn) do
+    adapter = Application.get_env(:http_signatures, :adapter)
+
+    {:ok, actor_id} = adapter.get_actor_id(conn)
+
+    assign(conn, :actor_id, actor_id)
+  end
+
+  defp maybe_assign_actor_id(conn), do: conn
+
   defp has_signature_header?(conn) do
     conn |> get_req_header("signature") |> Enum.at(0, false)
   end
@@ -62,4 +78,28 @@ defmodule Pleroma.Web.Plugs.HTTPSignaturePlug do
       conn
     end
   end
+
+  defp maybe_filter_requests(%{halted: true} = conn), do: conn
+
+  defp maybe_filter_requests(conn) do
+    if Pleroma.Config.get([:activitypub, :authorized_fetch_mode], false) do
+      %{host: host} = URI.parse(conn.assigns.actor_id)
+
+      if MRF.subdomain_match?(rejected_domains(), host) do
+        conn
+        |> put_status(:unauthorized)
+        |> halt()
+      else
+        conn
+      end
+    else
+      conn
+    end
+  end
+
+  defp rejected_domains do
+    Config.get([:instance, :rejected_instances])
+    |> Pleroma.Web.ActivityPub.MRF.instance_list_from_tuples()
+    |> Pleroma.Web.ActivityPub.MRF.subdomains_regex()
+  end
 end
diff --git a/test/pleroma/signature_test.exs b/test/pleroma/signature_test.exs
index 92d05f26c..8f94efdc3 100644
--- a/test/pleroma/signature_test.exs
+++ b/test/pleroma/signature_test.exs
@@ -70,6 +70,14 @@ defmodule Pleroma.SignatureTest do
     end
   end
 
+  describe "get_actor_id/1" do
+    test "it returns actor id" do
+      ap_id = "https://mastodon.social/users/lambadalambda"
+
+      assert Signature.get_actor_id(make_fake_conn(ap_id)) == {:ok, ap_id}
+    end
+  end
+
   describe "sign/2" do
     test "it returns signature headers" do
       user =
diff --git a/test/pleroma/web/plugs/http_signature_plug_test.exs b/test/pleroma/web/plugs/http_signature_plug_test.exs
index 2d8fba3cd..de68e8823 100644
--- a/test/pleroma/web/plugs/http_signature_plug_test.exs
+++ b/test/pleroma/web/plugs/http_signature_plug_test.exs
@@ -10,11 +10,15 @@ defmodule Pleroma.Web.Plugs.HTTPSignaturePlugTest do
   import Phoenix.Controller, only: [put_format: 2]
   import Mock
 
-  test "it call HTTPSignatures to check validity if the actor sighed it" do
+  test "it call HTTPSignatures to check validity if the actor signed it" do
     params = %{"actor" => "http://mastodon.example.org/users/admin"}
     conn = build_conn(:get, "/doesntmattter", params)
 
-    with_mock HTTPSignatures, validate_conn: fn _ -> true end do
+    with_mock HTTPSignatures,
+      validate_conn: fn _ -> true end,
+      signature_for_conn: fn _ ->
+        %{"keyId" => "http://mastodon.example.org/users/admin#main-key"}
+      end do
       conn =
         conn
         |> put_req_header(
@@ -41,7 +45,11 @@ defmodule Pleroma.Web.Plugs.HTTPSignaturePlugTest do
     end
 
     test "when signature header is present", %{conn: conn} do
-      with_mock HTTPSignatures, validate_conn: fn _ -> false end do
+      with_mock HTTPSignatures,
+        validate_conn: fn _ -> false end,
+        signature_for_conn: fn _ ->
+          %{"keyId" => "http://mastodon.example.org/users/admin#main-key"}
+        end do
         conn =
           conn
           |> put_req_header(
@@ -58,7 +66,11 @@ defmodule Pleroma.Web.Plugs.HTTPSignaturePlugTest do
         assert called(HTTPSignatures.validate_conn(:_))
       end
 
-      with_mock HTTPSignatures, validate_conn: fn _ -> true end do
+      with_mock HTTPSignatures,
+        validate_conn: fn _ -> true end,
+        signature_for_conn: fn _ ->
+          %{"keyId" => "http://mastodon.example.org/users/admin#main-key"}
+        end do
         conn =
           conn
           |> put_req_header(
@@ -82,4 +94,47 @@ defmodule Pleroma.Web.Plugs.HTTPSignaturePlugTest do
       assert conn.resp_body == "Request not signed"
     end
   end
+
+  test "rejects requests from `rejected_instances` when `authorized_fetch_mode` is enabled" do
+    clear_config([:activitypub, :authorized_fetch_mode], true)
+    clear_config([:instance, :rejected_instances], [{"mastodon.example.org", "no reason"}])
+
+    with_mock HTTPSignatures,
+      validate_conn: fn _ -> true end,
+      signature_for_conn: fn _ ->
+        %{"keyId" => "http://mastodon.example.org/users/admin#main-key"}
+      end do
+      conn =
+        build_conn(:get, "/doesntmattter", %{"actor" => "http://mastodon.example.org/users/admin"})
+        |> put_req_header(
+          "signature",
+          "keyId=\"http://mastodon.example.org/users/admin#main-key"
+        )
+        |> put_format("activity+json")
+        |> HTTPSignaturePlug.call(%{})
+
+      assert conn.assigns.valid_signature == true
+      assert conn.halted == true
+      assert called(HTTPSignatures.validate_conn(:_))
+    end
+
+    with_mock HTTPSignatures,
+      validate_conn: fn _ -> true end,
+      signature_for_conn: fn _ ->
+        %{"keyId" => "http://allowed.example.org/users/admin#main-key"}
+      end do
+      conn =
+        build_conn(:get, "/doesntmattter", %{"actor" => "http://allowed.example.org/users/admin"})
+        |> put_req_header(
+          "signature",
+          "keyId=\"http://allowed.example.org/users/admin#main-key"
+        )
+        |> put_format("activity+json")
+        |> HTTPSignaturePlug.call(%{})
+
+      assert conn.assigns.valid_signature == true
+      assert conn.halted == false
+      assert called(HTTPSignatures.validate_conn(:_))
+    end
+  end
 end

From 78d1105bffee7ece8a2b972d3cb58a6e41d86828 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= <git@mkljczk.pl>
Date: Sun, 19 Feb 2023 22:02:38 +0100
Subject: [PATCH 018/192] Fix down migration
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: marcin mikołajczak <git@mkljczk.pl>
---
 .../20220319000000_add_status_to_notifications_enum.exs        | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/priv/repo/migrations/20220319000000_add_status_to_notifications_enum.exs b/priv/repo/migrations/20220319000000_add_status_to_notifications_enum.exs
index 62c0afb63..c3bc85894 100644
--- a/priv/repo/migrations/20220319000000_add_status_to_notifications_enum.exs
+++ b/priv/repo/migrations/20220319000000_add_status_to_notifications_enum.exs
@@ -36,7 +36,8 @@ defmodule Pleroma.Repo.Migrations.AddStatusToNotificationsEnum do
       'reblog',
       'favourite',
       'pleroma:report',
-      'poll
+      'poll',
+      'update'
     )
     """
     |> execute()

From 2ae1b802f260e9ad8eaa585907d9505545ceb872 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Thu, 9 Mar 2023 10:21:11 +0100
Subject: [PATCH 019/192] AttachmentValidator: Add support for Honk "summary" +
 "name"

As used by Honk and supported by Mastodon
---
 .../object_validators/attachment_validator.ex         |  3 ++-
 .../object_validators/attachment_validator_test.exs   | 11 +++++++----
 2 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex b/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex
index 398020bff..766421e60 100644
--- a/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex
@@ -15,6 +15,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator do
     field(:type, :string)
     field(:mediaType, ObjectValidators.MIME, default: "application/octet-stream")
     field(:name, :string)
+    field(:summary, :string)
     field(:blurhash, :string)
 
     embeds_many :url, UrlObjectValidator, primary_key: false do
@@ -44,7 +45,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator do
       |> fix_url()
 
     struct
-    |> cast(data, [:id, :type, :mediaType, :name, :blurhash])
+    |> cast(data, [:id, :type, :mediaType, :name, :summary, :blurhash])
     |> cast_embed(:url, with: &url_changeset/2, required: true)
     |> validate_inclusion(:type, ~w[Link Document Audio Image Video])
     |> validate_required([:type, :mediaType])
diff --git a/test/pleroma/web/activity_pub/object_validators/attachment_validator_test.exs b/test/pleroma/web/activity_pub/object_validators/attachment_validator_test.exs
index 77f2044e9..8d561603c 100644
--- a/test/pleroma/web/activity_pub/object_validators/attachment_validator_test.exs
+++ b/test/pleroma/web/activity_pub/object_validators/attachment_validator_test.exs
@@ -25,19 +25,22 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidatorTest do
     end
 
     test "works with honkerific attachments" do
-      attachment = %{
+      honk = %{
         "mediaType" => "",
-        "name" => "",
-        "summary" => "298p3RG7j27tfsZ9RQ.jpg",
+        "summary" => "Select your spirit chonk",
+        "name" => "298p3RG7j27tfsZ9RQ.jpg",
         "type" => "Document",
         "url" => "https://honk.tedunangst.com/d/298p3RG7j27tfsZ9RQ.jpg"
       }
 
       assert {:ok, attachment} =
-               AttachmentValidator.cast_and_validate(attachment)
+               honk
+               |> AttachmentValidator.cast_and_validate()
                |> Ecto.Changeset.apply_action(:insert)
 
       assert attachment.mediaType == "application/octet-stream"
+      assert attachment.summary == "Select your spirit chonk"
+      assert attachment.name == "298p3RG7j27tfsZ9RQ.jpg"
     end
 
     test "works with an unknown but valid mime type" do

From 197647a04e66c1af3ae691a4507612fdbee9c48c Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Thu, 9 Mar 2023 10:35:57 +0100
Subject: [PATCH 020/192] MastoAPI Attachment: Use "summary" for descriptions
 if present

---
 .../web/api_spec/schemas/attachment.ex        |   6 +-
 .../web/mastodon_api/views/status_view.ex     |  17 ++-
 .../mastodon_api/views/status_view_test.exs   | 101 ++++++++++++------
 3 files changed, 87 insertions(+), 37 deletions(-)

diff --git a/lib/pleroma/web/api_spec/schemas/attachment.ex b/lib/pleroma/web/api_spec/schemas/attachment.ex
index 48634a14f..e89f2ddd0 100644
--- a/lib/pleroma/web/api_spec/schemas/attachment.ex
+++ b/lib/pleroma/web/api_spec/schemas/attachment.ex
@@ -50,7 +50,11 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Attachment do
       pleroma: %Schema{
         type: :object,
         properties: %{
-          mime_type: %Schema{type: :string, description: "mime type of the attachment"}
+          mime_type: %Schema{type: :string, description: "mime type of the attachment"},
+          name: %Schema{
+            type: :string,
+            description: "Name of the attachment, typically the filename"
+          }
         }
       }
     },
diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex
index 0a8c98b44..7a3af8acb 100644
--- a/lib/pleroma/web/mastodon_api/views/status_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/status_view.ex
@@ -571,6 +571,19 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
           to_string(attachment["id"] || hash_id)
       end
 
+    description =
+      if attachment["summary"] do
+        HTML.strip_tags(attachment["summary"])
+      else
+        attachment["name"]
+      end
+
+    name = if attachment["summary"], do: attachment["name"]
+
+    pleroma =
+      %{mime_type: media_type}
+      |> Maps.put_if_present(:name, name)
+
     %{
       id: attachment_id,
       url: href,
@@ -578,8 +591,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
       preview_url: href_preview,
       text_url: href,
       type: type,
-      description: attachment["name"],
-      pleroma: %{mime_type: media_type},
+      description: description,
+      pleroma: pleroma,
       blurhash: attachment["blurhash"]
     }
     |> Maps.put_if_present(:meta, meta)
diff --git a/test/pleroma/web/mastodon_api/views/status_view_test.exs b/test/pleroma/web/mastodon_api/views/status_view_test.exs
index f76b115b7..4580da75b 100644
--- a/test/pleroma/web/mastodon_api/views/status_view_test.exs
+++ b/test/pleroma/web/mastodon_api/views/status_view_test.exs
@@ -456,45 +456,78 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do
     assert mention.url == recipient.ap_id
   end
 
-  test "attachments" do
-    object = %{
-      "type" => "Image",
-      "url" => [
-        %{
-          "mediaType" => "image/png",
-          "href" => "someurl",
-          "width" => 200,
-          "height" => 100
-        }
-      ],
-      "blurhash" => "UJJ8X[xYW,%Jtq%NNFbXB5j]IVM|9GV=WHRn",
-      "uuid" => 6
-    }
+  describe "attachments" do
+    test "Complete Mastodon style" do
+      object = %{
+        "type" => "Image",
+        "url" => [
+          %{
+            "mediaType" => "image/png",
+            "href" => "someurl",
+            "width" => 200,
+            "height" => 100
+          }
+        ],
+        "blurhash" => "UJJ8X[xYW,%Jtq%NNFbXB5j]IVM|9GV=WHRn",
+        "uuid" => 6
+      }
 
-    expected = %{
-      id: "1638338801",
-      type: "image",
-      url: "someurl",
-      remote_url: "someurl",
-      preview_url: "someurl",
-      text_url: "someurl",
-      description: nil,
-      pleroma: %{mime_type: "image/png"},
-      meta: %{original: %{width: 200, height: 100, aspect: 2}},
-      blurhash: "UJJ8X[xYW,%Jtq%NNFbXB5j]IVM|9GV=WHRn"
-    }
+      expected = %{
+        id: "1638338801",
+        type: "image",
+        url: "someurl",
+        remote_url: "someurl",
+        preview_url: "someurl",
+        text_url: "someurl",
+        description: nil,
+        pleroma: %{mime_type: "image/png"},
+        meta: %{original: %{width: 200, height: 100, aspect: 2}},
+        blurhash: "UJJ8X[xYW,%Jtq%NNFbXB5j]IVM|9GV=WHRn"
+      }
 
-    api_spec = Pleroma.Web.ApiSpec.spec()
+      api_spec = Pleroma.Web.ApiSpec.spec()
 
-    assert expected == StatusView.render("attachment.json", %{attachment: object})
-    assert_schema(expected, "Attachment", api_spec)
+      assert expected == StatusView.render("attachment.json", %{attachment: object})
+      assert_schema(expected, "Attachment", api_spec)
 
-    # If theres a "id", use that instead of the generated one
-    object = Map.put(object, "id", 2)
-    result = StatusView.render("attachment.json", %{attachment: object})
+      # If theres a "id", use that instead of the generated one
+      object = Map.put(object, "id", 2)
+      result = StatusView.render("attachment.json", %{attachment: object})
 
-    assert %{id: "2"} = result
-    assert_schema(result, "Attachment", api_spec)
+      assert %{id: "2"} = result
+      assert_schema(result, "Attachment", api_spec)
+    end
+
+    test "Honkerific" do
+      object = %{
+        "type" => "Image",
+        "url" => [
+          %{
+            "mediaType" => "image/png",
+            "href" => "someurl"
+          }
+        ],
+        "name" => "fool.jpeg",
+        "summary" => "they have played us for absolute fools."
+      }
+
+      expected = %{
+        blurhash: nil,
+        description: "they have played us for absolute fools.",
+        id: "1638338801",
+        pleroma: %{mime_type: "image/png", name: "fool.jpeg"},
+        preview_url: "someurl",
+        remote_url: "someurl",
+        text_url: "someurl",
+        type: "image",
+        url: "someurl"
+      }
+
+      api_spec = Pleroma.Web.ApiSpec.spec()
+
+      assert expected == StatusView.render("attachment.json", %{attachment: object})
+      assert_schema(expected, "Attachment", api_spec)
+    end
   end
 
   test "put the url advertised in the Activity in to the url attribute" do

From 9363ef53a34c9d96191bccaece76dd4e01f493b6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= <git@mkljczk.pl>
Date: Sun, 14 May 2023 15:02:58 +0200
Subject: [PATCH 021/192] Add test for 'status' notification type for
 NotificationView
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: marcin mikołajczak <git@mkljczk.pl>
---
 .../views/notification_view_test.exs          | 27 +++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/test/pleroma/web/mastodon_api/views/notification_view_test.exs b/test/pleroma/web/mastodon_api/views/notification_view_test.exs
index 6ea894691..92de6c6a7 100644
--- a/test/pleroma/web/mastodon_api/views/notification_view_test.exs
+++ b/test/pleroma/web/mastodon_api/views/notification_view_test.exs
@@ -286,4 +286,31 @@ defmodule Pleroma.Web.MastodonAPI.NotificationViewTest do
 
     test_notifications_rendering([notification], user, [expected])
   end
+
+  test "Subscribed status notification" do
+    user = insert(:user)
+    subscriber = insert(:user)
+
+    User.subscribe(subscriber, user)
+
+    {:ok, activity} = CommonAPI.post(user, %{status: "hi"})
+    {:ok, [notification]} = Notification.create_notifications(activity)
+
+    user = User.get_cached_by_id(user.id)
+
+    expected = %{
+      id: to_string(notification.id),
+      pleroma: %{is_seen: false, is_muted: false},
+      type: "status",
+      account:
+        AccountView.render("show.json", %{
+          user: user,
+          for: subscriber
+        }),
+      status: StatusView.render("show.json", %{activity: activity, for: subscriber}),
+      created_at: Utils.to_masto_date(notification.inserted_at)
+    }
+
+    test_notifications_rendering([notification], subscriber, [expected])
+  end
 end

From 1ab4ab8d38687634735e1415f395b072718ab1ab Mon Sep 17 00:00:00 2001
From: tusooa <tusooa@kazv.moe>
Date: Tue, 18 Jul 2023 18:24:30 -0400
Subject: [PATCH 022/192] Extract translatable strings

---
 changelog.d/3907.skip                |  0
 priv/gettext/config_descriptions.pot | 84 ++++++++++++++++++++++++++++
 priv/gettext/errors.pot              | 36 +++++++++---
 priv/gettext/oauth_scopes.pot        | 40 +++++++++++++
 4 files changed, 152 insertions(+), 8 deletions(-)
 create mode 100644 changelog.d/3907.skip

diff --git a/changelog.d/3907.skip b/changelog.d/3907.skip
new file mode 100644
index 000000000..e69de29bb
diff --git a/priv/gettext/config_descriptions.pot b/priv/gettext/config_descriptions.pot
index 4f60e1c85..b4792868b 100644
--- a/priv/gettext/config_descriptions.pot
+++ b/priv/gettext/config_descriptions.pot
@@ -5973,3 +5973,87 @@ msgstr ""
 msgctxt "config label at :pleroma-:instance > :languages"
 msgid "Languages"
 msgstr ""
+
+#: lib/pleroma/docs/translator.ex:5
+#, elixir-autogen, elixir-format
+msgctxt "config description at :pleroma-:mrf_emoji"
+msgid "Reject or force-unlisted emojis whose URLs or names match a keyword or [Regex](https://hexdocs.pm/elixir/Regex.html)."
+msgstr ""
+
+#: lib/pleroma/docs/translator.ex:5
+#, elixir-autogen, elixir-format
+msgctxt "config description at :pleroma-:mrf_emoji > :federated_timeline_removal_shortcode"
+msgid "  A list of patterns which result in message with emojis whose shortcodes match being removed from federated timelines (a.k.a unlisted). This will apply only to statuses.\n\n  Each pattern can be a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`.\n"
+msgstr ""
+
+#: lib/pleroma/docs/translator.ex:5
+#, elixir-autogen, elixir-format
+msgctxt "config description at :pleroma-:mrf_emoji > :federated_timeline_removal_url"
+msgid "  A list of patterns which result in message with emojis whose URLs match being removed from federated timelines (a.k.a unlisted). This will apply only to statuses.\n\n  Each pattern can be a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`.\n"
+msgstr ""
+
+#: lib/pleroma/docs/translator.ex:5
+#, elixir-autogen, elixir-format
+msgctxt "config description at :pleroma-:mrf_emoji > :remove_shortcode"
+msgid "  A list of patterns which result in emoji whose shortcode matches being removed from the message. This will apply to statuses, emoji reactions, and user profiles.\n\n  Each pattern can be a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`.\n"
+msgstr ""
+
+#: lib/pleroma/docs/translator.ex:5
+#, elixir-autogen, elixir-format
+msgctxt "config description at :pleroma-:mrf_emoji > :remove_url"
+msgid "  A list of patterns which result in emoji whose URL matches being removed from the message. This will apply to statuses, emoji reactions, and user profiles.\n\n  Each pattern can be a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`.\n"
+msgstr ""
+
+#: lib/pleroma/docs/translator.ex:5
+#, elixir-autogen, elixir-format
+msgctxt "config description at :pleroma-Pleroma.User.Backup > :process_chunk_size"
+msgid "The number of activities to fetch in the backup job for each chunk."
+msgstr ""
+
+#: lib/pleroma/docs/translator.ex:5
+#, elixir-autogen, elixir-format
+msgctxt "config description at :pleroma-Pleroma.User.Backup > :process_wait_time"
+msgid "The amount of time to wait for backup to report progress, in milliseconds. If no progress is received from the backup job for that much time, terminate it and deem it failed."
+msgstr ""
+
+#: lib/pleroma/docs/translator.ex:5
+#, elixir-autogen, elixir-format
+msgctxt "config label at :pleroma-:mrf_emoji"
+msgid "MRF Emoji"
+msgstr ""
+
+#: lib/pleroma/docs/translator.ex:5
+#, elixir-autogen, elixir-format
+msgctxt "config label at :pleroma-:mrf_emoji > :federated_timeline_removal_shortcode"
+msgid "Federated timeline removal shortcode"
+msgstr ""
+
+#: lib/pleroma/docs/translator.ex:5
+#, elixir-autogen, elixir-format
+msgctxt "config label at :pleroma-:mrf_emoji > :federated_timeline_removal_url"
+msgid "Federated timeline removal url"
+msgstr ""
+
+#: lib/pleroma/docs/translator.ex:5
+#, elixir-autogen, elixir-format
+msgctxt "config label at :pleroma-:mrf_emoji > :remove_shortcode"
+msgid "Remove shortcode"
+msgstr ""
+
+#: lib/pleroma/docs/translator.ex:5
+#, elixir-autogen, elixir-format
+msgctxt "config label at :pleroma-:mrf_emoji > :remove_url"
+msgid "Remove url"
+msgstr ""
+
+#: lib/pleroma/docs/translator.ex:5
+#, elixir-autogen, elixir-format
+msgctxt "config label at :pleroma-Pleroma.User.Backup > :process_chunk_size"
+msgid "Process Chunk Size"
+msgstr ""
+
+#: lib/pleroma/docs/translator.ex:5
+#, elixir-autogen, elixir-format
+msgctxt "config label at :pleroma-Pleroma.User.Backup > :process_wait_time"
+msgid "Process Wait Time"
+msgstr ""
diff --git a/priv/gettext/errors.pot b/priv/gettext/errors.pot
index d320ee1bd..aca77f8fa 100644
--- a/priv/gettext/errors.pot
+++ b/priv/gettext/errors.pot
@@ -110,7 +110,7 @@ msgstr ""
 msgid "Can't display this activity"
 msgstr ""
 
-#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:334
+#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:346
 #, elixir-autogen, elixir-format
 msgid "Can't find user"
 msgstr ""
@@ -198,7 +198,7 @@ msgstr ""
 msgid "Invalid password."
 msgstr ""
 
-#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:267
+#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:279
 #, elixir-autogen, elixir-format
 msgid "Invalid request"
 msgstr ""
@@ -225,7 +225,7 @@ msgstr ""
 #: lib/pleroma/web/feed/tag_controller.ex:16
 #: lib/pleroma/web/feed/user_controller.ex:69
 #: lib/pleroma/web/o_status/o_status_controller.ex:132
-#: lib/pleroma/web/plugs/uploaded_media.ex:104
+#: lib/pleroma/web/plugs/uploaded_media.ex:84
 #, elixir-autogen, elixir-format
 msgid "Not found"
 msgstr ""
@@ -235,7 +235,7 @@ msgstr ""
 msgid "Poll's author can't vote"
 msgstr ""
 
-#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:499
+#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:511
 #: lib/pleroma/web/mastodon_api/controllers/fallback_controller.ex:20
 #: lib/pleroma/web/mastodon_api/controllers/poll_controller.ex:39
 #: lib/pleroma/web/mastodon_api/controllers/poll_controller.ex:51
@@ -341,7 +341,7 @@ msgstr ""
 msgid "CAPTCHA expired"
 msgstr ""
 
-#: lib/pleroma/web/plugs/uploaded_media.ex:77
+#: lib/pleroma/web/plugs/uploaded_media.ex:57
 #, elixir-autogen, elixir-format
 msgid "Failed"
 msgstr ""
@@ -361,7 +361,7 @@ msgstr ""
 msgid "Insufficient permissions: %{permissions}."
 msgstr ""
 
-#: lib/pleroma/web/plugs/uploaded_media.ex:131
+#: lib/pleroma/web/plugs/uploaded_media.ex:111
 #, elixir-autogen, elixir-format
 msgid "Internal Error"
 msgstr ""
@@ -557,7 +557,7 @@ msgstr ""
 msgid "Access denied"
 msgstr ""
 
-#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:331
+#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:343
 #, elixir-autogen, elixir-format
 msgid "This API requires an authenticated user"
 msgstr ""
@@ -567,7 +567,7 @@ msgstr ""
 msgid "User is not an admin."
 msgstr ""
 
-#: lib/pleroma/user/backup.ex:73
+#: lib/pleroma/user/backup.ex:78
 #, elixir-format
 msgid "Last export was less than a day ago"
 msgid_plural "Last export was less than %{days} days ago"
@@ -607,3 +607,23 @@ msgstr ""
 #, elixir-autogen, elixir-format
 msgid "User isn't privileged."
 msgstr ""
+
+#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:267
+#, elixir-autogen, elixir-format
+msgid "Bio is too long"
+msgstr ""
+
+#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:270
+#, elixir-autogen, elixir-format
+msgid "Name is too long"
+msgstr ""
+
+#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:273
+#, elixir-autogen, elixir-format
+msgid "One or more field entries are too long"
+msgstr ""
+
+#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:276
+#, elixir-autogen, elixir-format
+msgid "Too many field entries"
+msgstr ""
diff --git a/priv/gettext/oauth_scopes.pot b/priv/gettext/oauth_scopes.pot
index 50ad0dd9e..83328770e 100644
--- a/priv/gettext/oauth_scopes.pot
+++ b/priv/gettext/oauth_scopes.pot
@@ -219,3 +219,43 @@ msgstr ""
 #, elixir-autogen, elixir-format
 msgid "read:mutes"
 msgstr ""
+
+#: lib/pleroma/web/api_spec/scopes/translator.ex:5
+#, elixir-autogen, elixir-format
+msgid "push"
+msgstr ""
+
+#: lib/pleroma/web/api_spec/scopes/translator.ex:5
+#, elixir-autogen, elixir-format
+msgid "read:backups"
+msgstr ""
+
+#: lib/pleroma/web/api_spec/scopes/translator.ex:5
+#, elixir-autogen, elixir-format
+msgid "read:chats"
+msgstr ""
+
+#: lib/pleroma/web/api_spec/scopes/translator.ex:5
+#, elixir-autogen, elixir-format
+msgid "read:media"
+msgstr ""
+
+#: lib/pleroma/web/api_spec/scopes/translator.ex:5
+#, elixir-autogen, elixir-format
+msgid "read:reports"
+msgstr ""
+
+#: lib/pleroma/web/api_spec/scopes/translator.ex:5
+#, elixir-autogen, elixir-format
+msgid "write:chats"
+msgstr ""
+
+#: lib/pleroma/web/api_spec/scopes/translator.ex:5
+#, elixir-autogen, elixir-format
+msgid "write:follow"
+msgstr ""
+
+#: lib/pleroma/web/api_spec/scopes/translator.ex:5
+#, elixir-autogen, elixir-format
+msgid "write:reports"
+msgstr ""

From b6a9d87f16a4806eab7a6da874d6f75b65d4f214 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= <git@mkljczk.pl>
Date: Wed, 15 Mar 2023 19:44:42 +0100
Subject: [PATCH 023/192] Display reposted replies with exclude_replies: true
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: marcin mikołajczak <git@mkljczk.pl>
---
 changelog.d/show-reposter-replies.add        | 1 +
 lib/pleroma/web/activity_pub/activity_pub.ex | 5 +++--
 2 files changed, 4 insertions(+), 2 deletions(-)
 create mode 100644 changelog.d/show-reposter-replies.add

diff --git a/changelog.d/show-reposter-replies.add b/changelog.d/show-reposter-replies.add
new file mode 100644
index 000000000..3b852ec3b
--- /dev/null
+++ b/changelog.d/show-reposter-replies.add
@@ -0,0 +1 @@
+Display reposted replies with exclude_replies: true
\ No newline at end of file
diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index 3979d418e..4b956c680 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -964,8 +964,9 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   defp restrict_replies(query, %{exclude_replies: true}) do
     from(
-      [_activity, object] in query,
-      where: fragment("?->>'inReplyTo' is null", object.data)
+      [activity, object] in query,
+      where:
+        fragment("?->>'inReplyTo' is null or ?->>'type' = 'Announce'", object.data, activity.data)
     )
   end
 

From f271ea6e432d685c113582e5944d79e12c153016 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Sat, 16 Dec 2023 18:22:32 +0100
Subject: [PATCH 024/192] Move Plugs.RemoteIP.maybe_add_cidr/1 to
 InetHelper.parse_cidr/1

---
 lib/pleroma/helpers/inet_helper.ex | 11 +++++++++++
 lib/pleroma/web/plugs/remote_ip.ex | 14 ++------------
 2 files changed, 13 insertions(+), 12 deletions(-)

diff --git a/lib/pleroma/helpers/inet_helper.ex b/lib/pleroma/helpers/inet_helper.ex
index 704d37f8a..3500fc679 100644
--- a/lib/pleroma/helpers/inet_helper.ex
+++ b/lib/pleroma/helpers/inet_helper.ex
@@ -16,4 +16,15 @@ defmodule Pleroma.Helpers.InetHelper do
   def parse_address(ip) do
     :inet.parse_address(ip)
   end
+
+  def parse_cidr(proxy) when is_binary(proxy) do
+    proxy =
+      cond do
+        "/" in String.codepoints(proxy) -> proxy
+        InetCidr.v4?(InetCidr.parse_address!(proxy)) -> proxy <> "/32"
+        InetCidr.v6?(InetCidr.parse_address!(proxy)) -> proxy <> "/128"
+      end
+
+    InetCidr.parse(proxy, true)
+  end
 end
diff --git a/lib/pleroma/web/plugs/remote_ip.ex b/lib/pleroma/web/plugs/remote_ip.ex
index f207d9fef..3a4bffb50 100644
--- a/lib/pleroma/web/plugs/remote_ip.ex
+++ b/lib/pleroma/web/plugs/remote_ip.ex
@@ -8,6 +8,7 @@ defmodule Pleroma.Web.Plugs.RemoteIp do
   """
 
   alias Pleroma.Config
+  alias Pleroma.Helpers.InetHelper
   import Plug.Conn
 
   @behaviour Plug
@@ -30,19 +31,8 @@ defmodule Pleroma.Web.Plugs.RemoteIp do
     proxies =
       Config.get([__MODULE__, :proxies], [])
       |> Enum.concat(reserved)
-      |> Enum.map(&maybe_add_cidr/1)
+      |> Enum.map(&InetHelper.parse_cidr/1)
 
     {headers, proxies}
   end
-
-  defp maybe_add_cidr(proxy) when is_binary(proxy) do
-    proxy =
-      cond do
-        "/" in String.codepoints(proxy) -> proxy
-        InetCidr.v4?(InetCidr.parse_address!(proxy)) -> proxy <> "/32"
-        InetCidr.v6?(InetCidr.parse_address!(proxy)) -> proxy <> "/128"
-      end
-
-    InetCidr.parse(proxy, true)
-  end
 end

From 086ba59d0346be870dc7df2660fbb55666bf0af7 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Sat, 16 Dec 2023 18:56:46 +0100
Subject: [PATCH 025/192] HTTPSignaturePlug: Add
 :authorized_fetch_mode_exceptions

---
 changelog.d/auth-fetch-exception.add          |  1 +
 config/description.exs                        |  6 ++++++
 docs/configuration/cheatsheet.md              |  1 +
 lib/pleroma/web/plugs/http_signature_plug.ex  | 20 ++++++++++++++-----
 .../web/plugs/http_signature_plug_test.exs    | 19 ++++++++++++++++++
 5 files changed, 42 insertions(+), 5 deletions(-)
 create mode 100644 changelog.d/auth-fetch-exception.add

diff --git a/changelog.d/auth-fetch-exception.add b/changelog.d/auth-fetch-exception.add
new file mode 100644
index 000000000..98efb903e
--- /dev/null
+++ b/changelog.d/auth-fetch-exception.add
@@ -0,0 +1 @@
+HTTPSignaturePlug: Add :authorized_fetch_mode_exceptions configuration
\ No newline at end of file
diff --git a/config/description.exs b/config/description.exs
index b152981c4..2fed1a152 100644
--- a/config/description.exs
+++ b/config/description.exs
@@ -1771,6 +1771,12 @@ config :pleroma, :config_description, [
         type: :boolean,
         description: "Require HTTP signatures for AP fetches"
       },
+      %{
+        key: :authorized_fetch_mode_exceptions,
+        type: {:list, :string},
+        description:
+          "List of IPs (CIDR format accepted) to exempt from HTTP Signatures requirement (for example to allow debugging, you shouldn't otherwise need this)"
+      },
       %{
         key: :note_replies_output_limit,
         type: :integer,
diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md
index a4cae4dbb..06933ba76 100644
--- a/docs/configuration/cheatsheet.md
+++ b/docs/configuration/cheatsheet.md
@@ -279,6 +279,7 @@ Notes:
 * `deny_follow_blocked`: Whether to disallow following an account that has blocked the user in question
 * `sign_object_fetches`: Sign object fetches with HTTP signatures
 * `authorized_fetch_mode`: Require HTTP signatures for AP fetches
+* `authorized_fetch_mode_exceptions`: List of IPs (CIDR format accepted) to exempt from HTTP Signatures requirement (for example to allow debugging, you shouldn't otherwise need this)
 
 ## Pleroma.User
 
diff --git a/lib/pleroma/web/plugs/http_signature_plug.ex b/lib/pleroma/web/plugs/http_signature_plug.ex
index e814efc2c..7ec202662 100644
--- a/lib/pleroma/web/plugs/http_signature_plug.ex
+++ b/lib/pleroma/web/plugs/http_signature_plug.ex
@@ -3,6 +3,8 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.Plugs.HTTPSignaturePlug do
+  alias Pleroma.Helpers.InetHelper
+
   import Plug.Conn
   import Phoenix.Controller, only: [get_format: 1, text: 2]
   require Logger
@@ -89,12 +91,20 @@ defmodule Pleroma.Web.Plugs.HTTPSignaturePlug do
 
   defp maybe_require_signature(%{assigns: %{valid_signature: true}} = conn), do: conn
 
-  defp maybe_require_signature(conn) do
+  defp maybe_require_signature(%{remote_ip: remote_ip} = conn) do
     if Pleroma.Config.get([:activitypub, :authorized_fetch_mode], false) do
-      conn
-      |> put_status(:unauthorized)
-      |> text("Request not signed")
-      |> halt()
+      exceptions =
+        Pleroma.Config.get([:activitypub, :authorized_fetch_mode_exceptions], [])
+        |> Enum.map(&InetHelper.parse_cidr/1)
+
+      if Enum.any?(exceptions, fn x -> InetCidr.contains?(x, remote_ip) end) do
+        conn
+      else
+        conn
+        |> put_status(:unauthorized)
+        |> text("Request not signed")
+        |> halt()
+      end
     else
       conn
     end
diff --git a/test/pleroma/web/plugs/http_signature_plug_test.exs b/test/pleroma/web/plugs/http_signature_plug_test.exs
index 2d8fba3cd..deb7e4a23 100644
--- a/test/pleroma/web/plugs/http_signature_plug_test.exs
+++ b/test/pleroma/web/plugs/http_signature_plug_test.exs
@@ -81,5 +81,24 @@ defmodule Pleroma.Web.Plugs.HTTPSignaturePlugTest do
       assert conn.state == :sent
       assert conn.resp_body == "Request not signed"
     end
+
+    test "exempts specific IPs from `authorized_fetch_mode_exceptions`", %{conn: conn} do
+      clear_config([:activitypub, :authorized_fetch_mode_exceptions], ["192.168.0.0/24"])
+
+      with_mock HTTPSignatures, validate_conn: fn _ -> false end do
+        conn =
+          conn
+          |> Map.put(:remote_ip, {192, 168, 0, 1})
+          |> put_req_header(
+            "signature",
+            "keyId=\"http://mastodon.example.org/users/admin#main-key"
+          )
+          |> HTTPSignaturePlug.call(%{})
+
+        assert conn.remote_ip == {192, 168, 0, 1}
+        assert conn.halted == false
+        assert called(HTTPSignatures.validate_conn(:_))
+      end
+    end
   end
 end

From 7e3bbdded5dea73a0bad3a8905839e42d476e506 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Wed, 20 Dec 2023 23:39:12 +0000
Subject: [PATCH 026/192] Elixir 1.13 is the minimum required version

---
 .gitlab-ci.yml                 | 2 +-
 Dockerfile                     | 6 +++---
 changelog.d/bump-elixir.change | 1 +
 mix.exs                        | 2 +-
 4 files changed, 6 insertions(+), 5 deletions(-)
 create mode 100644 changelog.d/bump-elixir.change

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index eb31a8086..8f10790da 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -2,7 +2,7 @@ image: git.pleroma.social:5050/pleroma/pleroma/ci-base
 
 variables: &global_variables
   # Only used for the release
-  ELIXIR_VER: 1.12.3
+  ELIXIR_VER: 1.13.4
   POSTGRES_DB: pleroma_test
   POSTGRES_USER: postgres
   POSTGRES_PASSWORD: postgres
diff --git a/Dockerfile b/Dockerfile
index 69c3509de..72461305c 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,7 +1,7 @@
 ARG ELIXIR_IMG=hexpm/elixir
-ARG ELIXIR_VER=1.12.3
-ARG ERLANG_VER=24.2.1
-ARG ALPINE_VER=3.17.0
+ARG ELIXIR_VER=1.13.4
+ARG ERLANG_VER=24.3.4.15
+ARG ALPINE_VER=3.17.5
 
 FROM ${ELIXIR_IMG}:${ELIXIR_VER}-erlang-${ERLANG_VER}-alpine-${ALPINE_VER} as build
 
diff --git a/changelog.d/bump-elixir.change b/changelog.d/bump-elixir.change
new file mode 100644
index 000000000..afb25d4e7
--- /dev/null
+++ b/changelog.d/bump-elixir.change
@@ -0,0 +1 @@
+Elixir 1.13 is the minimum required version.
diff --git a/mix.exs b/mix.exs
index b4b77b161..2056e591d 100644
--- a/mix.exs
+++ b/mix.exs
@@ -5,7 +5,7 @@ defmodule Pleroma.Mixfile do
     [
       app: :pleroma,
       version: version("2.6.51"),
-      elixir: "~> 1.11",
+      elixir: "~> 1.13",
       elixirc_paths: elixirc_paths(Mix.env()),
       compilers: Mix.compilers(),
       elixirc_options: [warnings_as_errors: warnings_as_errors()],

From ddb9e90c405369496fdf9e6dfed593eff8d5dc5c Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Thu, 28 Dec 2023 15:59:25 -0500
Subject: [PATCH 027/192] Update minimum elixir version found in various docs

---
 docs/installation/debian_based_jp.md           | 2 +-
 docs/installation/generic_dependencies.include | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/installation/debian_based_jp.md b/docs/installation/debian_based_jp.md
index 1424ad7f4..502eefaf8 100644
--- a/docs/installation/debian_based_jp.md
+++ b/docs/installation/debian_based_jp.md
@@ -14,7 +14,7 @@ Note: This article is potentially outdated because at this time we may not have
 
 - PostgreSQL 9.6以上 (Ubuntu16.04では9.5しか提供されていないので，[](https://www.postgresql.org/download/linux/ubuntu/)こちらから新しいバージョンを入手してください)
 - `postgresql-contrib` 9.6以上 (同上)
-- Elixir 1.8 以上 ([Debianのリポジトリからインストールしないこと！！！ ここからインストールすること！](https://elixir-lang.org/install.html#unix-and-unix-like)。または [asdf](https://github.com/asdf-vm/asdf) をpleromaユーザーでインストールしてください)
+- Elixir 1.13 以上 ([Debianのリポジトリからインストールしないこと！！！ ここからインストールすること！](https://elixir-lang.org/install.html#unix-and-unix-like)。または [asdf](https://github.com/asdf-vm/asdf) をpleromaユーザーでインストールしてください)
 - `erlang-dev`
 - `erlang-nox`
 - `git`
diff --git a/docs/installation/generic_dependencies.include b/docs/installation/generic_dependencies.include
index 3365a36a8..e0cfd3264 100644
--- a/docs/installation/generic_dependencies.include
+++ b/docs/installation/generic_dependencies.include
@@ -1,7 +1,7 @@
 ## Required dependencies
 
 * PostgreSQL >=9.6
-* Elixir >=1.11.0 <1.15
+* Elixir >=1.13.0 <1.15
 * Erlang OTP >=22.2.0 <26
 * git
 * file / libmagic

From 1ed8ae2d8e86ed26d4e21f59e95995795bcb282b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= <git@mkljczk.pl>
Date: Wed, 31 Jan 2024 22:55:58 +0100
Subject: [PATCH 028/192] Add changelog
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: marcin mikołajczak <git@mkljczk.pl>
---
 changelog.d/status-notification-type.add | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/status-notification-type.add

diff --git a/changelog.d/status-notification-type.add b/changelog.d/status-notification-type.add
new file mode 100644
index 000000000..a6e94fa87
--- /dev/null
+++ b/changelog.d/status-notification-type.add
@@ -0,0 +1 @@
+Add "status" notification type
\ No newline at end of file

From ac977bdb1c58fac826f6325a3b1550ff389439ca Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Tue, 20 Feb 2024 08:45:48 +0100
Subject: [PATCH 029/192] StealEmojiPolicy: Sanitize shortcodes

Closes: https://git.pleroma.social/pleroma/pleroma/-/issues/3245
---
 .../activity_pub/mrf/steal_emoji_policy.ex    |  2 ++
 .../mrf/steal_emoji_policy_test.exs           | 26 +++++++++++++++++++
 2 files changed, 28 insertions(+)

diff --git a/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex b/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex
index f66c379b5..12accfadd 100644
--- a/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex
@@ -34,6 +34,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.StealEmojiPolicy do
           |> Path.basename()
           |> Path.extname()
 
+        shortcode = Path.basename(shortcode)
         file_path = Path.join(emoji_dir_path, shortcode <> (extension || ".png"))
 
         case File.write(file_path, response.body) do
@@ -76,6 +77,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.StealEmojiPolicy do
       new_emojis =
         foreign_emojis
         |> Enum.reject(fn {shortcode, _url} -> shortcode in installed_emoji end)
+        |> Enum.reject(fn {shortcode, _url} -> String.contains?(shortcode, ["/", "\\"]) end)
         |> Enum.filter(fn {shortcode, _url} ->
           reject_emoji? =
             [:mrf_steal_emoji, :rejected_shortcodes]
diff --git a/test/pleroma/web/activity_pub/mrf/steal_emoji_policy_test.exs b/test/pleroma/web/activity_pub/mrf/steal_emoji_policy_test.exs
index 89d32352f..e7fb337ec 100644
--- a/test/pleroma/web/activity_pub/mrf/steal_emoji_policy_test.exs
+++ b/test/pleroma/web/activity_pub/mrf/steal_emoji_policy_test.exs
@@ -60,6 +60,32 @@ defmodule Pleroma.Web.ActivityPub.MRF.StealEmojiPolicyTest do
            |> File.exists?()
   end
 
+  test "rejects invalid shortcodes", %{path: path} do
+    message = %{
+      "type" => "Create",
+      "object" => %{
+        "emoji" => [{"fired/fox", "https://example.org/emoji/firedfox"}],
+        "actor" => "https://example.org/users/admin"
+      }
+    }
+
+    fullpath = Path.join(path, "fired/fox.png")
+
+    Tesla.Mock.mock(fn %{method: :get, url: "https://example.org/emoji/firedfox"} ->
+      %Tesla.Env{status: 200, body: File.read!("test/fixtures/image.jpg")}
+    end)
+
+    clear_config(:mrf_steal_emoji, hosts: ["example.org"], size_limit: 284_468)
+
+    refute "firedfox" in installed()
+    refute File.exists?(path)
+
+    assert {:ok, _message} = StealEmojiPolicy.filter(message)
+
+    refute "fired/fox" in installed()
+    refute File.exists?(fullpath)
+  end
+
   test "reject regex shortcode", %{message: message} do
     refute "firedfox" in installed()
 

From be075a43363519505dcfe2dba1fbb19e0326b668 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Tue, 20 Feb 2024 09:16:36 +0100
Subject: [PATCH 030/192] Security release 2.6.2

---
 CHANGELOG.md | 5 +++++
 mix.exs      | 2 +-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 83b3065ce..92e5e6134 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
+## 2.6.2
+
+### Security
+- MRF StealEmojiPolicy: Sanitize shortcodes (thanks to Hazel K for the report
+
 ## 2.6.1
 ### Changed
 - - Document maximum supported version of Erlang & Elixir
diff --git a/mix.exs b/mix.exs
index d420c11e4..c95c2a82f 100644
--- a/mix.exs
+++ b/mix.exs
@@ -4,7 +4,7 @@ defmodule Pleroma.Mixfile do
   def project do
     [
       app: :pleroma,
-      version: version("2.6.1"),
+      version: version("2.6.2"),
       elixir: "~> 1.11",
       elixirc_paths: elixirc_paths(Mix.env()),
       compilers: [:phoenix] ++ Mix.compilers(),

From 7dfd148ff8a4f2d349d6d6f92d788effdaab36f3 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Sat, 9 Dec 2023 18:32:26 -0500
Subject: [PATCH 031/192] Logger metadata for inbound federation requests

---
 config/config.exs                                       | 4 ++--
 lib/pleroma/web/activity_pub/activity_pub_controller.ex | 8 ++++++++
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/config/config.exs b/config/config.exs
index 32c8509be..537517688 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -131,13 +131,13 @@ config :pleroma, Pleroma.Web.Endpoint,
 config :logger, :console,
   level: :debug,
   format: "\n$time $metadata[$level] $message\n",
-  metadata: [:request_id]
+  metadata: [:actor, :request_id, :type]
 
 config :logger, :ex_syslogger,
   level: :debug,
   ident: "pleroma",
   format: "$metadata[$level] $message",
-  metadata: [:request_id]
+  metadata: [:actor, :request_id, :type]
 
 config :mime, :types, %{
   "application/xml" => ["xml"],
diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex
index e38a94966..d2b2cae0b 100644
--- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex
@@ -52,6 +52,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
     when action in [:activity, :object]
   )
 
+  plug(:log_inbox_metadata when action in [:inbox])
   plug(:set_requester_reachable when action in [:inbox])
   plug(:relay_active? when action in [:relay])
 
@@ -521,6 +522,13 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
     conn
   end
 
+  defp log_inbox_metadata(conn = %{params: %{"actor" => actor, "type" => type}}, _) do
+    Logger.metadata(actor: actor, type: type)
+    conn
+  end
+
+  defp log_inbox_metadata(conn, _), do: conn
+
   def upload_media(%{assigns: %{user: %User{} = user}} = conn, %{"file" => file} = data) do
     with {:ok, object} <-
            ActivityPub.upload(

From 40823462e7779fb79a4fcd458daa5e7095a6030b Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Sun, 17 Dec 2023 18:20:22 -0500
Subject: [PATCH 032/192] Logger metadata for request path and authenticated
 user

---
 config/config.exs                             |  4 ++--
 lib/pleroma/web/endpoint.ex                   |  2 ++
 lib/pleroma/web/plugs/logger_metadata_path.ex | 12 ++++++++++++
 lib/pleroma/web/plugs/logger_metadata_user.ex | 18 ++++++++++++++++++
 lib/pleroma/web/router.ex                     |  8 ++++++++
 5 files changed, 42 insertions(+), 2 deletions(-)
 create mode 100644 lib/pleroma/web/plugs/logger_metadata_path.ex
 create mode 100644 lib/pleroma/web/plugs/logger_metadata_user.ex

diff --git a/config/config.exs b/config/config.exs
index 537517688..83e7a33e3 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -131,13 +131,13 @@ config :pleroma, Pleroma.Web.Endpoint,
 config :logger, :console,
   level: :debug,
   format: "\n$time $metadata[$level] $message\n",
-  metadata: [:actor, :request_id, :type]
+  metadata: [:actor, :path, :request_id, :type, :user]
 
 config :logger, :ex_syslogger,
   level: :debug,
   ident: "pleroma",
   format: "$metadata[$level] $message",
-  metadata: [:actor, :request_id, :type]
+  metadata: [:actor, :path, :request_id, :type, :user]
 
 config :mime, :types, %{
   "application/xml" => ["xml"],
diff --git a/lib/pleroma/web/endpoint.ex b/lib/pleroma/web/endpoint.ex
index 2e2104904..fef907ace 100644
--- a/lib/pleroma/web/endpoint.ex
+++ b/lib/pleroma/web/endpoint.ex
@@ -38,6 +38,8 @@ defmodule Pleroma.Web.Endpoint do
 
   plug(Plug.Telemetry, event_prefix: [:phoenix, :endpoint])
 
+  plug(Pleroma.Web.Plugs.LoggerMetadataPath)
+
   plug(Pleroma.Web.Plugs.SetLocalePlug)
   plug(CORSPlug)
   plug(Pleroma.Web.Plugs.HTTPSecurityPlug)
diff --git a/lib/pleroma/web/plugs/logger_metadata_path.ex b/lib/pleroma/web/plugs/logger_metadata_path.ex
new file mode 100644
index 000000000..a5553cfc8
--- /dev/null
+++ b/lib/pleroma/web/plugs/logger_metadata_path.ex
@@ -0,0 +1,12 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.Plugs.LoggerMetadataPath do
+  def init(opts), do: opts
+
+  def call(conn, _) do
+    Logger.metadata(path: conn.request_path)
+    conn
+  end
+end
diff --git a/lib/pleroma/web/plugs/logger_metadata_user.ex b/lib/pleroma/web/plugs/logger_metadata_user.ex
new file mode 100644
index 000000000..6a5c0041d
--- /dev/null
+++ b/lib/pleroma/web/plugs/logger_metadata_user.ex
@@ -0,0 +1,18 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.Plugs.LoggerMetadataUser do
+  alias Pleroma.User
+
+  def init(opts), do: opts
+
+  def call(%{assigns: %{user: user = %User{}}} = conn, _) do
+    Logger.metadata(user: user.nickname)
+    conn
+  end
+
+  def call(conn, _) do
+    conn
+  end
+end
diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index 4fe0cb02f..f0414cc35 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -29,6 +29,7 @@ defmodule Pleroma.Web.Router do
   pipeline :browser do
     plug(:accepts, ["html"])
     plug(:fetch_session)
+    plug(Pleroma.Web.Plugs.LoggerMetadataUser)
   end
 
   pipeline :oauth do
@@ -67,12 +68,14 @@ defmodule Pleroma.Web.Router do
     plug(:fetch_session)
     plug(:authenticate)
     plug(OpenApiSpex.Plug.PutApiSpec, module: Pleroma.Web.ApiSpec)
+    plug(Pleroma.Web.Plugs.LoggerMetadataUser)
   end
 
   pipeline :no_auth_or_privacy_expectations_api do
     plug(:base_api)
     plug(:after_auth)
     plug(Pleroma.Web.Plugs.IdempotencyPlug)
+    plug(Pleroma.Web.Plugs.LoggerMetadataUser)
   end
 
   # Pipeline for app-related endpoints (no user auth checks — app-bound tokens must be supported)
@@ -83,12 +86,14 @@ defmodule Pleroma.Web.Router do
   pipeline :api do
     plug(:expect_public_instance_or_user_authentication)
     plug(:no_auth_or_privacy_expectations_api)
+    plug(Pleroma.Web.Plugs.LoggerMetadataUser)
   end
 
   pipeline :authenticated_api do
     plug(:expect_user_authentication)
     plug(:no_auth_or_privacy_expectations_api)
     plug(Pleroma.Web.Plugs.EnsureAuthenticatedPlug)
+    plug(Pleroma.Web.Plugs.LoggerMetadataUser)
   end
 
   pipeline :admin_api do
@@ -99,6 +104,7 @@ defmodule Pleroma.Web.Router do
     plug(Pleroma.Web.Plugs.EnsureAuthenticatedPlug)
     plug(Pleroma.Web.Plugs.UserIsStaffPlug)
     plug(Pleroma.Web.Plugs.IdempotencyPlug)
+    plug(Pleroma.Web.Plugs.LoggerMetadataUser)
   end
 
   pipeline :require_admin do
@@ -179,6 +185,7 @@ defmodule Pleroma.Web.Router do
     plug(:browser)
     plug(:authenticate)
     plug(Pleroma.Web.Plugs.EnsureUserTokenAssignsPlug)
+    plug(Pleroma.Web.Plugs.LoggerMetadataUser)
   end
 
   pipeline :well_known do
@@ -193,6 +200,7 @@ defmodule Pleroma.Web.Router do
   pipeline :pleroma_api do
     plug(:accepts, ["html", "json"])
     plug(OpenApiSpex.Plug.PutApiSpec, module: Pleroma.Web.ApiSpec)
+    plug(Pleroma.Web.Plugs.LoggerMetadataUser)
   end
 
   pipeline :mailbox_preview do

From 99cee755d8798c0743b96fb11e55f283f0195b85 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Sun, 17 Dec 2023 18:20:34 -0500
Subject: [PATCH 033/192] Show Logger metadata in dev

---
 config/dev.exs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/config/dev.exs b/config/dev.exs
index fe8de5045..f23719fe3 100644
--- a/config/dev.exs
+++ b/config/dev.exs
@@ -35,8 +35,8 @@ config :pleroma, Pleroma.Emails.Mailer, adapter: Swoosh.Adapters.Local
 # configured to run both http and https servers on
 # different ports.
 
-# Do not include metadata nor timestamps in development logs
-config :logger, :console, format: "[$level] $message\n"
+# Do not include timestamps in development logs
+config :logger, :console, format: "$metadata[$level] $message\n"
 
 # Set a higher stacktrace during development. Avoid configuring such
 # in production as building large stacktraces may be expensive.

From 462d5aa5cbe3194bad56a7c973e9b200742ef6ca Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Tue, 19 Mar 2024 20:53:40 -0400
Subject: [PATCH 034/192] logger: remove request_id metadata which is not
 useful

---
 config/config.exs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/config/config.exs b/config/config.exs
index 83e7a33e3..383eb3768 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -131,13 +131,13 @@ config :pleroma, Pleroma.Web.Endpoint,
 config :logger, :console,
   level: :debug,
   format: "\n$time $metadata[$level] $message\n",
-  metadata: [:actor, :path, :request_id, :type, :user]
+  metadata: [:actor, :path, :type, :user]
 
 config :logger, :ex_syslogger,
   level: :debug,
   ident: "pleroma",
   format: "$metadata[$level] $message",
-  metadata: [:actor, :path, :request_id, :type, :user]
+  metadata: [:actor, :path, :type, :user]
 
 config :mime, :types, %{
   "application/xml" => ["xml"],

From 9e6cf45906b9d56834d032d4e0fa436cc5e17031 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= <git@mkljczk.pl>
Date: Sat, 6 Apr 2024 11:43:07 +0200
Subject: [PATCH 035/192] /api/v1/accounts/familiar_followers
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: marcin mikołajczak <git@mkljczk.pl>
---
 changelog.d/familiar-followers.add            |  1 +
 lib/pleroma/user.ex                           | 34 +++++++++++++
 .../api_spec/operations/account_operation.ex  | 42 ++++++++++++++++
 .../controllers/account_controller.ex         | 34 ++++++++++++-
 .../web/mastodon_api/views/account_view.ex    | 19 +++++++
 lib/pleroma/web/router.ex                     |  1 +
 test/pleroma/user_test.exs                    | 14 ++++++
 .../controllers/account_controller_test.exs   | 49 +++++++++++++++++++
 8 files changed, 193 insertions(+), 1 deletion(-)
 create mode 100644 changelog.d/familiar-followers.add

diff --git a/changelog.d/familiar-followers.add b/changelog.d/familiar-followers.add
new file mode 100644
index 000000000..6e7ec9d25
--- /dev/null
+++ b/changelog.d/familiar-followers.add
@@ -0,0 +1 @@
+Implement `/api/v1/accounts/familiar_followers`
\ No newline at end of file
diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index 778e20526..6d6aa98b5 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -1404,6 +1404,40 @@ defmodule Pleroma.User do
     |> Repo.all()
   end
 
+  @spec get_familiar_followers_query(User.t(), User.t(), pos_integer() | nil) :: Ecto.Query.t()
+  def get_familiar_followers_query(%User{} = user, %User{} = current_user, nil) do
+    friends =
+      get_friends_query(current_user)
+      |> where([u], not u.hide_follows)
+      |> select([u], u.id)
+
+    User.Query.build(%{is_active: true})
+    |> where([u], u.id not in ^[user.id, current_user.id])
+    |> join(:inner, [u], r in FollowingRelationship,
+      as: :followers_relationships,
+      on: r.following_id == ^user.id and r.follower_id == u.id
+    )
+    |> where([followers_relationships: r], r.state == ^:follow_accept)
+    |> where([followers_relationships: r], r.follower_id in subquery(friends))
+  end
+
+  def get_familiar_followers_query(%User{} = user, %User{} = current_user, page) do
+    user
+    |> get_familiar_followers_query(current_user, nil)
+    |> User.Query.paginate(page, 20)
+  end
+
+  @spec get_familiar_followers_query(User.t(), User.t()) :: Ecto.Query.t()
+  def get_familiar_followers_query(%User{} = user, %User{} = current_user),
+    do: get_familiar_followers_query(user, current_user, nil)
+
+  @spec get_familiar_followers(User.t(), User.t(), pos_integer() | nil) :: {:ok, list(User.t())}
+  def get_familiar_followers(%User{} = user, %User{} = current_user, page \\ nil) do
+    user
+    |> get_familiar_followers_query(current_user, page)
+    |> Repo.all()
+  end
+
   def increase_note_count(%User{} = user) do
     User
     |> where(id: ^user.id)
diff --git a/lib/pleroma/web/api_spec/operations/account_operation.ex b/lib/pleroma/web/api_spec/operations/account_operation.ex
index 36025e47a..244f18dc7 100644
--- a/lib/pleroma/web/api_spec/operations/account_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/account_operation.ex
@@ -11,6 +11,7 @@ defmodule Pleroma.Web.ApiSpec.AccountOperation do
   alias Pleroma.Web.ApiSpec.Schemas.ActorType
   alias Pleroma.Web.ApiSpec.Schemas.ApiError
   alias Pleroma.Web.ApiSpec.Schemas.BooleanLike
+  alias Pleroma.Web.ApiSpec.Schemas.FlakeID
   alias Pleroma.Web.ApiSpec.Schemas.List
   alias Pleroma.Web.ApiSpec.Schemas.Status
   alias Pleroma.Web.ApiSpec.Schemas.VisibilityScope
@@ -513,6 +514,47 @@ defmodule Pleroma.Web.ApiSpec.AccountOperation do
     }
   end
 
+  def familiar_followers_operation do
+    %Operation{
+      tags: ["Retrieve account information"],
+      summary: "Followers you know",
+      operationId: "AccountController.relationships",
+      description: "Returns followers of given account you know.",
+      security: [%{"oAuth" => ["read:follows"]}],
+      parameters: [
+        Operation.parameter(
+          :id,
+          :query,
+          %Schema{
+            oneOf: [%Schema{type: :array, items: %Schema{type: :string}}, %Schema{type: :string}]
+          },
+          "Account IDs",
+          example: "123"
+        )
+      ],
+      responses: %{
+        200 =>
+          Operation.response("Accounts", "application/json", %Schema{
+            title: "ArrayOfAccounts",
+            type: :array,
+            items: %Schema{
+              title: "Account",
+              type: :object,
+              properties: %{
+                id: FlakeID,
+                accounts: %Schema{
+                  title: "ArrayOfAccounts",
+                  type: :array,
+                  items: Account,
+                  example: [Account.schema().example]
+                }
+              }
+            }
+          })
+      }
+    }
+  end
+
   defp create_request do
     %Schema{
       title: "AccountCreateRequest",
diff --git a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex
index 9226a2deb..47e6f0a64 100644
--- a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex
+++ b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex
@@ -72,7 +72,10 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do
     %{scopes: ["follow", "write:blocks"]} when action in [:block, :unblock]
   )
 
-  plug(OAuthScopesPlug, %{scopes: ["read:follows"]} when action == :relationships)
+  plug(
+    OAuthScopesPlug,
+    %{scopes: ["read:follows"]} when action in [:relationships, :familiar_followers]
+  )
 
   plug(
     OAuthScopesPlug,
@@ -629,6 +632,35 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do
     )
   end
 
+  @doc "GET /api/v1/accounts/familiar_followers"
+  def familiar_followers(
+        %{assigns: %{user: user}, private: %{open_api_spex: %{params: %{id: id}}}} = conn,
+        _id
+      ) do
+    users =
+      User.get_all_by_ids(List.wrap(id))
+      |> Enum.map(&%{id: &1.id, accounts: get_familiar_followers(&1, user)})
+
+    conn
+    |> render("familiar_followers.json",
+      for: user,
+      users: users,
+      as: :user
+    )
+  end
+
+  defp get_familiar_followers(%{id: id} = user, %{id: id}) do
+    User.get_familiar_followers(user, user)
+  end
+
+  defp get_familiar_followers(%{hide_followers: true}, _current_user) do
+    []
+  end
+
+  defp get_familiar_followers(user, current_user) do
+    User.get_familiar_followers(user, current_user)
+  end
+
   @doc "GET /api/v1/identity_proofs"
   def identity_proofs(conn, params), do: MastodonAPIController.empty_array(conn, params)
 end
diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex
index 267c3e3ed..6976ca6e5 100644
--- a/lib/pleroma/web/mastodon_api/views/account_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/account_view.ex
@@ -193,6 +193,25 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do
     render_many(targets, AccountView, "relationship.json", render_opts)
   end
 
+  def render("familiar_followers.json", %{users: users} = opts) do
+    opts =
+      opts
+      |> Map.merge(%{as: :user})
+      |> Map.delete(:users)
+
+    users
+    |> render_many(AccountView, "familiar_followers.json", opts)
+  end
+
+  def render("familiar_followers.json", %{user: %{id: id, accounts: accounts}} = opts) do
+    accounts =
+      accounts
+      |> render_many(AccountView, "show.json", opts)
+      |> Enum.filter(&Enum.any?/1)
+
+    %{id: id, accounts: accounts}
+  end
+
   defp do_render("show.json", %{user: user} = opts) do
     self = opts[:for] == user
 
diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index 4fe0cb02f..644f6cc81 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -633,6 +633,7 @@ defmodule Pleroma.Web.Router do
     patch("/accounts/update_credentials", AccountController, :update_credentials)
 
     get("/accounts/relationships", AccountController, :relationships)
+    get("/accounts/familiar_followers", AccountController, :familiar_followers)
     get("/accounts/:id/lists", AccountController, :lists)
     get("/accounts/:id/identity_proofs", AccountController, :identity_proofs)
     get("/endorsements", AccountController, :endorsements)
diff --git a/test/pleroma/user_test.exs b/test/pleroma/user_test.exs
index a93f81659..48391d871 100644
--- a/test/pleroma/user_test.exs
+++ b/test/pleroma/user_test.exs
@@ -2894,6 +2894,20 @@ defmodule Pleroma.UserTest do
     end
   end
 
+  describe "get_familiar_followers/3" do
+    test "returns familiar followers for a pair of users" do
+      user1 = insert(:user)
+      %{id: id2} = user2 = insert(:user)
+      user3 = insert(:user)
+      _user4 = insert(:user)
+
+      User.follow(user1, user2)
+      User.follow(user2, user3)
+
+      assert [%{id: ^id2}] = User.get_familiar_followers(user3, user1)
+    end
+  end
+
   describe "account endorsements" do
     test "it pins people" do
       user = insert(:user)
diff --git a/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs
index aa7726a9c..e87b33960 100644
--- a/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs
+++ b/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs
@@ -2172,6 +2172,55 @@ defmodule Pleroma.Web.MastodonAPI.AccountControllerTest do
     end
   end
 
+  describe "familiar followers" do
+    setup do: oauth_access(["read:follows"])
+
+    test "fetch user familiar followers", %{user: user, conn: conn} do
+      %{id: id1} = other_user1 = insert(:user)
+      %{id: id2} = other_user2 = insert(:user)
+      _ = insert(:user)
+
+      User.follow(user, other_user1)
+      User.follow(other_user1, other_user2)
+
+      assert [%{"accounts" => [%{"id" => ^id1}], "id" => ^id2}] =
+               conn
+               |> put_req_header("content-type", "application/json")
+               |> get("/api/v1/accounts/familiar_followers?id[]=#{id2}")
+               |> json_response_and_validate_schema(200)
+    end
+
+    test "returns empty array if followers are hidden", %{user: user, conn: conn} do
+      other_user1 = insert(:user, hide_follows: true)
+      %{id: id2} = other_user2 = insert(:user)
+      _ = insert(:user)
+
+      User.follow(user, other_user1)
+      User.follow(other_user1, other_user2)
+
+      assert [%{"accounts" => [], "id" => ^id2}] =
+               conn
+               |> put_req_header("content-type", "application/json")
+               |> get("/api/v1/accounts/familiar_followers?id[]=#{id2}")
+               |> json_response_and_validate_schema(200)
+    end
+
+    test "it respects hide_followers", %{user: user, conn: conn} do
+      other_user1 = insert(:user)
+      %{id: id2} = other_user2 = insert(:user, hide_followers: true)
+      _ = insert(:user)
+
+      User.follow(user, other_user1)
+      User.follow(other_user1, other_user2)
+
+      assert [%{"accounts" => [], "id" => ^id2}] =
+               conn
+               |> put_req_header("content-type", "application/json")
+               |> get("/api/v1/accounts/familiar_followers?id[]=#{id2}")
+               |> json_response_and_validate_schema(200)
+    end
+  end
+
   describe "remove from followers" do
     setup do: oauth_access(["follow"])
 

From 88412daf118f15a40119f0b47a740a442ec5040c Mon Sep 17 00:00:00 2001
From: Haelwenn <contact+git.pleroma.social@hacktivis.me>
Date: Fri, 12 Apr 2024 09:15:06 +0000
Subject: [PATCH 036/192] Apply @lanodan's suggestion
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: marcin mikołajczak <git@mkljczk.pl>
---
 lib/pleroma/web/api_spec/operations/account_operation.ex | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/lib/pleroma/web/api_spec/operations/account_operation.ex b/lib/pleroma/web/api_spec/operations/account_operation.ex
index 244f18dc7..85f02166f 100644
--- a/lib/pleroma/web/api_spec/operations/account_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/account_operation.ex
@@ -517,9 +517,10 @@ defmodule Pleroma.Web.ApiSpec.AccountOperation do
   def familiar_followers_operation do
     %Operation{
       tags: ["Retrieve account information"],
-      summary: "Followers you know",
-      operationId: "AccountController.relationships",
-      description: "Returns followers of given account you know.",
+      summary: "Followers that you follow",
+      operationId: "AccountController.familiar_followers",
+      description:
+        "Obtain a list of all accounts that follow a given account, filtered for accounts you follow.",
       security: [%{"oAuth" => ["read:follows"]}],
       parameters: [
         Operation.parameter(

From 1bf3ae07b6719a762310615811a317035175ad2e Mon Sep 17 00:00:00 2001
From: faried nawaz <faried@gmail.com>
Date: Mon, 18 Sep 2023 02:13:01 +0500
Subject: [PATCH 037/192] add options to mix pleroma.database prune_objects to
 delete more activities

---
 lib/mix/tasks/pleroma/database.ex        | 165 +++++++-
 test/mix/tasks/pleroma/database_test.exs | 487 ++++++++++++++++++++++-
 2 files changed, 625 insertions(+), 27 deletions(-)

diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex
index 93ee57dc3..13ac6536c 100644
--- a/lib/mix/tasks/pleroma/database.ex
+++ b/lib/mix/tasks/pleroma/database.ex
@@ -67,43 +67,168 @@ defmodule Mix.Tasks.Pleroma.Database do
       OptionParser.parse(
         args,
         strict: [
-          vacuum: :boolean
+          vacuum: :boolean,
+          keep_threads: :boolean,
+          keep_non_public: :boolean,
+          prune_orphaned_activities: :boolean
         ]
       )
 
     start_pleroma()
 
     deadline = Pleroma.Config.get([:instance, :remote_post_retention_days])
+    time_deadline = NaiveDateTime.utc_now() |> NaiveDateTime.add(-(deadline * 86_400))
 
-    Logger.info("Pruning objects older than #{deadline} days")
+    log_message = "Pruning objects older than #{deadline} days"
 
-    time_deadline =
-      NaiveDateTime.utc_now()
-      |> NaiveDateTime.add(-(deadline * 86_400))
+    log_message =
+      if Keyword.get(options, :keep_non_public) do
+        log_message <> ", keeping non public posts"
+      else
+        log_message
+      end
 
-    from(o in Object,
-      where:
-        fragment(
-          "?->'to' \\? ? OR ?->'cc' \\? ?",
-          o.data,
-          ^Pleroma.Constants.as_public(),
-          o.data,
-          ^Pleroma.Constants.as_public()
-        ),
-      where: o.inserted_at < ^time_deadline,
-      where:
+    log_message =
+      if Keyword.get(options, :keep_threads) do
+        log_message <> ", keeping threads intact"
+      else
+        log_message
+      end
+
+    log_message =
+      if Keyword.get(options, :prune_orphaned_activities) do
+        log_message <> ", pruning orphaned activities"
+      else
+        log_message
+      end
+
+    log_message =
+      if Keyword.get(options, :vacuum) do
+        log_message <>
+          ", doing a full vacuum (you shouldn't do this as a recurring maintanance task)"
+      else
+        log_message
+      end
+
+    Logger.info(log_message)
+
+    if Keyword.get(options, :keep_threads) do
+      # We want to delete objects from threads where
+      # 1. the newest post is still old
+      # 2. none of the activities is local
+      # 3. none of the activities is bookmarked
+      # 4. optionally none of the posts is non-public
+      deletable_context =
+        if Keyword.get(options, :keep_non_public) do
+          Pleroma.Activity
+          |> join(:left, [a], b in Pleroma.Bookmark, on: a.id == b.activity_id)
+          |> group_by([a], fragment("? ->> 'context'::text", a.data))
+          |> having(
+            [a],
+            not fragment(
+              # Posts (checked on Create Activity) is non-public
+              "bool_or((not(?->'to' \\? ? OR ?->'cc' \\? ?)) and ? ->> 'type' = 'Create')",
+              a.data,
+              ^Pleroma.Constants.as_public(),
+              a.data,
+              ^Pleroma.Constants.as_public(),
+              a.data
+            )
+          )
+        else
+          Pleroma.Activity
+          |> join(:left, [a], b in Pleroma.Bookmark, on: a.id == b.activity_id)
+          |> group_by([a], fragment("? ->> 'context'::text", a.data))
+        end
+        |> having([a], max(a.updated_at) < ^time_deadline)
+        |> having([a], not fragment("bool_or(?)", a.local))
+        |> having([_, b], fragment("max(?::text) is null", b.id))
+        |> select([a], fragment("? ->> 'context'::text", a.data))
+
+      Pleroma.Object
+      |> where([o], fragment("? ->> 'context'::text", o.data) in subquery(deletable_context))
+    else
+      if Keyword.get(options, :keep_non_public) do
+        Pleroma.Object
+        |> where(
+          [o],
+          fragment(
+            "?->'to' \\? ? OR ?->'cc' \\? ?",
+            o.data,
+            ^Pleroma.Constants.as_public(),
+            o.data,
+            ^Pleroma.Constants.as_public()
+          )
+        )
+      else
+        Pleroma.Object
+      end
+      |> where([o], o.updated_at < ^time_deadline)
+      |> where(
+        [o],
         fragment("split_part(?->>'actor', '/', 3) != ?", o.data, ^Pleroma.Web.Endpoint.host())
-    )
+      )
+    end
     |> Repo.delete_all(timeout: :infinity)
 
-    prune_hashtags_query = """
+    if !Keyword.get(options, :keep_threads) do
+      # Without the --keep-threads option, it's possible that bookmarked
+      # objects have been deleted. We remove the corresponding bookmarks.
+      """
+      delete from public.bookmarks
+      where id in (
+        select b.id from public.bookmarks b
+        left join public.activities a on b.activity_id = a.id
+        left join public.objects o on a."data" ->> 'object' = o.data ->> 'id'
+        where o.id is null
+      )
+      """
+      |> Repo.query([], timeout: :infinity)
+    end
+
+    if Keyword.get(options, :prune_orphaned_activities) do
+      # Prune activities who link to a single object
+      """
+      delete from public.activities
+      where id in (
+        select a.id from public.activities a
+        left join public.objects o on a.data ->> 'object' = o.data ->> 'id'
+        left join public.activities a2 on a.data ->> 'object' = a2.data ->> 'id'
+        left join public.users u  on a.data ->> 'object' = u.ap_id
+        where not a.local
+        and jsonb_typeof(a."data" -> 'object') = 'string'
+        and o.id is null
+        and a2.id is null
+        and u.id is null
+      )
+      """
+      |> Repo.query([], timeout: :infinity)
+
+      # Prune activities who link to an array of objects
+      """
+      delete from public.activities
+      where id in (
+        select a.id from public.activities a
+        join json_array_elements_text((a."data" -> 'object')::json) as j on jsonb_typeof(a."data" -> 'object') = 'array'
+        left join public.objects o on j.value = o.data ->> 'id'
+        left join public.activities a2 on j.value = a2.data ->> 'id'
+        left join public.users u  on j.value = u.ap_id
+        group by a.id
+        having max(o.data ->> 'id') is null
+        and max(a2.data ->> 'id') is null
+        and max(u.ap_id) is null
+      )
+      """
+      |> Repo.query([], timeout: :infinity)
+    end
+
+    """
     DELETE FROM hashtags AS ht
     WHERE NOT EXISTS (
       SELECT 1 FROM hashtags_objects hto
       WHERE ht.id = hto.hashtag_id)
     """
-
-    Repo.query(prune_hashtags_query)
+    |> Repo.query()
 
     if Keyword.get(options, :vacuum) do
       Maintenance.vacuum("full")
diff --git a/test/mix/tasks/pleroma/database_test.exs b/test/mix/tasks/pleroma/database_test.exs
index fbc939171..01fd97e2d 100644
--- a/test/mix/tasks/pleroma/database_test.exs
+++ b/test/mix/tasks/pleroma/database_test.exs
@@ -7,6 +7,7 @@ defmodule Mix.Tasks.Pleroma.DatabaseTest do
   use Oban.Testing, repo: Pleroma.Repo
 
   alias Pleroma.Activity
+  alias Pleroma.Bookmark
   alias Pleroma.Object
   alias Pleroma.Repo
   alias Pleroma.User
@@ -45,28 +46,500 @@ defmodule Mix.Tasks.Pleroma.DatabaseTest do
   end
 
   describe "prune_objects" do
-    test "it prunes old objects from the database" do
-      insert(:note)
+    setup do
       deadline = Pleroma.Config.get([:instance, :remote_post_retention_days]) + 1
 
-      date =
+      old_insert_date =
         Timex.now()
         |> Timex.shift(days: -deadline)
         |> Timex.to_naive_datetime()
         |> NaiveDateTime.truncate(:second)
 
-      %{id: id} =
+      %{old_insert_date: old_insert_date}
+    end
+
+    test "it prunes old objects from the database", %{old_insert_date: old_insert_date} do
+      insert(:note)
+
+      %{id: note_remote_public_id} =
         :note
         |> insert()
-        |> Ecto.Changeset.change(%{inserted_at: date})
+        |> Ecto.Changeset.change(%{updated_at: old_insert_date})
         |> Repo.update!()
 
-      assert length(Repo.all(Object)) == 2
+      note_remote_non_public =
+        %{id: note_remote_non_public_id, data: note_remote_non_public_data} =
+        :note
+        |> insert()
+
+      note_remote_non_public
+      |> Ecto.Changeset.change(%{
+        updated_at: old_insert_date,
+        data: note_remote_non_public_data |> update_in(["to"], fn _ -> [] end)
+      })
+      |> Repo.update!()
+
+      assert length(Repo.all(Object)) == 3
 
       Mix.Tasks.Pleroma.Database.run(["prune_objects"])
 
       assert length(Repo.all(Object)) == 1
-      refute Object.get_by_id(id)
+      refute Object.get_by_id(note_remote_public_id)
+      refute Object.get_by_id(note_remote_non_public_id)
+    end
+
+    test "it cleans up bookmarks", %{old_insert_date: old_insert_date} do
+      user = insert(:user)
+      {:ok, old_object_activity} = CommonAPI.post(user, %{status: "yadayada"})
+
+      Repo.one(Object)
+      |> Ecto.Changeset.change(%{updated_at: old_insert_date})
+      |> Repo.update!()
+
+      {:ok, new_object_activity} = CommonAPI.post(user, %{status: "yadayada"})
+
+      {:ok, _} = Bookmark.create(user.id, old_object_activity.id)
+      {:ok, _} = Bookmark.create(user.id, new_object_activity.id)
+
+      assert length(Repo.all(Object)) == 2
+      assert length(Repo.all(Bookmark)) == 2
+
+      Mix.Tasks.Pleroma.Database.run(["prune_objects"])
+
+      assert length(Repo.all(Object)) == 1
+      assert length(Repo.all(Bookmark)) == 1
+      refute Bookmark.get(user.id, old_object_activity.id)
+    end
+
+    test "with the --keep-non-public option it still keeps non-public posts even if they are not local",
+         %{old_insert_date: old_insert_date} do
+      insert(:note)
+
+      %{id: note_remote_id} =
+        :note
+        |> insert()
+        |> Ecto.Changeset.change(%{updated_at: old_insert_date})
+        |> Repo.update!()
+
+      note_remote_non_public =
+        %{data: note_remote_non_public_data} =
+        :note
+        |> insert()
+
+      note_remote_non_public
+      |> Ecto.Changeset.change(%{
+        updated_at: old_insert_date,
+        data: note_remote_non_public_data |> update_in(["to"], fn _ -> [] end)
+      })
+      |> Repo.update!()
+
+      assert length(Repo.all(Object)) == 3
+
+      Mix.Tasks.Pleroma.Database.run(["prune_objects", "--keep-non-public"])
+
+      assert length(Repo.all(Object)) == 2
+      refute Object.get_by_id(note_remote_id)
+    end
+
+    test "with the --keep-threads and --keep-non-public option it keeps old threads with non-public replies even if the interaction is not local",
+         %{old_insert_date: old_insert_date} do
+      # For non-public we only check Create Activities because only these are relevant for threads
+      # Flags are always non-public, Announces from relays can be non-public...
+
+      remote_user1 = insert(:user, local: false)
+      remote_user2 = insert(:user, local: false)
+
+      # Old remote non-public reply (should be kept)
+      {:ok, old_remote_post1_activity} =
+        CommonAPI.post(remote_user1, %{status: "some thing", local: false})
+
+      old_remote_post1_activity
+      |> Ecto.Changeset.change(%{local: false, updated_at: old_insert_date})
+      |> Repo.update!()
+
+      {:ok, old_remote_non_public_reply_activity} =
+        CommonAPI.post(remote_user2, %{
+          status: "some reply",
+          in_reply_to_status_id: old_remote_post1_activity.id
+        })
+
+      old_remote_non_public_reply_activity
+      |> Ecto.Changeset.change(%{
+        local: false,
+        updated_at: old_insert_date,
+        data: old_remote_non_public_reply_activity.data |> update_in(["to"], fn _ -> [] end)
+      })
+      |> Repo.update!()
+
+      # Old remote non-public Announce (should be removed)
+      {:ok, old_remote_post2_activity = %{data: %{"object" => old_remote_post2_id}}} =
+        CommonAPI.post(remote_user1, %{status: "some thing", local: false})
+
+      old_remote_post2_activity
+      |> Ecto.Changeset.change(%{local: false, updated_at: old_insert_date})
+      |> Repo.update!()
+
+      {:ok, old_remote_non_public_repeat_activity} =
+        CommonAPI.repeat(old_remote_post2_activity.id, remote_user2)
+
+      old_remote_non_public_repeat_activity
+      |> Ecto.Changeset.change(%{
+        local: false,
+        updated_at: old_insert_date,
+        data: old_remote_non_public_repeat_activity.data |> update_in(["to"], fn _ -> [] end)
+      })
+      |> Repo.update!()
+
+      assert length(Repo.all(Object)) == 3
+
+      Mix.Tasks.Pleroma.Database.run(["prune_objects", "--keep-threads", "--keep-non-public"])
+
+      Repo.all(Pleroma.Activity)
+      assert length(Repo.all(Object)) == 2
+      refute Object.get_by_ap_id(old_remote_post2_id)
+    end
+
+    test "with the --keep-threads option it still keeps non-old threads even with no local interactions" do
+      remote_user = insert(:user, local: false)
+      remote_user2 = insert(:user, local: false)
+
+      {:ok, remote_post_activity} =
+        CommonAPI.post(remote_user, %{status: "some thing", local: false})
+
+      {:ok, remote_post_reply_activity} =
+        CommonAPI.post(remote_user2, %{
+          status: "some reply",
+          in_reply_to_status_id: remote_post_activity.id
+        })
+
+      remote_post_activity
+      |> Ecto.Changeset.change(%{local: false})
+      |> Repo.update!()
+
+      remote_post_reply_activity
+      |> Ecto.Changeset.change(%{local: false})
+      |> Repo.update!()
+
+      assert length(Repo.all(Object)) == 2
+
+      Mix.Tasks.Pleroma.Database.run(["prune_objects", "--keep-threads"])
+
+      assert length(Repo.all(Object)) == 2
+    end
+
+    test "with the --keep-threads option it deletes old threads with no local interaction", %{
+      old_insert_date: old_insert_date
+    } do
+      remote_user = insert(:user, local: false)
+      remote_user2 = insert(:user, local: false)
+
+      {:ok, old_remote_post_activity} =
+        CommonAPI.post(remote_user, %{status: "some thing", local: false})
+
+      old_remote_post_activity
+      |> Ecto.Changeset.change(%{local: false, updated_at: old_insert_date})
+      |> Repo.update!()
+
+      {:ok, old_remote_post_reply_activity} =
+        CommonAPI.post(remote_user2, %{
+          status: "some reply",
+          in_reply_to_status_id: old_remote_post_activity.id
+        })
+
+      old_remote_post_reply_activity
+      |> Ecto.Changeset.change(%{local: false, updated_at: old_insert_date})
+      |> Repo.update!()
+
+      {:ok, old_favourite_activity} =
+        CommonAPI.favorite(remote_user2, old_remote_post_activity.id)
+
+      old_favourite_activity
+      |> Ecto.Changeset.change(%{local: false, updated_at: old_insert_date})
+      |> Repo.update!()
+
+      {:ok, old_repeat_activity} = CommonAPI.repeat(old_remote_post_activity.id, remote_user2)
+
+      old_repeat_activity
+      |> Ecto.Changeset.change(%{local: false, updated_at: old_insert_date})
+      |> Repo.update!()
+
+      assert length(Repo.all(Object)) == 2
+
+      Mix.Tasks.Pleroma.Database.run(["prune_objects", "--keep-threads"])
+
+      assert length(Repo.all(Object)) == 0
+    end
+
+    test "with the --keep-threads option it keeps old threads with local interaction", %{
+      old_insert_date: old_insert_date
+    } do
+      remote_user = insert(:user, local: false)
+      local_user = insert(:user, local: true)
+
+      # local reply
+      {:ok, old_remote_post1_activity} =
+        CommonAPI.post(remote_user, %{status: "some thing", local: false})
+
+      old_remote_post1_activity
+      |> Ecto.Changeset.change(%{local: false, updated_at: old_insert_date})
+      |> Repo.update!()
+
+      {:ok, old_local_post2_reply_activity} =
+        CommonAPI.post(local_user, %{
+          status: "some reply",
+          in_reply_to_status_id: old_remote_post1_activity.id
+        })
+
+      old_local_post2_reply_activity
+      |> Ecto.Changeset.change(%{local: true, updated_at: old_insert_date})
+      |> Repo.update!()
+
+      # local Like
+      {:ok, old_remote_post3_activity} =
+        CommonAPI.post(remote_user, %{status: "some thing", local: false})
+
+      old_remote_post3_activity
+      |> Ecto.Changeset.change(%{local: false, updated_at: old_insert_date})
+      |> Repo.update!()
+
+      {:ok, old_favourite_activity} = CommonAPI.favorite(local_user, old_remote_post3_activity.id)
+
+      old_favourite_activity
+      |> Ecto.Changeset.change(%{local: true, updated_at: old_insert_date})
+      |> Repo.update!()
+
+      # local Announce
+      {:ok, old_remote_post4_activity} =
+        CommonAPI.post(remote_user, %{status: "some thing", local: false})
+
+      old_remote_post4_activity
+      |> Ecto.Changeset.change(%{local: false, updated_at: old_insert_date})
+      |> Repo.update!()
+
+      {:ok, old_repeat_activity} = CommonAPI.repeat(old_remote_post4_activity.id, local_user)
+
+      old_repeat_activity
+      |> Ecto.Changeset.change(%{local: true, updated_at: old_insert_date})
+      |> Repo.update!()
+
+      assert length(Repo.all(Object)) == 4
+
+      Mix.Tasks.Pleroma.Database.run(["prune_objects", "--keep-threads"])
+
+      assert length(Repo.all(Object)) == 4
+    end
+
+    test "with the --keep-threads option it keeps old threads with bookmarked posts", %{
+      old_insert_date: old_insert_date
+    } do
+      remote_user = insert(:user, local: false)
+      local_user = insert(:user, local: true)
+
+      {:ok, old_remote_post_activity} =
+        CommonAPI.post(remote_user, %{status: "some thing", local: false})
+
+      old_remote_post_activity
+      |> Ecto.Changeset.change(%{local: false, updated_at: old_insert_date})
+      |> Repo.update!()
+
+      Pleroma.Bookmark.create(local_user.id, old_remote_post_activity.id)
+
+      assert length(Repo.all(Object)) == 1
+
+      Mix.Tasks.Pleroma.Database.run(["prune_objects", "--keep-threads"])
+
+      assert length(Repo.all(Object)) == 1
+    end
+
+    test "We don't have unexpected tables which may contain objects that are referenced by activities" do
+      # We can delete orphaned activities. For that we look for the objects they reference in the 'objects', 'activities', and 'users' table.
+      # If someone adds another table with objects (idk, maybe with separate relations, or collections or w/e), then we need to make sure we
+      # add logic for that in the 'prune_objects' task so that we don't wrongly delete their corresponding activities.
+      # So when someone adds (or removes) a table, this test will fail.
+      # Either the table contains objects which can be referenced from the activities table
+      # => in that case the prune_objects job should be adapted so we don't delete activities who still have the referenced object.
+      # Or it doesn't contain objects which can be referenced from the activities table
+      # => in that case you can add/remove the table to/from this (sorted) list.
+
+      assert Repo.query!(
+               "SELECT table_name FROM information_schema.tables WHERE table_schema='public' AND table_type='BASE TABLE';"
+             ).rows
+             |> Enum.sort() == [
+               ["activities"],
+               ["announcement_read_relationships"],
+               ["announcements"],
+               ["apps"],
+               ["backups"],
+               ["bookmarks"],
+               ["chat_message_references"],
+               ["chats"],
+               ["config"],
+               ["conversation_participation_recipient_ships"],
+               ["conversation_participations"],
+               ["conversations"],
+               ["counter_cache"],
+               ["data_migration_failed_ids"],
+               ["data_migrations"],
+               ["deliveries"],
+               ["filters"],
+               ["following_relationships"],
+               ["hashtags"],
+               ["hashtags_objects"],
+               ["instances"],
+               ["lists"],
+               ["markers"],
+               ["mfa_tokens"],
+               ["moderation_log"],
+               ["notifications"],
+               ["oauth_authorizations"],
+               ["oauth_tokens"],
+               ["oban_jobs"],
+               ["oban_peers"],
+               ["objects"],
+               ["password_reset_tokens"],
+               ["push_subscriptions"],
+               ["registrations"],
+               ["report_notes"],
+               ["scheduled_activities"],
+               ["schema_migrations"],
+               ["thread_mutes"],
+               # ["user_follows_hashtag"],                  # not in pleroma
+               # ["user_frontend_setting_profiles"],        # not in pleroma
+               ["user_invite_tokens"],
+               ["user_notes"],
+               ["user_relationships"],
+               ["users"]
+             ]
+    end
+
+    test "it prunes orphaned activities with the --prune-orphaned-activities" do
+      # Add a remote activity which references an Object
+      %Object{} |> Map.merge(%{data: %{"id" => "object_for_activity"}}) |> Repo.insert()
+
+      %Activity{}
+      |> Map.merge(%{
+        local: false,
+        data: %{"id" => "remote_activity_with_object", "object" => "object_for_activity"}
+      })
+      |> Repo.insert()
+
+      # Add a remote activity which references an activity
+      %Activity{}
+      |> Map.merge(%{
+        local: false,
+        data: %{
+          "id" => "remote_activity_with_activity",
+          "object" => "remote_activity_with_object"
+        }
+      })
+      |> Repo.insert()
+
+      # Add a remote activity which references an Actor
+      %User{} |> Map.merge(%{ap_id: "actor"}) |> Repo.insert()
+
+      %Activity{}
+      |> Map.merge(%{
+        local: false,
+        data: %{"id" => "remote_activity_with_actor", "object" => "actor"}
+      })
+      |> Repo.insert()
+
+      # Add a remote activity without existing referenced object, activity or actor
+      %Activity{}
+      |> Map.merge(%{
+        local: false,
+        data: %{
+          "id" => "remote_activity_without_existing_referenced_object",
+          "object" => "non_existing"
+        }
+      })
+      |> Repo.insert()
+
+      # Add a local activity without existing referenced object, activity or actor
+      %Activity{}
+      |> Map.merge(%{
+        local: true,
+        data: %{"id" => "local_activity_with_actor", "object" => "non_existing"}
+      })
+      |> Repo.insert()
+
+      # The remote activities without existing reference, and only the remote activities without existing reference, are deleted
+      # if, and only if, we provide the --prune-orphaned-activities option
+      assert length(Repo.all(Activity)) == 5
+      Mix.Tasks.Pleroma.Database.run(["prune_objects"])
+      assert length(Repo.all(Activity)) == 5
+      Mix.Tasks.Pleroma.Database.run(["prune_objects", "--prune-orphaned-activities"])
+      activities = Repo.all(Activity)
+
+      assert "remote_activity_without_existing_referenced_object" not in Enum.map(
+               activities,
+               fn a -> a.data["id"] end
+             )
+
+      assert length(activities) == 4
+    end
+
+    test "it prunes orphaned activities with the --prune-orphaned-activities when the objects are referenced from an array" do
+      %Object{} |> Map.merge(%{data: %{"id" => "existing_object"}}) |> Repo.insert()
+      %User{} |> Map.merge(%{ap_id: "existing_actor"}) |> Repo.insert()
+
+      # Multiple objects, one object exists (keep)
+      %Activity{}
+      |> Map.merge(%{
+        local: false,
+        data: %{
+          "id" => "remote_activity_existing_object",
+          "object" => ["non_ existing_object", "existing_object"]
+        }
+      })
+      |> Repo.insert()
+
+      # Multiple objects, one actor exists (keep)
+      %Activity{}
+      |> Map.merge(%{
+        local: false,
+        data: %{
+          "id" => "remote_activity_existing_actor",
+          "object" => ["non_ existing_object", "existing_actor"]
+        }
+      })
+      |> Repo.insert()
+
+      # Multiple objects, one activity exists (keep)
+      %Activity{}
+      |> Map.merge(%{
+        local: false,
+        data: %{
+          "id" => "remote_activity_existing_activity",
+          "object" => ["non_ existing_object", "remote_activity_existing_actor"]
+        }
+      })
+      |> Repo.insert()
+
+      # Multiple objects none exist (prune)
+      %Activity{}
+      |> Map.merge(%{
+        local: false,
+        data: %{
+          "id" => "remote_activity_without_existing_referenced_object",
+          "object" => ["owo", "whats_this"]
+        }
+      })
+      |> Repo.insert()
+
+      assert length(Repo.all(Activity)) == 4
+      Mix.Tasks.Pleroma.Database.run(["prune_objects"])
+      assert length(Repo.all(Activity)) == 4
+      Mix.Tasks.Pleroma.Database.run(["prune_objects", "--prune-orphaned-activities"])
+      activities = Repo.all(Activity)
+      assert length(activities) == 3
+
+      assert "remote_activity_without_existing_referenced_object" not in Enum.map(
+               activities,
+               fn a -> a.data["id"] end
+             )
+
+      assert length(activities) == 3
     end
   end
 

From fdc3cbb8cbefad2161cc408b4440420d6e4b2a88 Mon Sep 17 00:00:00 2001
From: faried nawaz <faried@gmail.com>
Date: Mon, 18 Sep 2023 02:13:52 +0500
Subject: [PATCH 038/192] add documentation for the prune_objects mix task
 options

---
 changelog.d/akkoma-prune-options.add      |  1 +
 docs/administration/CLI_tasks/database.md | 17 +++++++++++++----
 2 files changed, 14 insertions(+), 4 deletions(-)
 create mode 100644 changelog.d/akkoma-prune-options.add

diff --git a/changelog.d/akkoma-prune-options.add b/changelog.d/akkoma-prune-options.add
new file mode 100644
index 000000000..6bc5e7f92
--- /dev/null
+++ b/changelog.d/akkoma-prune-options.add
@@ -0,0 +1 @@
+Add options to the mix prune_objects task
diff --git a/docs/administration/CLI_tasks/database.md b/docs/administration/CLI_tasks/database.md
index c53c49921..c5e51e555 100644
--- a/docs/administration/CLI_tasks/database.md
+++ b/docs/administration/CLI_tasks/database.md
@@ -21,16 +21,18 @@ Replaces embedded objects with references to them in the `objects` table. Only n
     mix pleroma.database remove_embedded_objects [option ...]
     ```
 
-
 ### Options
 - `--vacuum` - run `VACUUM FULL` after the embedded objects are replaced with their references
 
 ## Prune old remote posts from the database
 
-This will prune remote posts older than 90 days (configurable with [`config :pleroma, :instance, remote_post_retention_days`](../../configuration/cheatsheet.md#instance)) from the database, they will be refetched from source when accessed.
+This will prune remote posts older than 90 days (configurable with [`config :pleroma, :instance, remote_post_retention_days`](../../configuration/cheatsheet.md#instance)) from the database. Pruned posts may be refetched in some cases.
+
+!!! note
+    The disk space will only be reclaimed after a proper vacuum. By default Postgresql does this for you on a regular basis, but if your instance has been running for a long time and there are many rows deleted, it may be advantageous to use `VACUUM FULL` (e.g. by using the `--vacuum` option).
 
 !!! danger
-    The disk space will only be reclaimed after `VACUUM FULL`. You may run out of disk space during the execution of the task or vacuuming if you don't have about 1/3rds of the database size free.
+    You may run out of disk space during the execution of the task or vacuuming if you don't have about 1/3rds of the database size free. Vacuum causes a substantial increase in I/O traffic, and may lead to a degraded experience while it is running.
 
 === "OTP"
 
@@ -45,7 +47,11 @@ This will prune remote posts older than 90 days (configurable with [`config :ple
     ```
 
 ### Options
-- `--vacuum` - run `VACUUM FULL` after the objects are pruned
+
+- `--keep-threads` - Don't prune posts when they are part of a thread where at least one post has seen local interaction (e.g. one of the posts is a local post, or is favourited by a local user, or has been repeated by a local user...). It also won't delete posts when at least one of the posts in that thread is kept (e.g. because one of the posts has seen recent activity).
+- `--keep-non-public` - Keep non-public posts like DM's and followers-only, even if they are remote.
+- `--prune-orphaned-activities` - Also prune orphaned activities afterwards. Activities are things like Like, Create, Announce, Flag (aka reports). They can significantly help reduce the database size.  Note: this can take a very long time.
+- `--vacuum` - Run `VACUUM FULL` after the objects are pruned. This should not be used on a regular basis, but is useful if your instance has been running for a long time before pruning.
 
 ## Create a conversation for all existing DMs
 
@@ -93,6 +99,9 @@ Can be safely re-run
 
 ## Vacuum the database
 
+!!! note
+    By default Postgresql has an autovacuum deamon running. While the tasks described here can help in some cases, they shouldn't be needed on a regular basis. See [the Postgresql docs on vacuuming](https://www.postgresql.org/docs/current/sql-vacuum.html) for more information on this.
+
 ### Analyze
 
 Running an `analyze` vacuum job can improve performance by updating statistics used by the query planner. **It is safe to cancel this.**

From cd7e2138d11901fc7a0c8c2f22b7a5d57383a555 Mon Sep 17 00:00:00 2001
From: Lain Soykaf <lain@lain.com>
Date: Tue, 14 May 2024 14:13:37 +0400
Subject: [PATCH 039/192] Search: Basic Qdrant/Ollama search

---
 config/config.exs                       |   9 ++
 lib/mix/tasks/pleroma/search/indexer.ex |  60 ++++++++++++
 lib/pleroma/search/qdrant_search.ex     | 117 ++++++++++++++++++++++++
 3 files changed, 186 insertions(+)
 create mode 100644 lib/mix/tasks/pleroma/search/indexer.ex
 create mode 100644 lib/pleroma/search/qdrant_search.ex

diff --git a/config/config.exs b/config/config.exs
index b69044a2b..f74eda6b2 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -915,6 +915,15 @@ config :pleroma, Pleroma.Application,
 
 config :pleroma, Pleroma.Uploaders.Uploader, timeout: 30_000
 
+config :pleroma, Pleroma.Search.QdrantSearch,
+  qdrant_url: "http://127.0.0.1:6333/",
+  qdrant_api_key: nil,
+  ollama_url: "http://127.0.0.1:11434",
+  ollama_model: "snowflake-arctic-embed:xs",
+  qdrant_index_configuration: %{
+    vectors: %{size: 384, distance: "Cosine"}
+  }
+
 # Import environment specific config. This must remain at the bottom
 # of this file so it overrides the configuration defined above.
 import_config "#{Mix.env()}.exs"
diff --git a/lib/mix/tasks/pleroma/search/indexer.ex b/lib/mix/tasks/pleroma/search/indexer.ex
new file mode 100644
index 000000000..ffa2f3c94
--- /dev/null
+++ b/lib/mix/tasks/pleroma/search/indexer.ex
@@ -0,0 +1,60 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Mix.Tasks.Pleroma.Search.Indexer do
+  import Mix.Pleroma
+  import Ecto.Query
+
+  alias Pleroma.Workers.SearchIndexingWorker
+
+  def run(["index" | options]) do
+    {options, [], []} =
+      OptionParser.parse(
+        options,
+        strict: [
+          limit: :integer
+        ]
+      )
+
+    start_pleroma()
+
+    limit = Keyword.get(options, :limit, 100_000)
+
+    per_step = 1000
+    chunks = max(div(limit, per_step), 1)
+
+    1..chunks
+    |> Enum.each(fn step ->
+      q =
+        from(a in Pleroma.Activity,
+          limit: ^per_step,
+          offset: ^per_step * (^step - 1),
+          select: [:id],
+          order_by: [desc: :id]
+        )
+
+      {:ok, ids} =
+        Pleroma.Repo.transaction(fn ->
+          Pleroma.Repo.stream(q, timeout: :infinity)
+          |> Enum.map(fn a ->
+            a.id
+          end)
+        end)
+
+      IO.puts("Got #{length(ids)} activities, adding to indexer")
+
+      ids
+      |> Enum.chunk_every(100)
+      |> Enum.each(fn chunk ->
+        IO.puts("Adding #{length(chunk)} activities to indexing queue")
+
+        chunk
+        |> Enum.map(fn id ->
+          SearchIndexingWorker.new(%{"op" => "add_to_index", "activity" => id})
+        end)
+        |> Oban.insert_all()
+      end)
+    end)
+  end
+end
diff --git a/lib/pleroma/search/qdrant_search.ex b/lib/pleroma/search/qdrant_search.ex
new file mode 100644
index 000000000..fcaa9e686
--- /dev/null
+++ b/lib/pleroma/search/qdrant_search.ex
@@ -0,0 +1,117 @@
+defmodule Pleroma.Search.QdrantSearch do
+  @behaviour Pleroma.Search.SearchBackend
+  import Ecto.Query
+  alias Pleroma.Activity
+
+  alias __MODULE__.QdrantClient
+  alias __MODULE__.OllamaClient
+
+  import Pleroma.Search.Meilisearch, only: [object_to_search_data: 1]
+
+  def initialize_index() do
+    payload = Pleroma.Config.get([Pleroma.Search.QdrantSearch, :qdrant_index_configuration])
+    QdrantClient.put("/collections/posts", payload)
+  end
+
+  def drop_index() do
+    QdrantClient.delete("/collections/posts")
+  end
+
+  def get_embedding(text) do
+    with {:ok, %{body: %{"embedding" => embedding}}} <-
+           OllamaClient.post("/api/embeddings", %{
+             prompt: text,
+             model: Pleroma.Config.get([Pleroma.Search.QdrantSearch, :ollama_model])
+           })
+           |> IO.inspect() do
+      {:ok, embedding}
+    else
+      _ ->
+        {:error, "Failed to get embedding"}
+    end
+  end
+
+  defp build_index_payload(activity, embedding) do
+    %{
+      points: [
+        %{
+          id: activity.id |> FlakeId.from_string() |> Ecto.UUID.cast!(),
+          vector: embedding
+        }
+      ]
+    }
+  end
+
+  defp build_search_payload(embedding) do
+    %{
+      vector: embedding,
+      limit: 20
+    }
+  end
+
+  @impl true
+  def add_to_index(activity) do
+    # This will only index public or unlisted notes
+    maybe_search_data = object_to_search_data(activity.object)
+    IO.puts("TRYING TO INDEX\n\n")
+
+    if activity.data["type"] == "Create" and maybe_search_data do
+      with {:ok, embedding} <- get_embedding(maybe_search_data.content),
+           {:ok, %{status: 200}} <-
+             QdrantClient.put(
+               "/collections/posts/points",
+               build_index_payload(activity, embedding)
+             ) do
+        :ok
+      else
+        e -> {:error, e}
+      end
+    else
+      :ok
+    end
+  end
+
+  @impl true
+  def search(_user, query, _options) do
+    with {:ok, embedding} <- get_embedding(query),
+         {:ok, %{body: %{"result" => result}}} <-
+           QdrantClient.post("/collections/posts/points/search", build_search_payload(embedding)) do
+      ids =
+        Enum.map(result, fn %{"id" => id} ->
+          Ecto.UUID.dump!(id)
+        end)
+
+      from(a in Activity, where: a.id in ^ids)
+      |> Activity.with_preloaded_object()
+      |> Activity.restrict_deactivated_users()
+      |> Ecto.Query.order_by([a], fragment("array_position(?, ?)", ^ids, a.id))
+      |> Pleroma.Repo.all()
+    else
+      _ ->
+        []
+    end
+  end
+
+  @impl true
+  def remove_from_index(_object) do
+    :ok
+  end
+end
+
+defmodule Pleroma.Search.QdrantSearch.OllamaClient do
+  use Tesla
+
+  plug(Tesla.Middleware.BaseUrl, Pleroma.Config.get([Pleroma.Search.QdrantSearch, :ollama_url]))
+  plug(Tesla.Middleware.JSON)
+end
+
+defmodule Pleroma.Search.QdrantSearch.QdrantClient do
+  use Tesla
+
+  plug(Tesla.Middleware.BaseUrl, Pleroma.Config.get([Pleroma.Search.QdrantSearch, :qdrant_url]))
+  plug(Tesla.Middleware.JSON)
+
+  plug(Tesla.Middleware.Headers, [
+    {"api-key", Pleroma.Config.get([Pleroma.Search.QdrantSearch, :qdrant_api_key])}
+  ])
+end

From bb08a766f4c1bd84c98e245c1871c46fcc7c7a8d Mon Sep 17 00:00:00 2001
From: Lain Soykaf <lain@lain.com>
Date: Tue, 14 May 2024 14:26:41 +0400
Subject: [PATCH 040/192] QdrantSearch: Remove debugging stuff

---
 lib/pleroma/search/qdrant_search.ex | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/lib/pleroma/search/qdrant_search.ex b/lib/pleroma/search/qdrant_search.ex
index fcaa9e686..726a30b3b 100644
--- a/lib/pleroma/search/qdrant_search.ex
+++ b/lib/pleroma/search/qdrant_search.ex
@@ -22,8 +22,7 @@ defmodule Pleroma.Search.QdrantSearch do
            OllamaClient.post("/api/embeddings", %{
              prompt: text,
              model: Pleroma.Config.get([Pleroma.Search.QdrantSearch, :ollama_model])
-           })
-           |> IO.inspect() do
+           }) do
       {:ok, embedding}
     else
       _ ->
@@ -53,7 +52,6 @@ defmodule Pleroma.Search.QdrantSearch do
   def add_to_index(activity) do
     # This will only index public or unlisted notes
     maybe_search_data = object_to_search_data(activity.object)
-    IO.puts("TRYING TO INDEX\n\n")
 
     if activity.data["type"] == "Create" and maybe_search_data do
       with {:ok, embedding} <- get_embedding(maybe_search_data.content),

From 1490ff30af7001adc386b4fec54c62e1a524d7d6 Mon Sep 17 00:00:00 2001
From: Lain Soykaf <lain@lain.com>
Date: Tue, 14 May 2024 15:09:38 +0400
Subject: [PATCH 041/192] QdrantSearch: Add query prefix.

---
 lib/pleroma/search/qdrant_search.ex | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/lib/pleroma/search/qdrant_search.ex b/lib/pleroma/search/qdrant_search.ex
index 726a30b3b..31e7754ae 100644
--- a/lib/pleroma/search/qdrant_search.ex
+++ b/lib/pleroma/search/qdrant_search.ex
@@ -71,6 +71,8 @@ defmodule Pleroma.Search.QdrantSearch do
 
   @impl true
   def search(_user, query, _options) do
+    query = "Represent this sentence for searching relevant passages: #{query}"
+
     with {:ok, embedding} <- get_embedding(query),
          {:ok, %{body: %{"result" => result}}} <-
            QdrantClient.post("/collections/posts/points/search", build_search_payload(embedding)) do

From c50f0f31f418037063bd97efcdc0f60b89594212 Mon Sep 17 00:00:00 2001
From: Lain Soykaf <lain@lain.com>
Date: Tue, 14 May 2024 16:56:58 +0400
Subject: [PATCH 042/192] Docs/Search: Add basic documentation of the qdrant
 search

---
 docs/configuration/search.md | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/docs/configuration/search.md b/docs/configuration/search.md
index 0316c9bf4..682d1e52a 100644
--- a/docs/configuration/search.md
+++ b/docs/configuration/search.md
@@ -10,6 +10,12 @@ To use built-in search that has no external dependencies, set the search module
 
 While it has no external dependencies, it has problems with performance and relevancy.
 
+## QdrantSearch
+
+This uses the vector search engine [Qdrant](https://qdrant.tech) to search the posts in a vector space. This needs a way to generate embeddings, for now only the [Ollama](Ollama) api is supported.
+
+The default settings will support a setup where both Ollama and Qdrant run on the same system as pleroma. The embedding model used by Ollama will need to be pulled first (e.g. `ollama pull snowflake-arctic-embed:xs`) for the embedding to work.
+
 ## Meilisearch
 
 Note that it's quite a bit more memory hungry than PostgreSQL (around 4-5G for ~1.2 million

From 1261c43a7af48ed6e6753461944659391c4c58cc Mon Sep 17 00:00:00 2001
From: Lain Soykaf <lain@lain.com>
Date: Tue, 14 May 2024 17:19:36 +0400
Subject: [PATCH 043/192] SearchBackend: Add create_index

---
 lib/mix/tasks/pleroma/search/indexer.ex | 6 ++++++
 lib/pleroma/search/qdrant_search.ex     | 3 ++-
 lib/pleroma/search/search_backend.ex    | 5 +++++
 3 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/lib/mix/tasks/pleroma/search/indexer.ex b/lib/mix/tasks/pleroma/search/indexer.ex
index ffa2f3c94..326646b69 100644
--- a/lib/mix/tasks/pleroma/search/indexer.ex
+++ b/lib/mix/tasks/pleroma/search/indexer.ex
@@ -8,6 +8,12 @@ defmodule Mix.Tasks.Pleroma.Search.Indexer do
 
   alias Pleroma.Workers.SearchIndexingWorker
 
+  def run(["create_index"]) do
+    Application.ensure_all_started(:pleroma)
+
+    Pleroma.Config.get([Pleroma.Search, :module]).create_index()
+  end
+
   def run(["index" | options]) do
     {options, [], []} =
       OptionParser.parse(
diff --git a/lib/pleroma/search/qdrant_search.ex b/lib/pleroma/search/qdrant_search.ex
index 31e7754ae..315262cb3 100644
--- a/lib/pleroma/search/qdrant_search.ex
+++ b/lib/pleroma/search/qdrant_search.ex
@@ -8,7 +8,8 @@ defmodule Pleroma.Search.QdrantSearch do
 
   import Pleroma.Search.Meilisearch, only: [object_to_search_data: 1]
 
-  def initialize_index() do
+  @impl true
+  def create_index() do
     payload = Pleroma.Config.get([Pleroma.Search.QdrantSearch, :qdrant_index_configuration])
     QdrantClient.put("/collections/posts", payload)
   end
diff --git a/lib/pleroma/search/search_backend.ex b/lib/pleroma/search/search_backend.ex
index 68bc48cec..5be0169d0 100644
--- a/lib/pleroma/search/search_backend.ex
+++ b/lib/pleroma/search/search_backend.ex
@@ -21,4 +21,9 @@ defmodule Pleroma.Search.SearchBackend do
   from index.
   """
   @callback remove_from_index(object :: Pleroma.Object.t()) :: :ok | {:error, any()}
+
+  @doc """
+  Create the index
+  """
+  @callback create_index() :: :ok | {:error, any()}
 end

From a9be4907c0d7b34e5564584d2d040632c32f2aa3 Mon Sep 17 00:00:00 2001
From: Lain Soykaf <lain@lain.com>
Date: Thu, 16 May 2024 10:47:24 +0400
Subject: [PATCH 044/192] SearchBackend: Add drop_index

---
 lib/mix/tasks/pleroma/search/indexer.ex | 18 ++++++++++++++++--
 lib/pleroma/search/database_search.ex   |  6 ++++++
 lib/pleroma/search/meilisearch.ex       |  6 ++++++
 lib/pleroma/search/qdrant_search.ex     | 14 ++++++++++++--
 lib/pleroma/search/search_backend.ex    |  5 +++++
 5 files changed, 45 insertions(+), 4 deletions(-)

diff --git a/lib/mix/tasks/pleroma/search/indexer.ex b/lib/mix/tasks/pleroma/search/indexer.ex
index 326646b69..81a9fced6 100644
--- a/lib/mix/tasks/pleroma/search/indexer.ex
+++ b/lib/mix/tasks/pleroma/search/indexer.ex
@@ -9,9 +9,23 @@ defmodule Mix.Tasks.Pleroma.Search.Indexer do
   alias Pleroma.Workers.SearchIndexingWorker
 
   def run(["create_index"]) do
-    Application.ensure_all_started(:pleroma)
+    start_pleroma()
 
-    Pleroma.Config.get([Pleroma.Search, :module]).create_index()
+    with :ok <- Pleroma.Config.get([Pleroma.Search, :module]).create_index() do
+      IO.puts("Index created")
+    else
+      e -> IO.puts("Could not create index: #{inspect(e)}")
+    end
+  end
+
+  def run(["drop_index"]) do
+    start_pleroma()
+
+    with :ok <- Pleroma.Config.get([Pleroma.Search, :module]).drop_index() do
+      IO.puts("Index dropped")
+    else
+      e -> IO.puts("Could not drop index: #{inspect(e)}")
+    end
   end
 
   def run(["index" | options]) do
diff --git a/lib/pleroma/search/database_search.ex b/lib/pleroma/search/database_search.ex
index 31bfc7e33..24a1ff431 100644
--- a/lib/pleroma/search/database_search.ex
+++ b/lib/pleroma/search/database_search.ex
@@ -48,6 +48,12 @@ defmodule Pleroma.Search.DatabaseSearch do
   @impl true
   def remove_from_index(_object), do: :ok
 
+  @impl true
+  def create_index, do: :ok
+
+  @impl true
+  def drop_index, do: :ok
+
   def maybe_restrict_author(query, %User{} = author) do
     Activity.Queries.by_author(query, author)
   end
diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex
index 2bff663e8..50f5984d6 100644
--- a/lib/pleroma/search/meilisearch.ex
+++ b/lib/pleroma/search/meilisearch.ex
@@ -10,6 +10,12 @@ defmodule Pleroma.Search.Meilisearch do
 
   @behaviour Pleroma.Search.SearchBackend
 
+  @impl true
+  def create_index, do: :ok
+
+  @impl true
+  def drop_index, do: :ok
+
   defp meili_headers do
     private_key = Config.get([Pleroma.Search.Meilisearch, :private_key])
 
diff --git a/lib/pleroma/search/qdrant_search.ex b/lib/pleroma/search/qdrant_search.ex
index 315262cb3..4bd35c17c 100644
--- a/lib/pleroma/search/qdrant_search.ex
+++ b/lib/pleroma/search/qdrant_search.ex
@@ -11,11 +11,21 @@ defmodule Pleroma.Search.QdrantSearch do
   @impl true
   def create_index() do
     payload = Pleroma.Config.get([Pleroma.Search.QdrantSearch, :qdrant_index_configuration])
-    QdrantClient.put("/collections/posts", payload)
+
+    with {:ok, %{status: 200}} <- QdrantClient.put("/collections/posts", payload) do
+      :ok
+    else
+      e -> {:error, e}
+    end
   end
 
+  @impl true
   def drop_index() do
-    QdrantClient.delete("/collections/posts")
+    with {:ok, %{status: 200}} <- QdrantClient.delete("/collections/posts") do
+      :ok
+    else
+      e -> {:error, e}
+    end
   end
 
   def get_embedding(text) do
diff --git a/lib/pleroma/search/search_backend.ex b/lib/pleroma/search/search_backend.ex
index 5be0169d0..9735ab3f4 100644
--- a/lib/pleroma/search/search_backend.ex
+++ b/lib/pleroma/search/search_backend.ex
@@ -26,4 +26,9 @@ defmodule Pleroma.Search.SearchBackend do
   Create the index
   """
   @callback create_index() :: :ok | {:error, any()}
+
+  @doc """
+  Drop the index
+  """
+  @callback drop_index() :: :ok | {:error, any()}
 end

From 069ce4448c556af90293cde9b9872c3d53eb894b Mon Sep 17 00:00:00 2001
From: Lain Soykaf <lain@lain.com>
Date: Sat, 18 May 2024 11:55:17 +0400
Subject: [PATCH 045/192] Add basic fastembed server

---
 python/fastembed-server.py | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)
 create mode 100644 python/fastembed-server.py

diff --git a/python/fastembed-server.py b/python/fastembed-server.py
new file mode 100644
index 000000000..fa3f7c82b
--- /dev/null
+++ b/python/fastembed-server.py
@@ -0,0 +1,21 @@
+from fastembed import TextEmbedding
+from fastapi import FastAPI
+from pydantic import BaseModel
+
+model = TextEmbedding("snowflake/snowflake-arctic-embed-xs")
+
+app = FastAPI()
+
+class EmbeddingRequest(BaseModel):
+    model: str
+    prompt: str
+
+@app.post("/api/embeddings")
+def embeddings(request: EmbeddingRequest):
+    embeddings = next(model.embed(request.prompt)).tolist()
+    return {"embedding": embeddings}
+
+if __name__ == "__main__":
+    import uvicorn
+
+    uvicorn.run(app, host="0.0.0.0", port=11345)

From 769773a500d4c6ec021776b493f7d98c9f87e81e Mon Sep 17 00:00:00 2001
From: Lain Soykaf <lain@lain.com>
Date: Sat, 18 May 2024 12:08:42 +0400
Subject: [PATCH 046/192] Add dockerfile

---
 python/Dockerfile | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 python/Dockerfile

diff --git a/python/Dockerfile b/python/Dockerfile
new file mode 100644
index 000000000..f83c1c1b3
--- /dev/null
+++ b/python/Dockerfile
@@ -0,0 +1,8 @@
+FROM python:3.9
+
+WORKDIR /code
+COPY fastembed-server.py /workdir/fastembed-server.py
+
+RUN pip install --no-cache-dir --upgrade fastembed fastapi uvicorn
+
+CMD ["python", "/workdir/fastembed-server.py"]

From 61e9027131843858b017d3b7c18c3a396d5656a9 Mon Sep 17 00:00:00 2001
From: Lain Soykaf <lain@lain.com>
Date: Sat, 18 May 2024 12:19:42 +0400
Subject: [PATCH 047/192] Add docker compose file for fastembed server

---
 python/compose.yml | 5 +++++
 1 file changed, 5 insertions(+)
 create mode 100644 python/compose.yml

diff --git a/python/compose.yml b/python/compose.yml
new file mode 100644
index 000000000..d4cb31722
--- /dev/null
+++ b/python/compose.yml
@@ -0,0 +1,5 @@
+services:
+  web:
+    build: .
+    ports:
+      - "11345:11345"

From 933117785fb1b5b671c61d09671cf6418b105187 Mon Sep 17 00:00:00 2001
From: Lain Soykaf <lain@lain.com>
Date: Sat, 18 May 2024 13:43:47 +0400
Subject: [PATCH 048/192] QdrantSearch: Add basic test

---
 lib/pleroma/search/qdrant_search.ex        | 11 ++--
 test/pleroma/search/qdrant_search_test.exs | 65 ++++++++++++++++++++++
 2 files changed, 72 insertions(+), 4 deletions(-)
 create mode 100644 test/pleroma/search/qdrant_search_test.exs

diff --git a/lib/pleroma/search/qdrant_search.ex b/lib/pleroma/search/qdrant_search.ex
index 4bd35c17c..2d9315a2f 100644
--- a/lib/pleroma/search/qdrant_search.ex
+++ b/lib/pleroma/search/qdrant_search.ex
@@ -5,12 +5,13 @@ defmodule Pleroma.Search.QdrantSearch do
 
   alias __MODULE__.QdrantClient
   alias __MODULE__.OllamaClient
+  alias Pleroma.Config.Getting, as: Config
 
   import Pleroma.Search.Meilisearch, only: [object_to_search_data: 1]
 
   @impl true
   def create_index() do
-    payload = Pleroma.Config.get([Pleroma.Search.QdrantSearch, :qdrant_index_configuration])
+    payload = Config.get([Pleroma.Search.QdrantSearch, :qdrant_index_configuration])
 
     with {:ok, %{status: 200}} <- QdrantClient.put("/collections/posts", payload) do
       :ok
@@ -32,7 +33,7 @@ defmodule Pleroma.Search.QdrantSearch do
     with {:ok, %{body: %{"embedding" => embedding}}} <-
            OllamaClient.post("/api/embeddings", %{
              prompt: text,
-             model: Pleroma.Config.get([Pleroma.Search.QdrantSearch, :ollama_model])
+             model: Config.get([Pleroma.Search.QdrantSearch, :ollama_model])
            }) do
       {:ok, embedding}
     else
@@ -111,15 +112,17 @@ end
 
 defmodule Pleroma.Search.QdrantSearch.OllamaClient do
   use Tesla
+  alias Pleroma.Config.Getting, as: Config
 
-  plug(Tesla.Middleware.BaseUrl, Pleroma.Config.get([Pleroma.Search.QdrantSearch, :ollama_url]))
+  plug(Tesla.Middleware.BaseUrl, Config.get([Pleroma.Search.QdrantSearch, :ollama_url]))
   plug(Tesla.Middleware.JSON)
 end
 
 defmodule Pleroma.Search.QdrantSearch.QdrantClient do
   use Tesla
+  alias Pleroma.Config.Getting, as: Config
 
-  plug(Tesla.Middleware.BaseUrl, Pleroma.Config.get([Pleroma.Search.QdrantSearch, :qdrant_url]))
+  plug(Tesla.Middleware.BaseUrl, Config.get([Pleroma.Search.QdrantSearch, :qdrant_url]))
   plug(Tesla.Middleware.JSON)
 
   plug(Tesla.Middleware.Headers, [
diff --git a/test/pleroma/search/qdrant_search_test.exs b/test/pleroma/search/qdrant_search_test.exs
new file mode 100644
index 000000000..9be246a9a
--- /dev/null
+++ b/test/pleroma/search/qdrant_search_test.exs
@@ -0,0 +1,65 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Search.QdrantSearchTest do
+  use Pleroma.DataCase, async: true
+  use Oban.Testing, repo: Pleroma.Repo
+
+  import Pleroma.Factory
+  import Mox
+
+  alias Pleroma.Web.CommonAPI
+  alias Pleroma.UnstubbedConfigMock, as: Config
+  alias Pleroma.Search.QdrantSearch
+  alias Pleroma.Workers.SearchIndexingWorker
+
+  describe "Qdrant search" do
+    test "indexes a public post on creation" do
+      user = insert(:user)
+
+      Tesla.Mock.mock(fn
+        %{method: :post, url: "https://ollama.url/api/embeddings"} ->
+          send(self(), "posted_to_ollama")
+          Tesla.Mock.json(%{embedding: [1, 2, 3]})
+
+        %{method: :put, url: "https://qdrant.url/collections/posts/points", body: body} ->
+          send(self(), "posted_to_qdrant")
+
+          assert match?(%{"points" => [%{"vector" => [1, 2, 3]}]}, Jason.decode!(body))
+
+          Tesla.Mock.json("ok")
+      end)
+
+      Config
+      |> expect(:get, 4, fn
+        [Pleroma.Search, :module], nil ->
+          QdrantSearch
+
+        [Pleroma.Search.QdrantSearch, key], nil ->
+          %{
+            ollama_model: "a_model",
+            ollama_url: "https://ollama.url",
+            qdrant_url: "https://qdrant.url"
+          }[key]
+      end)
+
+      {:ok, activity} =
+        CommonAPI.post(user, %{
+          status: "guys i just don't wanna leave the swamp",
+          visibility: "public"
+        })
+
+      args = %{"op" => "add_to_index", "activity" => activity.id}
+
+      assert_enqueued(
+        worker: SearchIndexingWorker,
+        args: args
+      )
+
+      assert :ok = perform_job(SearchIndexingWorker, args)
+      assert_received("posted_to_ollama")
+      assert_received("posted_to_qdrant")
+    end
+  end
+end

From e3933a067feae1f087616f675657d6ff99b2782b Mon Sep 17 00:00:00 2001
From: Lain Soykaf <lain@lain.com>
Date: Sat, 18 May 2024 14:04:32 +0400
Subject: [PATCH 049/192] QdrantSearch: Implement post deletion

---
 lib/pleroma/search/qdrant_search.ex        | 18 +++++++++++++-----
 test/pleroma/search/qdrant_search_test.exs | 16 ++++++++++++++--
 2 files changed, 27 insertions(+), 7 deletions(-)

diff --git a/lib/pleroma/search/qdrant_search.ex b/lib/pleroma/search/qdrant_search.ex
index 2d9315a2f..acfaaff52 100644
--- a/lib/pleroma/search/qdrant_search.ex
+++ b/lib/pleroma/search/qdrant_search.ex
@@ -81,6 +81,19 @@ defmodule Pleroma.Search.QdrantSearch do
     end
   end
 
+  @impl true
+  def remove_from_index(object) do
+    activity = Activity.get_by_object_ap_id_with_object(object.data["id"])
+    id = activity.id |> FlakeId.from_string() |> Ecto.UUID.cast!()
+
+    with {:ok, %{status: 200}} <-
+           QdrantClient.post("/collections/posts/points/delete", %{"points" => [id]}) do
+      :ok
+    else
+      e -> {:error, e}
+    end
+  end
+
   @impl true
   def search(_user, query, _options) do
     query = "Represent this sentence for searching relevant passages: #{query}"
@@ -103,11 +116,6 @@ defmodule Pleroma.Search.QdrantSearch do
         []
     end
   end
-
-  @impl true
-  def remove_from_index(_object) do
-    :ok
-  end
 end
 
 defmodule Pleroma.Search.QdrantSearch.OllamaClient do
diff --git a/test/pleroma/search/qdrant_search_test.exs b/test/pleroma/search/qdrant_search_test.exs
index 9be246a9a..e816311aa 100644
--- a/test/pleroma/search/qdrant_search_test.exs
+++ b/test/pleroma/search/qdrant_search_test.exs
@@ -15,7 +15,7 @@ defmodule Pleroma.Search.QdrantSearchTest do
   alias Pleroma.Workers.SearchIndexingWorker
 
   describe "Qdrant search" do
-    test "indexes a public post on creation" do
+    test "indexes a public post on creation, deletes from the index on deletion" do
       user = insert(:user)
 
       Tesla.Mock.mock(fn
@@ -29,10 +29,14 @@ defmodule Pleroma.Search.QdrantSearchTest do
           assert match?(%{"points" => [%{"vector" => [1, 2, 3]}]}, Jason.decode!(body))
 
           Tesla.Mock.json("ok")
+
+        %{method: :post, url: "https://qdrant.url/collections/posts/points/delete"} ->
+          send(self(), "deleted_from_qdrant")
+          Tesla.Mock.json("ok")
       end)
 
       Config
-      |> expect(:get, 4, fn
+      |> expect(:get, 6, fn
         [Pleroma.Search, :module], nil ->
           QdrantSearch
 
@@ -60,6 +64,14 @@ defmodule Pleroma.Search.QdrantSearchTest do
       assert :ok = perform_job(SearchIndexingWorker, args)
       assert_received("posted_to_ollama")
       assert_received("posted_to_qdrant")
+
+      {:ok, _} = CommonAPI.delete(activity.id, user)
+
+      delete_args = %{"op" => "remove_from_index", "object" => activity.object.id}
+      assert_enqueued(worker: SearchIndexingWorker, args: delete_args)
+      assert :ok = perform_job(SearchIndexingWorker, delete_args)
+
+      assert_received("deleted_from_qdrant")
     end
   end
 end

From 39525bcec7c685cb28ca4702b6e145a78e733fee Mon Sep 17 00:00:00 2001
From: Lain Soykaf <lain@lain.com>
Date: Sat, 18 May 2024 14:07:47 +0400
Subject: [PATCH 050/192] Add qdrant changelog

---
 changelog.d/qdrant_search.add | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/qdrant_search.add

diff --git a/changelog.d/qdrant_search.add b/changelog.d/qdrant_search.add
new file mode 100644
index 000000000..6f9e39e23
--- /dev/null
+++ b/changelog.d/qdrant_search.add
@@ -0,0 +1 @@
+Add Qdrant/Ollama search

From 3345ddd2d4ef380929cc231118a5fb6486c0bd5c Mon Sep 17 00:00:00 2001
From: Lain Soykaf <lain@lain.com>
Date: Sat, 18 May 2024 15:02:22 +0400
Subject: [PATCH 051/192] Linting

---
 lib/pleroma/search/qdrant_search.ex        | 11 ++++++-----
 test/pleroma/search/qdrant_search_test.exs |  4 ++--
 2 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/lib/pleroma/search/qdrant_search.ex b/lib/pleroma/search/qdrant_search.ex
index acfaaff52..a6c6c6a0d 100644
--- a/lib/pleroma/search/qdrant_search.ex
+++ b/lib/pleroma/search/qdrant_search.ex
@@ -1,16 +1,17 @@
 defmodule Pleroma.Search.QdrantSearch do
   @behaviour Pleroma.Search.SearchBackend
   import Ecto.Query
-  alias Pleroma.Activity
 
-  alias __MODULE__.QdrantClient
-  alias __MODULE__.OllamaClient
+  alias Pleroma.Activity
   alias Pleroma.Config.Getting, as: Config
 
+  alias __MODULE__.OllamaClient
+  alias __MODULE__.QdrantClient
+
   import Pleroma.Search.Meilisearch, only: [object_to_search_data: 1]
 
   @impl true
-  def create_index() do
+  def create_index do
     payload = Config.get([Pleroma.Search.QdrantSearch, :qdrant_index_configuration])
 
     with {:ok, %{status: 200}} <- QdrantClient.put("/collections/posts", payload) do
@@ -21,7 +22,7 @@ defmodule Pleroma.Search.QdrantSearch do
   end
 
   @impl true
-  def drop_index() do
+  def drop_index do
     with {:ok, %{status: 200}} <- QdrantClient.delete("/collections/posts") do
       :ok
     else
diff --git a/test/pleroma/search/qdrant_search_test.exs b/test/pleroma/search/qdrant_search_test.exs
index e816311aa..698894cdb 100644
--- a/test/pleroma/search/qdrant_search_test.exs
+++ b/test/pleroma/search/qdrant_search_test.exs
@@ -9,9 +9,9 @@ defmodule Pleroma.Search.QdrantSearchTest do
   import Pleroma.Factory
   import Mox
 
-  alias Pleroma.Web.CommonAPI
-  alias Pleroma.UnstubbedConfigMock, as: Config
   alias Pleroma.Search.QdrantSearch
+  alias Pleroma.UnstubbedConfigMock, as: Config
+  alias Pleroma.Web.CommonAPI
   alias Pleroma.Workers.SearchIndexingWorker
 
   describe "Qdrant search" do

From d07d49227fd3bf716fa22e402685f27e31a0f6d3 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Thu, 16 May 2024 16:26:21 -0400
Subject: [PATCH 052/192] PleromaAPI: marking notifications as read no longer
 returns notifications

---
 changelog.d/mark-read.fix                     |  1 +
 lib/pleroma/notification.ex                   | 17 +++---------
 .../pleroma_notification_operation.ex         |  8 +-----
 .../controllers/notification_controller.ex    | 20 ++++++++------
 test/pleroma/notification_test.exs            |  4 +--
 .../notification_controller_test.exs          | 27 ++++++++++---------
 6 files changed, 33 insertions(+), 44 deletions(-)
 create mode 100644 changelog.d/mark-read.fix

diff --git a/changelog.d/mark-read.fix b/changelog.d/mark-read.fix
new file mode 100644
index 000000000..346eb19e2
--- /dev/null
+++ b/changelog.d/mark-read.fix
@@ -0,0 +1 @@
+The query for marking notifications as read has been simplified
diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex
index a80279fa6..cb9bd92b8 100644
--- a/lib/pleroma/notification.ex
+++ b/lib/pleroma/notification.ex
@@ -280,15 +280,10 @@ defmodule Pleroma.Notification do
         select: n.id
       )
 
-    {:ok, %{ids: {_, notification_ids}}} =
-      Multi.new()
-      |> Multi.update_all(:ids, query, set: [seen: true, updated_at: NaiveDateTime.utc_now()])
-      |> Marker.multi_set_last_read_id(user, "notifications")
-      |> Repo.transaction()
-
-    for_user_query(user)
-    |> where([n], n.id in ^notification_ids)
-    |> Repo.all()
+    Multi.new()
+    |> Multi.update_all(:ids, query, set: [seen: true, updated_at: NaiveDateTime.utc_now()])
+    |> Marker.multi_set_last_read_id(user, "notifications")
+    |> Repo.transaction()
   end
 
   @spec read_one(User.t(), String.t()) ::
@@ -299,10 +294,6 @@ defmodule Pleroma.Notification do
       |> Multi.update(:update, changeset(notification, %{seen: true}))
       |> Marker.multi_set_last_read_id(user, "notifications")
       |> Repo.transaction()
-      |> case do
-        {:ok, %{update: notification}} -> {:ok, notification}
-        {:error, :update, changeset, _} -> {:error, changeset}
-      end
     end
   end
 
diff --git a/lib/pleroma/web/api_spec/operations/pleroma_notification_operation.ex b/lib/pleroma/web/api_spec/operations/pleroma_notification_operation.ex
index a994345db..0e2865191 100644
--- a/lib/pleroma/web/api_spec/operations/pleroma_notification_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/pleroma_notification_operation.ex
@@ -5,7 +5,6 @@
 defmodule Pleroma.Web.ApiSpec.PleromaNotificationOperation do
   alias OpenApiSpex.Operation
   alias OpenApiSpex.Schema
-  alias Pleroma.Web.ApiSpec.NotificationOperation
   alias Pleroma.Web.ApiSpec.Schemas.ApiError
 
   import Pleroma.Web.ApiSpec.Helpers
@@ -35,12 +34,7 @@ defmodule Pleroma.Web.ApiSpec.PleromaNotificationOperation do
           Operation.response(
             "A Notification or array of Notifications",
             "application/json",
-            %Schema{
-              anyOf: [
-                %Schema{type: :array, items: NotificationOperation.notification()},
-                NotificationOperation.notification()
-              ]
-            }
+            %Schema{type: :string}
           ),
         400 => Operation.response("Bad Request", "application/json", ApiError)
       }
diff --git a/lib/pleroma/web/pleroma_api/controllers/notification_controller.ex b/lib/pleroma/web/pleroma_api/controllers/notification_controller.ex
index f860eaf7e..435ccfabe 100644
--- a/lib/pleroma/web/pleroma_api/controllers/notification_controller.ex
+++ b/lib/pleroma/web/pleroma_api/controllers/notification_controller.ex
@@ -23,8 +23,9 @@ defmodule Pleroma.Web.PleromaAPI.NotificationController do
         } = conn,
         _
       ) do
-    with {:ok, notification} <- Notification.read_one(user, notification_id) do
-      render(conn, "show.json", notification: notification, for: user)
+    with {:ok, _} <- Notification.read_one(user, notification_id) do
+      conn
+      |> json("ok")
     else
       {:error, message} ->
         conn
@@ -38,11 +39,14 @@ defmodule Pleroma.Web.PleromaAPI.NotificationController do
           conn,
         _
       ) do
-    notifications =
-      user
-      |> Notification.set_read_up_to(max_id)
-      |> Enum.take(80)
-
-    render(conn, "index.json", notifications: notifications, for: user)
+    with {:ok, _} <- Notification.set_read_up_to(user, max_id) do
+      conn
+      |> json("ok")
+    else
+      {:error, message} ->
+        conn
+        |> put_status(:bad_request)
+        |> json(%{"error" => message})
+    end
   end
 end
diff --git a/test/pleroma/notification_test.exs b/test/pleroma/notification_test.exs
index 392fd53c2..1dd0c26f7 100644
--- a/test/pleroma/notification_test.exs
+++ b/test/pleroma/notification_test.exs
@@ -449,9 +449,7 @@ defmodule Pleroma.NotificationTest do
           status: "hey yet again @#{other_user.nickname}!"
         })
 
-      [_, read_notification] = Notification.set_read_up_to(other_user, n2.id)
-
-      assert read_notification.activity.object
+      Notification.set_read_up_to(other_user, n2.id)
 
       [n3, n2, n1] = Notification.for_user(other_user)
 
diff --git a/test/pleroma/web/pleroma_api/controllers/notification_controller_test.exs b/test/pleroma/web/pleroma_api/controllers/notification_controller_test.exs
index b8c7964f9..036cbf176 100644
--- a/test/pleroma/web/pleroma_api/controllers/notification_controller_test.exs
+++ b/test/pleroma/web/pleroma_api/controllers/notification_controller_test.exs
@@ -21,13 +21,11 @@ defmodule Pleroma.Web.PleromaAPI.NotificationControllerTest do
       {:ok, [notification1]} = Notification.create_notifications(activity1)
       {:ok, [notification2]} = Notification.create_notifications(activity2)
 
-      response =
-        conn
-        |> put_req_header("content-type", "application/json")
-        |> post("/api/v1/pleroma/notifications/read", %{id: notification1.id})
-        |> json_response_and_validate_schema(:ok)
+      conn
+      |> put_req_header("content-type", "application/json")
+      |> post("/api/v1/pleroma/notifications/read", %{id: notification1.id})
+      |> json_response_and_validate_schema(:ok)
 
-      assert %{"pleroma" => %{"is_seen" => true}} = response
       assert Repo.get(Notification, notification1.id).seen
       refute Repo.get(Notification, notification2.id).seen
     end
@@ -40,14 +38,17 @@ defmodule Pleroma.Web.PleromaAPI.NotificationControllerTest do
 
       [notification3, notification2, notification1] = Notification.for_user(user1, %{limit: 3})
 
-      [response1, response2] =
-        conn
-        |> put_req_header("content-type", "application/json")
-        |> post("/api/v1/pleroma/notifications/read", %{max_id: notification2.id})
-        |> json_response_and_validate_schema(:ok)
+      refute Repo.get(Notification, notification1.id).seen
+      refute Repo.get(Notification, notification2.id).seen
+      refute Repo.get(Notification, notification3.id).seen
+
+      conn
+      |> put_req_header("content-type", "application/json")
+      |> post("/api/v1/pleroma/notifications/read", %{max_id: notification2.id})
+      |> json_response_and_validate_schema(:ok)
+
+      [notification3, notification2, notification1] = Notification.for_user(user1, %{limit: 3})
 
-      assert %{"pleroma" => %{"is_seen" => true}} = response1
-      assert %{"pleroma" => %{"is_seen" => true}} = response2
       assert Repo.get(Notification, notification1.id).seen
       assert Repo.get(Notification, notification2.id).seen
       refute Repo.get(Notification, notification3.id).seen

From 72ec261a69a7dda7ab95667e425824ab7758b636 Mon Sep 17 00:00:00 2001
From: Lain Soykaf <lain@lain.com>
Date: Sun, 19 May 2024 12:17:46 +0400
Subject: [PATCH 053/192] B QdrantSearch: Switch to OpenAI api

---
 changelog.d/qdrant_search.add              |  2 +-
 config/config.exs                          |  7 ++++---
 lib/pleroma/search/qdrant_search.ex        | 19 ++++++++++++-------
 test/pleroma/search/qdrant_search_test.exs | 15 +++++++++------
 4 files changed, 26 insertions(+), 17 deletions(-)

diff --git a/changelog.d/qdrant_search.add b/changelog.d/qdrant_search.add
index 6f9e39e23..9801131d1 100644
--- a/changelog.d/qdrant_search.add
+++ b/changelog.d/qdrant_search.add
@@ -1 +1 @@
-Add Qdrant/Ollama search
+Add Qdrant/OpenAI embedding search
diff --git a/config/config.exs b/config/config.exs
index f74eda6b2..dd0150c66 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -917,9 +917,10 @@ config :pleroma, Pleroma.Uploaders.Uploader, timeout: 30_000
 
 config :pleroma, Pleroma.Search.QdrantSearch,
   qdrant_url: "http://127.0.0.1:6333/",
-  qdrant_api_key: nil,
-  ollama_url: "http://127.0.0.1:11434",
-  ollama_model: "snowflake-arctic-embed:xs",
+  qdrant_api_key: "",
+  openai_url: "http://127.0.0.1:11345",
+  openai_model: "snowflake",
+  openai_api_key: "",
   qdrant_index_configuration: %{
     vectors: %{size: 384, distance: "Cosine"}
   }
diff --git a/lib/pleroma/search/qdrant_search.ex b/lib/pleroma/search/qdrant_search.ex
index a6c6c6a0d..5ae04be78 100644
--- a/lib/pleroma/search/qdrant_search.ex
+++ b/lib/pleroma/search/qdrant_search.ex
@@ -5,7 +5,7 @@ defmodule Pleroma.Search.QdrantSearch do
   alias Pleroma.Activity
   alias Pleroma.Config.Getting, as: Config
 
-  alias __MODULE__.OllamaClient
+  alias __MODULE__.OpenAIClient
   alias __MODULE__.QdrantClient
 
   import Pleroma.Search.Meilisearch, only: [object_to_search_data: 1]
@@ -31,10 +31,10 @@ defmodule Pleroma.Search.QdrantSearch do
   end
 
   def get_embedding(text) do
-    with {:ok, %{body: %{"embedding" => embedding}}} <-
-           OllamaClient.post("/api/embeddings", %{
-             prompt: text,
-             model: Config.get([Pleroma.Search.QdrantSearch, :ollama_model])
+    with {:ok, %{body: %{"data" => [%{"embedding" => embedding}]}}} <-
+           OpenAIClient.post("/v1/embeddings", %{
+             input: text,
+             model: Config.get([Pleroma.Search.QdrantSearch, :openai_model])
            }) do
       {:ok, embedding}
     else
@@ -119,12 +119,17 @@ defmodule Pleroma.Search.QdrantSearch do
   end
 end
 
-defmodule Pleroma.Search.QdrantSearch.OllamaClient do
+defmodule Pleroma.Search.QdrantSearch.OpenAIClient do
   use Tesla
   alias Pleroma.Config.Getting, as: Config
 
-  plug(Tesla.Middleware.BaseUrl, Config.get([Pleroma.Search.QdrantSearch, :ollama_url]))
+  plug(Tesla.Middleware.BaseUrl, Config.get([Pleroma.Search.QdrantSearch, :openai_url]))
   plug(Tesla.Middleware.JSON)
+
+  plug(Tesla.Middleware.Headers, [
+    {"Authorization",
+     "Bearer #{Pleroma.Config.get([Pleroma.Search.QdrantSearch, :openai_api_key])}"}
+  ])
 end
 
 defmodule Pleroma.Search.QdrantSearch.QdrantClient do
diff --git a/test/pleroma/search/qdrant_search_test.exs b/test/pleroma/search/qdrant_search_test.exs
index 698894cdb..a2f9cc7ec 100644
--- a/test/pleroma/search/qdrant_search_test.exs
+++ b/test/pleroma/search/qdrant_search_test.exs
@@ -19,9 +19,12 @@ defmodule Pleroma.Search.QdrantSearchTest do
       user = insert(:user)
 
       Tesla.Mock.mock(fn
-        %{method: :post, url: "https://ollama.url/api/embeddings"} ->
-          send(self(), "posted_to_ollama")
-          Tesla.Mock.json(%{embedding: [1, 2, 3]})
+        %{method: :post, url: "https://openai.url/v1/embeddings"} ->
+          send(self(), "posted_to_openai")
+
+          Tesla.Mock.json(%{
+            data: [%{embedding: [1, 2, 3]}]
+          })
 
         %{method: :put, url: "https://qdrant.url/collections/posts/points", body: body} ->
           send(self(), "posted_to_qdrant")
@@ -42,8 +45,8 @@ defmodule Pleroma.Search.QdrantSearchTest do
 
         [Pleroma.Search.QdrantSearch, key], nil ->
           %{
-            ollama_model: "a_model",
-            ollama_url: "https://ollama.url",
+            openai_model: "a_model",
+            openai_url: "https://openai.url",
             qdrant_url: "https://qdrant.url"
           }[key]
       end)
@@ -62,7 +65,7 @@ defmodule Pleroma.Search.QdrantSearchTest do
       )
 
       assert :ok = perform_job(SearchIndexingWorker, args)
-      assert_received("posted_to_ollama")
+      assert_received("posted_to_openai")
       assert_received("posted_to_qdrant")
 
       {:ok, _} = CommonAPI.delete(activity.id, user)

From b9af017a4cf1025c7d8245fa4f1dbcb678ddd4b9 Mon Sep 17 00:00:00 2001
From: Lain Soykaf <lain@lain.com>
Date: Sun, 19 May 2024 12:33:49 +0400
Subject: [PATCH 054/192] B FastembedServer: Switch to OpenAI api, support
 changing models

---
 python/fastembed-server.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/python/fastembed-server.py b/python/fastembed-server.py
index fa3f7c82b..dd4a7a9c8 100644
--- a/python/fastembed-server.py
+++ b/python/fastembed-server.py
@@ -2,18 +2,20 @@ from fastembed import TextEmbedding
 from fastapi import FastAPI
 from pydantic import BaseModel
 
-model = TextEmbedding("snowflake/snowflake-arctic-embed-xs")
+models = {}
 
 app = FastAPI()
 
 class EmbeddingRequest(BaseModel):
     model: str
-    prompt: str
+    input: str
 
-@app.post("/api/embeddings")
+@app.post("/v1/embeddings")
 def embeddings(request: EmbeddingRequest):
-    embeddings = next(model.embed(request.prompt)).tolist()
-    return {"embedding": embeddings}
+    model = models.get(request.model) or TextEmbedding(request.model)
+    models[request.model] = model
+    embeddings = next(model.embed(request.input)).tolist()
+    return {"data": [{"embedding": embeddings}]}
 
 if __name__ == "__main__":
     import uvicorn

From c139a9f38c06ab4485b98b56b9ad4cce4d57be12 Mon Sep 17 00:00:00 2001
From: Lain Soykaf <lain@lain.com>
Date: Sun, 19 May 2024 12:39:54 +0400
Subject: [PATCH 055/192] B Config: Set default Qdrant embedder to our
 fastembed-api server

---
 config/config.exs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/config/config.exs b/config/config.exs
index dd0150c66..c3b20947d 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -919,7 +919,7 @@ config :pleroma, Pleroma.Search.QdrantSearch,
   qdrant_url: "http://127.0.0.1:6333/",
   qdrant_api_key: "",
   openai_url: "http://127.0.0.1:11345",
-  openai_model: "snowflake",
+  openai_model: "snowflake/snowflake-arctic-embed-xs",
   openai_api_key: "",
   qdrant_index_configuration: %{
     vectors: %{size: 384, distance: "Cosine"}

From e142ea400a9ed3595f8d432edd90ea26fc7d2eb5 Mon Sep 17 00:00:00 2001
From: Lain Soykaf <lain@lain.com>
Date: Sun, 19 May 2024 12:42:08 +0400
Subject: [PATCH 056/192] Docs: Switch docs from Ollama to OpenAI.

---
 docs/configuration/search.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/configuration/search.md b/docs/configuration/search.md
index 682d1e52a..388f5acd1 100644
--- a/docs/configuration/search.md
+++ b/docs/configuration/search.md
@@ -12,9 +12,9 @@ While it has no external dependencies, it has problems with performance and rele
 
 ## QdrantSearch
 
-This uses the vector search engine [Qdrant](https://qdrant.tech) to search the posts in a vector space. This needs a way to generate embeddings, for now only the [Ollama](Ollama) api is supported.
+This uses the vector search engine [Qdrant](https://qdrant.tech) to search the posts in a vector space. This needs a way to generate embeddings and uses the [OpenAI API](https://platform.openai.com/docs/guides/embeddings/what-are-embeddings). This is implemented by several project besides OpenAI itself, including the python-based fastembed-server found in `supplemental/search/fastembed-api`.
 
-The default settings will support a setup where both Ollama and Qdrant run on the same system as pleroma. The embedding model used by Ollama will need to be pulled first (e.g. `ollama pull snowflake-arctic-embed:xs`) for the embedding to work.
+The default settings will support a setup where both the fastembed server and Qdrant run on the same system as pleroma.
 
 ## Meilisearch
 

From dd48810186e3b4ee14e1d3727f37bd470d0711a4 Mon Sep 17 00:00:00 2001
From: Lain Soykaf <lain@lain.com>
Date: Sun, 19 May 2024 12:47:08 +0400
Subject: [PATCH 057/192] B FastembedAPI: Move to more appropriate folder

---
 {python => supplemental/search/fastembed-api}/Dockerfile          | 0
 {python => supplemental/search/fastembed-api}/compose.yml         | 0
 {python => supplemental/search/fastembed-api}/fastembed-server.py | 0
 3 files changed, 0 insertions(+), 0 deletions(-)
 rename {python => supplemental/search/fastembed-api}/Dockerfile (100%)
 rename {python => supplemental/search/fastembed-api}/compose.yml (100%)
 rename {python => supplemental/search/fastembed-api}/fastembed-server.py (100%)

diff --git a/python/Dockerfile b/supplemental/search/fastembed-api/Dockerfile
similarity index 100%
rename from python/Dockerfile
rename to supplemental/search/fastembed-api/Dockerfile
diff --git a/python/compose.yml b/supplemental/search/fastembed-api/compose.yml
similarity index 100%
rename from python/compose.yml
rename to supplemental/search/fastembed-api/compose.yml
diff --git a/python/fastembed-server.py b/supplemental/search/fastembed-api/fastembed-server.py
similarity index 100%
rename from python/fastembed-server.py
rename to supplemental/search/fastembed-api/fastembed-server.py

From 8329ad521419119f89e3e2577269475190cfe921 Mon Sep 17 00:00:00 2001
From: Lain Soykaf <lain@lain.com>
Date: Sun, 19 May 2024 12:59:03 +0400
Subject: [PATCH 058/192] B FastembedAPI: Add requirements.txt

---
 supplemental/search/fastembed-api/Dockerfile       | 3 ++-
 supplemental/search/fastembed-api/requirements.txt | 4 ++++
 2 files changed, 6 insertions(+), 1 deletion(-)
 create mode 100644 supplemental/search/fastembed-api/requirements.txt

diff --git a/supplemental/search/fastembed-api/Dockerfile b/supplemental/search/fastembed-api/Dockerfile
index f83c1c1b3..c1e0ef51f 100644
--- a/supplemental/search/fastembed-api/Dockerfile
+++ b/supplemental/search/fastembed-api/Dockerfile
@@ -2,7 +2,8 @@ FROM python:3.9
 
 WORKDIR /code
 COPY fastembed-server.py /workdir/fastembed-server.py
+COPY requirements.txt /workdir/requirements.txt
 
-RUN pip install --no-cache-dir --upgrade fastembed fastapi uvicorn
+RUN pip install -r /workdir/requirements.txt
 
 CMD ["python", "/workdir/fastembed-server.py"]
diff --git a/supplemental/search/fastembed-api/requirements.txt b/supplemental/search/fastembed-api/requirements.txt
new file mode 100644
index 000000000..db67a8402
--- /dev/null
+++ b/supplemental/search/fastembed-api/requirements.txt
@@ -0,0 +1,4 @@
+fastapi==0.111.0
+fastembed==0.2.7
+pydantic==1.10.15
+uvicorn==0.29.0

From 23881842ae33a294e344cef0cc2f1385ea6819f9 Mon Sep 17 00:00:00 2001
From: Lain Soykaf <lain@lain.com>
Date: Sun, 19 May 2024 13:04:27 +0400
Subject: [PATCH 059/192] B FastembedAPI: Add readme

---
 supplemental/search/fastembed-api/README.md | 6 ++++++
 1 file changed, 6 insertions(+)
 create mode 100644 supplemental/search/fastembed-api/README.md

diff --git a/supplemental/search/fastembed-api/README.md b/supplemental/search/fastembed-api/README.md
new file mode 100644
index 000000000..63a037207
--- /dev/null
+++ b/supplemental/search/fastembed-api/README.md
@@ -0,0 +1,6 @@
+# About
+This is a minimal implementation of the [OpenAI Embeddings API](https://platform.openai.com/docs/guides/embeddings/what-are-embeddings) meant to be used with the QdrantSearch backend. 
+
+# Usage
+
+The easiest way to run it is to just use docker compose with `docker compose up`. This starts the server on the default configured port. Different models can be used, for a full list of supported models, check the [fastembed documentation](https://qdrant.github.io/fastembed/examples/Supported_Models/). The first time a model is requested it will be downloaded, which can take a few seconds.

From 6a3a0cc0f5995185428c92f3c53e9c8524ea6856 Mon Sep 17 00:00:00 2001
From: Lain Soykaf <lain@lain.com>
Date: Sun, 19 May 2024 13:20:37 +0400
Subject: [PATCH 060/192] Docs: Write docs for the QdrantSearch

---
 docs/configuration/search.md | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/docs/configuration/search.md b/docs/configuration/search.md
index 388f5acd1..6598e533f 100644
--- a/docs/configuration/search.md
+++ b/docs/configuration/search.md
@@ -14,7 +14,25 @@ While it has no external dependencies, it has problems with performance and rele
 
 This uses the vector search engine [Qdrant](https://qdrant.tech) to search the posts in a vector space. This needs a way to generate embeddings and uses the [OpenAI API](https://platform.openai.com/docs/guides/embeddings/what-are-embeddings). This is implemented by several project besides OpenAI itself, including the python-based fastembed-server found in `supplemental/search/fastembed-api`.
 
-The default settings will support a setup where both the fastembed server and Qdrant run on the same system as pleroma.
+The default settings will support a setup where both the fastembed server and Qdrant run on the same system as pleroma. To use it, set the search provider and run the fastembed server, see the README in `supplemental/search/fastembed-api`:
+
+https://qdrant.github.io/fastembed/examples/Supported_Models/
+
+> config :pleroma, Pleroma.Search, module: Pleroma.Search.QdrantSearch
+
+You will also need to create the Qdrant index once by running `mix pleroma.search.indexer create_index`. Running `mix pleroma.search.indexer index` will retroactively index the last 100_000 activities.
+
+### Indexing and model options
+
+To see the available configuration options, check out the QdrantSearch section in `config/config.exs`.
+
+The default indexing option work for the default model (`snowflake-arctic-embed-xs`). To optimize for a low memory footprint, adjust the index configuration as described in the [Qdrant docs](https://qdrant.tech/documentation/guides/optimize/).
+
+Different embedding models will need different vector size settings. You can see a list of the models supported by the fastembed server [here](https://qdrant.github.io/fastembed/examples/Supported_Models), including their vector dimensions. These vector dimensions need to be set in the `qdrant_index_configuration`. 
+
+E.g, If you want to use `sentence-transformers/all-MiniLM-L6-v2` as a model, you will not need to adjust things, because it and `snowflake-arctic-embed-xs` are both 384 dimensional models. If you want to use `snowflake/snowflake-arctic-embed-l`, you will need to adjust the `size` parameter in the `qdrant_index_configuration` to 1024, as it has a dimension of 1024.
+
+When using a different model, you will need do drop the index and recreate it (`mix pleroma.search.indexer drop_index` and `mix pleroma.search.indexer create_index`), as the different embeddings are not compatible with each other.
 
 ## Meilisearch
 

From 6ec306d0684f3c5c05d768a3c431008925f21f15 Mon Sep 17 00:00:00 2001
From: Lain Soykaf <lain@lain.com>
Date: Sun, 19 May 2024 13:24:24 +0400
Subject: [PATCH 061/192] Docs: Add more information about index memory
 consumption.

---
 docs/configuration/search.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/configuration/search.md b/docs/configuration/search.md
index 6598e533f..ed85acd2a 100644
--- a/docs/configuration/search.md
+++ b/docs/configuration/search.md
@@ -26,7 +26,7 @@ You will also need to create the Qdrant index once by running `mix pleroma.searc
 
 To see the available configuration options, check out the QdrantSearch section in `config/config.exs`.
 
-The default indexing option work for the default model (`snowflake-arctic-embed-xs`). To optimize for a low memory footprint, adjust the index configuration as described in the [Qdrant docs](https://qdrant.tech/documentation/guides/optimize/).
+The default indexing option work for the default model (`snowflake-arctic-embed-xs`). To optimize for a low memory footprint, adjust the index configuration as described in the [Qdrant docs](https://qdrant.tech/documentation/guides/optimize/). See also [this blog post](https://qdrant.tech/articles/memory-consumption/) that goes into detail.
 
 Different embedding models will need different vector size settings. You can see a list of the models supported by the fastembed server [here](https://qdrant.github.io/fastembed/examples/Supported_Models), including their vector dimensions. These vector dimensions need to be set in the `qdrant_index_configuration`. 
 

From dbaab6f54e306e5fb930ce1ed0699631c8aeaae1 Mon Sep 17 00:00:00 2001
From: Lain Soykaf <lain@lain.com>
Date: Sun, 19 May 2024 13:38:31 +0400
Subject: [PATCH 062/192] Docs: Mention running the Qdrant server

---
 docs/configuration/search.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/configuration/search.md b/docs/configuration/search.md
index ed85acd2a..d34f84d4f 100644
--- a/docs/configuration/search.md
+++ b/docs/configuration/search.md
@@ -16,10 +16,10 @@ This uses the vector search engine [Qdrant](https://qdrant.tech) to search the p
 
 The default settings will support a setup where both the fastembed server and Qdrant run on the same system as pleroma. To use it, set the search provider and run the fastembed server, see the README in `supplemental/search/fastembed-api`:
 
-https://qdrant.github.io/fastembed/examples/Supported_Models/
-
 > config :pleroma, Pleroma.Search, module: Pleroma.Search.QdrantSearch
 
+Then, start the Qdrant server, see [here](https://qdrant.tech/documentation/quick-start/) for instructions.
+
 You will also need to create the Qdrant index once by running `mix pleroma.search.indexer create_index`. Running `mix pleroma.search.indexer index` will retroactively index the last 100_000 activities.
 
 ### Indexing and model options

From 1b4f1db9b2990f725a06f0dff41980c51853c5e9 Mon Sep 17 00:00:00 2001
From: Lain Soykaf <lain@lain.com>
Date: Sun, 19 May 2024 14:41:05 +0400
Subject: [PATCH 063/192] QdrantSearch: Support pagination.

---
 lib/pleroma/search/qdrant_search.ex | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/lib/pleroma/search/qdrant_search.ex b/lib/pleroma/search/qdrant_search.ex
index 5ae04be78..283c43075 100644
--- a/lib/pleroma/search/qdrant_search.ex
+++ b/lib/pleroma/search/qdrant_search.ex
@@ -54,10 +54,11 @@ defmodule Pleroma.Search.QdrantSearch do
     }
   end
 
-  defp build_search_payload(embedding) do
+  defp build_search_payload(embedding, options) do
     %{
       vector: embedding,
-      limit: 20
+      limit: options[:limit] || 20,
+      offset: options[:offset] || 0
     }
   end
 
@@ -96,12 +97,15 @@ defmodule Pleroma.Search.QdrantSearch do
   end
 
   @impl true
-  def search(_user, query, _options) do
+  def search(_user, query, options) do
     query = "Represent this sentence for searching relevant passages: #{query}"
 
     with {:ok, embedding} <- get_embedding(query),
          {:ok, %{body: %{"result" => result}}} <-
-           QdrantClient.post("/collections/posts/points/search", build_search_payload(embedding)) do
+           QdrantClient.post(
+             "/collections/posts/points/search",
+             build_search_payload(embedding, options)
+           ) do
       ids =
         Enum.map(result, fn %{"id" => id} ->
           Ecto.UUID.dump!(id)

From 226874c9d603be72699d5aa5434616efffe3f239 Mon Sep 17 00:00:00 2001
From: Lain Soykaf <lain@lain.com>
Date: Mon, 20 May 2024 13:12:12 +0400
Subject: [PATCH 064/192] CI: Add new builders for base images

---
 .gitlab-ci.yml                         | 6 +++---
 ci/elixir-1.13/Dockerfile              | 8 ++++++++
 ci/elixir-1.13/build_and_push.sh       | 1 +
 ci/elixir-1.15-otp25/build_and_push.sh | 2 +-
 4 files changed, 13 insertions(+), 4 deletions(-)
 create mode 100644 ci/elixir-1.13/Dockerfile
 create mode 100755 ci/elixir-1.13/build_and_push.sh

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 2e321c978..eba769af8 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,4 +1,4 @@
-image: git.pleroma.social:5050/pleroma/pleroma/ci-base
+image: git.pleroma.social:5050/pleroma/pleroma/ci-base:1.13.4-otp24
 
 variables: &global_variables
   # Only used for the release
@@ -72,7 +72,7 @@ check-changelog:
   tags:
     - amd64
 
-build-1.12.3:
+build-1.13.4:
   extends:
   - .build_changes_policy
   - .using-ci-base
@@ -85,7 +85,7 @@ build-1.15.7-otp-25:
   - .build_changes_policy
   - .using-ci-base
   stage: build
-  image: git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.15
+  image: git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.15-otp25
   allow_failure: true
   script:
   - mix compile --force
diff --git a/ci/elixir-1.13/Dockerfile b/ci/elixir-1.13/Dockerfile
new file mode 100644
index 000000000..b8bceb3d9
--- /dev/null
+++ b/ci/elixir-1.13/Dockerfile
@@ -0,0 +1,8 @@
+FROM elixir:1.13.4-otp-24
+
+# Single RUN statement, otherwise intermediate images are created
+# https://docs.docker.com/develop/develop-images/dockerfile_best-practices/#run
+RUN apt-get update &&\
+	apt-get install -y libmagic-dev cmake libimage-exiftool-perl ffmpeg &&\
+	mix local.hex --force &&\
+	mix local.rebar --force
diff --git a/ci/elixir-1.13/build_and_push.sh b/ci/elixir-1.13/build_and_push.sh
new file mode 100755
index 000000000..53af4245f
--- /dev/null
+++ b/ci/elixir-1.13/build_and_push.sh
@@ -0,0 +1 @@
+docker buildx build --platform linux/amd64,linux/arm64 -t git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.13-otp24 --push .
diff --git a/ci/elixir-1.15-otp25/build_and_push.sh b/ci/elixir-1.15-otp25/build_and_push.sh
index 06fe74f34..a28e0d33c 100755
--- a/ci/elixir-1.15-otp25/build_and_push.sh
+++ b/ci/elixir-1.15-otp25/build_and_push.sh
@@ -1 +1 @@
-docker buildx build --platform linux/amd64 -t git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.15-otp25 --push .
+docker buildx build --platform linux/amd64,linux/arm64 -t git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.15-otp25 --push .

From f8411a351de07f14fdc9c9eca30109feaadf6f93 Mon Sep 17 00:00:00 2001
From: Lain Soykaf <lain@lain.com>
Date: Mon, 20 May 2024 13:30:31 +0400
Subject: [PATCH 065/192] CI: Specify version fully in base image tag

---
 ci/elixir-1.13/build_and_push.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/elixir-1.13/build_and_push.sh b/ci/elixir-1.13/build_and_push.sh
index 53af4245f..d848344a3 100755
--- a/ci/elixir-1.13/build_and_push.sh
+++ b/ci/elixir-1.13/build_and_push.sh
@@ -1 +1 @@
-docker buildx build --platform linux/amd64,linux/arm64 -t git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.13-otp24 --push .
+docker buildx build --platform linux/amd64,linux/arm64 -t git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.13.4-otp24 --push .

From f5c029524752e1820ea29f6557647823ae89ecf1 Mon Sep 17 00:00:00 2001
From: Lain Soykaf <lain@lain.com>
Date: Mon, 20 May 2024 13:32:25 +0400
Subject: [PATCH 066/192] CI: Specify correct image name.

---
 .gitlab-ci.yml                   | 2 +-
 ci/elixir-1.13/build_and_push.sh | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index eba769af8..21d7b2242 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,4 +1,4 @@
-image: git.pleroma.social:5050/pleroma/pleroma/ci-base:1.13.4-otp24
+image: git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.13.4-otp-24
 
 variables: &global_variables
   # Only used for the release
diff --git a/ci/elixir-1.13/build_and_push.sh b/ci/elixir-1.13/build_and_push.sh
index d848344a3..64e1856db 100755
--- a/ci/elixir-1.13/build_and_push.sh
+++ b/ci/elixir-1.13/build_and_push.sh
@@ -1 +1 @@
-docker buildx build --platform linux/amd64,linux/arm64 -t git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.13.4-otp24 --push .
+docker buildx build --platform linux/amd64,linux/arm64 -t git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.13.4-otp-24 --push .

From 36fa0debfe66d3b706eeaa09227edd8b82c70aba Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= <git@mkljczk.pl>
Date: Mon, 20 May 2024 23:25:50 +0200
Subject: [PATCH 067/192] Fix `get_notified_from`
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: marcin mikołajczak <git@mkljczk.pl>
---
 lib/pleroma/notification.ex | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex
index 55c47e966..942aa7198 100644
--- a/lib/pleroma/notification.ex
+++ b/lib/pleroma/notification.ex
@@ -526,7 +526,7 @@ defmodule Pleroma.Notification do
     Enum.filter(potential_receivers, fn u -> u.ap_id in notification_enabled_ap_ids end)
   end
 
-  def get_notified_from_activity(_, _local_only), do: {[], []}
+  def get_notified_from_activity(_, _local_only), do: []
 
   def get_notified_subscribers_from_activity(activity, local_only \\ true)
 
@@ -544,7 +544,7 @@ defmodule Pleroma.Notification do
     Enum.filter(potential_receivers, fn u -> u.ap_id in notification_enabled_ap_ids end)
   end
 
-  def get_notified_subscribers_from_activity(_, _), do: {[], []}
+  def get_notified_subscribers_from_activity(_, _), do: []
 
   # For some activities, only notify the author of the object
   def get_potential_receiver_ap_ids(%{data: %{"type" => type, "object" => object_id}})

From d1b053f3ba4170021c511b0d06a41405d3ab07d3 Mon Sep 17 00:00:00 2001
From: Lain Soykaf <lain@lain.com>
Date: Wed, 22 May 2024 12:57:30 +0400
Subject: [PATCH 068/192] Webfinger: Add test showing wrong webfinger behavior

---
 .../webfinger/graf-imposter-webfinger.json    | 41 +++++++++++++++++++
 test/pleroma/web/web_finger_test.exs          | 15 +++++++
 2 files changed, 56 insertions(+)
 create mode 100644 test/fixtures/webfinger/graf-imposter-webfinger.json

diff --git a/test/fixtures/webfinger/graf-imposter-webfinger.json b/test/fixtures/webfinger/graf-imposter-webfinger.json
new file mode 100644
index 000000000..e7010f606
--- /dev/null
+++ b/test/fixtures/webfinger/graf-imposter-webfinger.json
@@ -0,0 +1,41 @@
+{
+    "subject": "acct:graf@poa.st",
+    "aliases": [
+        "https://fba.ryona.agenc/webfingertest"
+    ],
+    "links": [
+        {
+            "rel": "http://webfinger.net/rel/profile-page",
+            "type": "text/html",
+            "href": "https://fba.ryona.agenc/webfingertest"
+        },
+        {
+            "rel": "self",
+            "type": "application/activity+json",
+            "href": "https://fba.ryona.agenc/webfingertest"
+        },
+        {
+            "rel": "http://ostatus.org/schema/1.0/subscribe",
+            "template": "https://fba.ryona.agenc/contact/follow?url={uri}"
+        },
+        {
+            "rel": "http://schemas.google.com/g/2010#updates-from",
+            "type": "application/atom+xml",
+            "href": ""
+        },
+        {
+            "rel": "salmon",
+            "href": "https://fba.ryona.agenc/salmon/friendica"
+        },
+        {
+            "rel": "http://microformats.org/profile/hcard",
+            "type": "text/html",
+            "href": "https://fba.ryona.agenc/hcard/friendica"
+        },
+        {
+            "rel": "http://joindiaspora.com/seed_location",
+            "type": "text/html",
+            "href": "https://fba.ryona.agenc"
+        }
+    ]
+}
diff --git a/test/pleroma/web/web_finger_test.exs b/test/pleroma/web/web_finger_test.exs
index be5e08776..6530fbc56 100644
--- a/test/pleroma/web/web_finger_test.exs
+++ b/test/pleroma/web/web_finger_test.exs
@@ -204,4 +204,19 @@ defmodule Pleroma.Web.WebFingerTest do
       assert :error = WebFinger.finger("pekorino@pawoo.net")
     end
   end
+
+  test "prevents forgeries" do
+    Tesla.Mock.mock(fn
+      %{url: "https://fba.ryona.agency/.well-known/webfinger?resource=acct:graf@fba.ryona.agency"} ->
+        fake_webfinger =
+          File.read!("test/fixtures/webfinger/graf-imposter-webfinger.json") |> Jason.decode!()
+
+        Tesla.Mock.json(fake_webfinger)
+
+      %{url: "https://fba.ryona.agency/.well-known/host-meta"} ->
+        {:ok, %Tesla.Env{status: 404}}
+    end)
+
+    refute {:ok, _} = WebFinger.finger("graf@fba.ryona.agency")
+  end
 end

From b15f8b06425edbfc3a7cef2a55c609b12ee14377 Mon Sep 17 00:00:00 2001
From: Alex Gleason <alex@alexgleason.me>
Date: Wed, 23 Aug 2023 13:10:19 -0500
Subject: [PATCH 069/192] Prevent webfinger spoofing

---
 lib/pleroma/web/web_finger.ex                 | 16 ++++++++
 .../tesla_mock/gleasonator.com_host_meta      |  4 ++
 test/fixtures/tesla_mock/webfinger_spoof.json | 28 ++++++++++++++
 test/pleroma/web/web_finger_test.exs          | 38 +++++++++++--------
 4 files changed, 71 insertions(+), 15 deletions(-)
 create mode 100644 test/fixtures/tesla_mock/gleasonator.com_host_meta
 create mode 100644 test/fixtures/tesla_mock/webfinger_spoof.json

diff --git a/lib/pleroma/web/web_finger.ex b/lib/pleroma/web/web_finger.ex
index 26fb8af84..a84a4351b 100644
--- a/lib/pleroma/web/web_finger.ex
+++ b/lib/pleroma/web/web_finger.ex
@@ -216,10 +216,26 @@ defmodule Pleroma.Web.WebFinger do
         _ ->
           {:error, {:content_type, nil}}
       end
+      |> case do
+        {:ok, data} -> validate_webfinger(address, data)
+        error -> error
+      end
     else
       error ->
         Logger.debug("Couldn't finger #{account}: #{inspect(error)}")
         error
     end
   end
+
+  defp validate_webfinger(url, %{"subject" => "acct:" <> acct} = data) do
+    with %URI{host: request_host} <- URI.parse(url),
+         [_name, acct_host] <- String.split(acct, "@"),
+         {_, true} <- {:hosts_match, acct_host == request_host} do
+      {:ok, data}
+    else
+      _ -> {:error, {:webfinger_invalid, url, data}}
+    end
+  end
+
+  defp validate_webfinger(url, data), do: {:error, {:webfinger_invalid, url, data}}
 end
diff --git a/test/fixtures/tesla_mock/gleasonator.com_host_meta b/test/fixtures/tesla_mock/gleasonator.com_host_meta
new file mode 100644
index 000000000..c1a432519
--- /dev/null
+++ b/test/fixtures/tesla_mock/gleasonator.com_host_meta
@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<XRD xmlns="http://docs.oasis-open.org/ns/xri/xrd-1.0">
+  <Link rel="lrdd" template="https://gleasonator.com/.well-known/webfinger?resource={uri}" type="application/xrd+xml" />
+</XRD>
\ No newline at end of file
diff --git a/test/fixtures/tesla_mock/webfinger_spoof.json b/test/fixtures/tesla_mock/webfinger_spoof.json
new file mode 100644
index 000000000..7c2a11f69
--- /dev/null
+++ b/test/fixtures/tesla_mock/webfinger_spoof.json
@@ -0,0 +1,28 @@
+{
+  "aliases": [
+    "https://gleasonator.com/users/alex",
+    "https://mitra.social/users/alex"
+  ],
+  "links": [
+    {
+      "href": "https://gleasonator.com/users/alex",
+      "rel": "http://webfinger.net/rel/profile-page",
+      "type": "text/html"
+    },
+    {
+      "href": "https://gleasonator.com/users/alex",
+      "rel": "self",
+      "type": "application/activity+json"
+    },
+    {
+      "href": "https://gleasonator.com/users/alex",
+      "rel": "self",
+      "type": "application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\""
+    },
+    {
+      "rel": "http://ostatus.org/schema/1.0/subscribe",
+      "template": "https://gleasonator.com/ostatus_subscribe?acct={uri}"
+    }
+  ],
+  "subject": "acct:trump@whitehouse.gov"
+}
diff --git a/test/pleroma/web/web_finger_test.exs b/test/pleroma/web/web_finger_test.exs
index 6530fbc56..84a8e19d5 100644
--- a/test/pleroma/web/web_finger_test.exs
+++ b/test/pleroma/web/web_finger_test.exs
@@ -76,15 +76,6 @@ defmodule Pleroma.Web.WebFingerTest do
       {:ok, _data} = WebFinger.finger(user)
     end
 
-    test "returns the ActivityPub actor URI and subscribe address for an ActivityPub user with the ld+json mimetype" do
-      user = "kaniini@gerzilla.de"
-
-      {:ok, data} = WebFinger.finger(user)
-
-      assert data["ap_id"] == "https://gerzilla.de/channel/kaniini"
-      assert data["subscribe_address"] == "https://gerzilla.de/follow?f=&url={uri}"
-    end
-
     test "it work for AP-only user" do
       user = "kpherox@mstdn.jp"
 
@@ -99,12 +90,6 @@ defmodule Pleroma.Web.WebFingerTest do
       assert data["subscribe_address"] == "https://mstdn.jp/authorize_interaction?acct={uri}"
     end
 
-    test "it works for friendica" do
-      user = "lain@squeet.me"
-
-      {:ok, _data} = WebFinger.finger(user)
-    end
-
     test "it gets the xrd endpoint" do
       {:ok, template} = WebFinger.find_lrdd_template("social.heldscal.la")
 
@@ -203,6 +188,29 @@ defmodule Pleroma.Web.WebFingerTest do
 
       assert :error = WebFinger.finger("pekorino@pawoo.net")
     end
+
+    test "prevents spoofing" do
+      Tesla.Mock.mock(fn
+        %{
+          url: "https://gleasonator.com/.well-known/webfinger?resource=acct:alex@gleasonator.com"
+        } ->
+          {:ok,
+           %Tesla.Env{
+             status: 200,
+             body: File.read!("test/fixtures/tesla_mock/webfinger_spoof.json"),
+             headers: [{"content-type", "application/jrd+json"}]
+           }}
+
+        %{url: "https://gleasonator.com/.well-known/host-meta"} ->
+          {:ok,
+           %Tesla.Env{
+             status: 200,
+             body: File.read!("test/fixtures/tesla_mock/gleasonator.com_host_meta")
+           }}
+      end)
+
+      {:error, _data} = WebFinger.finger("alex@gleasonator.com")
+    end
   end
 
   test "prevents forgeries" do

From 206ea92837f8016d66a2b87f7f7338d814735a92 Mon Sep 17 00:00:00 2001
From: Lain Soykaf <lain@lain.com>
Date: Wed, 22 May 2024 12:59:10 +0400
Subject: [PATCH 070/192] Webfinger: Fix test

---
 test/pleroma/web/web_finger_test.exs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/test/pleroma/web/web_finger_test.exs b/test/pleroma/web/web_finger_test.exs
index 84a8e19d5..8a550a6ba 100644
--- a/test/pleroma/web/web_finger_test.exs
+++ b/test/pleroma/web/web_finger_test.exs
@@ -213,6 +213,7 @@ defmodule Pleroma.Web.WebFingerTest do
     end
   end
 
+  @tag capture_log: true
   test "prevents forgeries" do
     Tesla.Mock.mock(fn
       %{url: "https://fba.ryona.agency/.well-known/webfinger?resource=acct:graf@fba.ryona.agency"} ->
@@ -225,6 +226,6 @@ defmodule Pleroma.Web.WebFingerTest do
         {:ok, %Tesla.Env{status: 404}}
     end)
 
-    refute {:ok, _} = WebFinger.finger("graf@fba.ryona.agency")
+    assert {:error, _} = WebFinger.finger("graf@fba.ryona.agency")
   end
 end

From 4491e8c9a3e2cdeb1b8e9cb98015dc1d0435c65c Mon Sep 17 00:00:00 2001
From: Lain Soykaf <lain@lain.com>
Date: Wed, 22 May 2024 13:01:23 +0400
Subject: [PATCH 071/192] Add changelog

---
 changelog.d/fix-webfinger-spoofing.fix | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/fix-webfinger-spoofing.fix

diff --git a/changelog.d/fix-webfinger-spoofing.fix b/changelog.d/fix-webfinger-spoofing.fix
new file mode 100644
index 000000000..7b3c9490a
--- /dev/null
+++ b/changelog.d/fix-webfinger-spoofing.fix
@@ -0,0 +1 @@
+Fix webfinger spoofing.

From 91c93ce3cd62a916c7d367979473f94e36cf1873 Mon Sep 17 00:00:00 2001
From: Lain Soykaf <lain@lain.com>
Date: Wed, 22 May 2024 13:14:59 +0400
Subject: [PATCH 072/192] Changelog: Adjust changelog type

---
 ...fix-webfinger-spoofing.fix => fix-webfinger-spoofing.security} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename changelog.d/{fix-webfinger-spoofing.fix => fix-webfinger-spoofing.security} (100%)

diff --git a/changelog.d/fix-webfinger-spoofing.fix b/changelog.d/fix-webfinger-spoofing.security
similarity index 100%
rename from changelog.d/fix-webfinger-spoofing.fix
rename to changelog.d/fix-webfinger-spoofing.security

From 84bb854056e406d5235dd442c28127891a8a8a86 Mon Sep 17 00:00:00 2001
From: Lain Soykaf <lain@lain.com>
Date: Wed, 22 May 2024 15:12:29 +0400
Subject: [PATCH 073/192] Webfinger: Allow managing account for subdomain

---
 lib/pleroma/web/web_finger.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pleroma/web/web_finger.ex b/lib/pleroma/web/web_finger.ex
index a84a4351b..e149d9247 100644
--- a/lib/pleroma/web/web_finger.ex
+++ b/lib/pleroma/web/web_finger.ex
@@ -230,7 +230,7 @@ defmodule Pleroma.Web.WebFinger do
   defp validate_webfinger(url, %{"subject" => "acct:" <> acct} = data) do
     with %URI{host: request_host} <- URI.parse(url),
          [_name, acct_host] <- String.split(acct, "@"),
-         {_, true} <- {:hosts_match, acct_host == request_host} do
+         {_, true} <- {:hosts_match_or_subdomain, String.ends_with?(request_host, acct_host)} do
       {:ok, data}
     else
       _ -> {:error, {:webfinger_invalid, url, data}}

From 29b968ce2006de47d8f1dbc161756e35ba5944a1 Mon Sep 17 00:00:00 2001
From: Lain Soykaf <lain@lain.com>
Date: Wed, 22 May 2024 12:57:30 +0400
Subject: [PATCH 074/192] Webfinger: Add test showing wrong webfinger behavior

---
 .../webfinger/graf-imposter-webfinger.json    | 41 +++++++++++++++++++
 test/pleroma/web/web_finger_test.exs          | 15 +++++++
 2 files changed, 56 insertions(+)
 create mode 100644 test/fixtures/webfinger/graf-imposter-webfinger.json

diff --git a/test/fixtures/webfinger/graf-imposter-webfinger.json b/test/fixtures/webfinger/graf-imposter-webfinger.json
new file mode 100644
index 000000000..e7010f606
--- /dev/null
+++ b/test/fixtures/webfinger/graf-imposter-webfinger.json
@@ -0,0 +1,41 @@
+{
+    "subject": "acct:graf@poa.st",
+    "aliases": [
+        "https://fba.ryona.agenc/webfingertest"
+    ],
+    "links": [
+        {
+            "rel": "http://webfinger.net/rel/profile-page",
+            "type": "text/html",
+            "href": "https://fba.ryona.agenc/webfingertest"
+        },
+        {
+            "rel": "self",
+            "type": "application/activity+json",
+            "href": "https://fba.ryona.agenc/webfingertest"
+        },
+        {
+            "rel": "http://ostatus.org/schema/1.0/subscribe",
+            "template": "https://fba.ryona.agenc/contact/follow?url={uri}"
+        },
+        {
+            "rel": "http://schemas.google.com/g/2010#updates-from",
+            "type": "application/atom+xml",
+            "href": ""
+        },
+        {
+            "rel": "salmon",
+            "href": "https://fba.ryona.agenc/salmon/friendica"
+        },
+        {
+            "rel": "http://microformats.org/profile/hcard",
+            "type": "text/html",
+            "href": "https://fba.ryona.agenc/hcard/friendica"
+        },
+        {
+            "rel": "http://joindiaspora.com/seed_location",
+            "type": "text/html",
+            "href": "https://fba.ryona.agenc"
+        }
+    ]
+}
diff --git a/test/pleroma/web/web_finger_test.exs b/test/pleroma/web/web_finger_test.exs
index be5e08776..6530fbc56 100644
--- a/test/pleroma/web/web_finger_test.exs
+++ b/test/pleroma/web/web_finger_test.exs
@@ -204,4 +204,19 @@ defmodule Pleroma.Web.WebFingerTest do
       assert :error = WebFinger.finger("pekorino@pawoo.net")
     end
   end
+
+  test "prevents forgeries" do
+    Tesla.Mock.mock(fn
+      %{url: "https://fba.ryona.agency/.well-known/webfinger?resource=acct:graf@fba.ryona.agency"} ->
+        fake_webfinger =
+          File.read!("test/fixtures/webfinger/graf-imposter-webfinger.json") |> Jason.decode!()
+
+        Tesla.Mock.json(fake_webfinger)
+
+      %{url: "https://fba.ryona.agency/.well-known/host-meta"} ->
+        {:ok, %Tesla.Env{status: 404}}
+    end)
+
+    refute {:ok, _} = WebFinger.finger("graf@fba.ryona.agency")
+  end
 end

From 364f6e1620876dcfc1d228e2db17190d74b6f0ce Mon Sep 17 00:00:00 2001
From: Alex Gleason <alex@alexgleason.me>
Date: Wed, 23 Aug 2023 13:10:19 -0500
Subject: [PATCH 075/192] Prevent webfinger spoofing

---
 lib/pleroma/web/web_finger.ex                 | 16 ++++++++
 .../tesla_mock/gleasonator.com_host_meta      |  4 ++
 test/fixtures/tesla_mock/webfinger_spoof.json | 28 ++++++++++++++
 test/pleroma/web/web_finger_test.exs          | 38 +++++++++++--------
 4 files changed, 71 insertions(+), 15 deletions(-)
 create mode 100644 test/fixtures/tesla_mock/gleasonator.com_host_meta
 create mode 100644 test/fixtures/tesla_mock/webfinger_spoof.json

diff --git a/lib/pleroma/web/web_finger.ex b/lib/pleroma/web/web_finger.ex
index f95dc2458..0d6a686c3 100644
--- a/lib/pleroma/web/web_finger.ex
+++ b/lib/pleroma/web/web_finger.ex
@@ -216,10 +216,26 @@ defmodule Pleroma.Web.WebFinger do
         _ ->
           {:error, {:content_type, nil}}
       end
+      |> case do
+        {:ok, data} -> validate_webfinger(address, data)
+        error -> error
+      end
     else
       error ->
         Logger.debug("Couldn't finger #{account}: #{inspect(error)}")
         error
     end
   end
+
+  defp validate_webfinger(url, %{"subject" => "acct:" <> acct} = data) do
+    with %URI{host: request_host} <- URI.parse(url),
+         [_name, acct_host] <- String.split(acct, "@"),
+         {_, true} <- {:hosts_match, acct_host == request_host} do
+      {:ok, data}
+    else
+      _ -> {:error, {:webfinger_invalid, url, data}}
+    end
+  end
+
+  defp validate_webfinger(url, data), do: {:error, {:webfinger_invalid, url, data}}
 end
diff --git a/test/fixtures/tesla_mock/gleasonator.com_host_meta b/test/fixtures/tesla_mock/gleasonator.com_host_meta
new file mode 100644
index 000000000..c1a432519
--- /dev/null
+++ b/test/fixtures/tesla_mock/gleasonator.com_host_meta
@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<XRD xmlns="http://docs.oasis-open.org/ns/xri/xrd-1.0">
+  <Link rel="lrdd" template="https://gleasonator.com/.well-known/webfinger?resource={uri}" type="application/xrd+xml" />
+</XRD>
\ No newline at end of file
diff --git a/test/fixtures/tesla_mock/webfinger_spoof.json b/test/fixtures/tesla_mock/webfinger_spoof.json
new file mode 100644
index 000000000..7c2a11f69
--- /dev/null
+++ b/test/fixtures/tesla_mock/webfinger_spoof.json
@@ -0,0 +1,28 @@
+{
+  "aliases": [
+    "https://gleasonator.com/users/alex",
+    "https://mitra.social/users/alex"
+  ],
+  "links": [
+    {
+      "href": "https://gleasonator.com/users/alex",
+      "rel": "http://webfinger.net/rel/profile-page",
+      "type": "text/html"
+    },
+    {
+      "href": "https://gleasonator.com/users/alex",
+      "rel": "self",
+      "type": "application/activity+json"
+    },
+    {
+      "href": "https://gleasonator.com/users/alex",
+      "rel": "self",
+      "type": "application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\""
+    },
+    {
+      "rel": "http://ostatus.org/schema/1.0/subscribe",
+      "template": "https://gleasonator.com/ostatus_subscribe?acct={uri}"
+    }
+  ],
+  "subject": "acct:trump@whitehouse.gov"
+}
diff --git a/test/pleroma/web/web_finger_test.exs b/test/pleroma/web/web_finger_test.exs
index 6530fbc56..84a8e19d5 100644
--- a/test/pleroma/web/web_finger_test.exs
+++ b/test/pleroma/web/web_finger_test.exs
@@ -76,15 +76,6 @@ defmodule Pleroma.Web.WebFingerTest do
       {:ok, _data} = WebFinger.finger(user)
     end
 
-    test "returns the ActivityPub actor URI and subscribe address for an ActivityPub user with the ld+json mimetype" do
-      user = "kaniini@gerzilla.de"
-
-      {:ok, data} = WebFinger.finger(user)
-
-      assert data["ap_id"] == "https://gerzilla.de/channel/kaniini"
-      assert data["subscribe_address"] == "https://gerzilla.de/follow?f=&url={uri}"
-    end
-
     test "it work for AP-only user" do
       user = "kpherox@mstdn.jp"
 
@@ -99,12 +90,6 @@ defmodule Pleroma.Web.WebFingerTest do
       assert data["subscribe_address"] == "https://mstdn.jp/authorize_interaction?acct={uri}"
     end
 
-    test "it works for friendica" do
-      user = "lain@squeet.me"
-
-      {:ok, _data} = WebFinger.finger(user)
-    end
-
     test "it gets the xrd endpoint" do
       {:ok, template} = WebFinger.find_lrdd_template("social.heldscal.la")
 
@@ -203,6 +188,29 @@ defmodule Pleroma.Web.WebFingerTest do
 
       assert :error = WebFinger.finger("pekorino@pawoo.net")
     end
+
+    test "prevents spoofing" do
+      Tesla.Mock.mock(fn
+        %{
+          url: "https://gleasonator.com/.well-known/webfinger?resource=acct:alex@gleasonator.com"
+        } ->
+          {:ok,
+           %Tesla.Env{
+             status: 200,
+             body: File.read!("test/fixtures/tesla_mock/webfinger_spoof.json"),
+             headers: [{"content-type", "application/jrd+json"}]
+           }}
+
+        %{url: "https://gleasonator.com/.well-known/host-meta"} ->
+          {:ok,
+           %Tesla.Env{
+             status: 200,
+             body: File.read!("test/fixtures/tesla_mock/gleasonator.com_host_meta")
+           }}
+      end)
+
+      {:error, _data} = WebFinger.finger("alex@gleasonator.com")
+    end
   end
 
   test "prevents forgeries" do

From eafcb7b4ec368038aafa440ea32abe417a805f41 Mon Sep 17 00:00:00 2001
From: Lain Soykaf <lain@lain.com>
Date: Wed, 22 May 2024 12:59:10 +0400
Subject: [PATCH 076/192] Webfinger: Fix test

---
 test/pleroma/web/web_finger_test.exs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/test/pleroma/web/web_finger_test.exs b/test/pleroma/web/web_finger_test.exs
index 84a8e19d5..8a550a6ba 100644
--- a/test/pleroma/web/web_finger_test.exs
+++ b/test/pleroma/web/web_finger_test.exs
@@ -213,6 +213,7 @@ defmodule Pleroma.Web.WebFingerTest do
     end
   end
 
+  @tag capture_log: true
   test "prevents forgeries" do
     Tesla.Mock.mock(fn
       %{url: "https://fba.ryona.agency/.well-known/webfinger?resource=acct:graf@fba.ryona.agency"} ->
@@ -225,6 +226,6 @@ defmodule Pleroma.Web.WebFingerTest do
         {:ok, %Tesla.Env{status: 404}}
     end)
 
-    refute {:ok, _} = WebFinger.finger("graf@fba.ryona.agency")
+    assert {:error, _} = WebFinger.finger("graf@fba.ryona.agency")
   end
 end

From 275fdb26c1472d3109721590080dea863c769794 Mon Sep 17 00:00:00 2001
From: Lain Soykaf <lain@lain.com>
Date: Wed, 22 May 2024 13:01:23 +0400
Subject: [PATCH 077/192] Add changelog

---
 changelog.d/fix-webfinger-spoofing.fix | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/fix-webfinger-spoofing.fix

diff --git a/changelog.d/fix-webfinger-spoofing.fix b/changelog.d/fix-webfinger-spoofing.fix
new file mode 100644
index 000000000..7b3c9490a
--- /dev/null
+++ b/changelog.d/fix-webfinger-spoofing.fix
@@ -0,0 +1 @@
+Fix webfinger spoofing.

From 2212287b0047d356592da82b02170b25fa1a4011 Mon Sep 17 00:00:00 2001
From: Lain Soykaf <lain@lain.com>
Date: Wed, 22 May 2024 13:14:59 +0400
Subject: [PATCH 078/192] Changelog: Adjust changelog type

---
 ...fix-webfinger-spoofing.fix => fix-webfinger-spoofing.security} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename changelog.d/{fix-webfinger-spoofing.fix => fix-webfinger-spoofing.security} (100%)

diff --git a/changelog.d/fix-webfinger-spoofing.fix b/changelog.d/fix-webfinger-spoofing.security
similarity index 100%
rename from changelog.d/fix-webfinger-spoofing.fix
rename to changelog.d/fix-webfinger-spoofing.security

From 20fa400082df4c504768190f1ecbd407c9a6376f Mon Sep 17 00:00:00 2001
From: Lain Soykaf <lain@lain.com>
Date: Wed, 22 May 2024 15:12:29 +0400
Subject: [PATCH 079/192] Webfinger: Allow managing account for subdomain

---
 lib/pleroma/web/web_finger.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pleroma/web/web_finger.ex b/lib/pleroma/web/web_finger.ex
index 0d6a686c3..668d7d576 100644
--- a/lib/pleroma/web/web_finger.ex
+++ b/lib/pleroma/web/web_finger.ex
@@ -230,7 +230,7 @@ defmodule Pleroma.Web.WebFinger do
   defp validate_webfinger(url, %{"subject" => "acct:" <> acct} = data) do
     with %URI{host: request_host} <- URI.parse(url),
          [_name, acct_host] <- String.split(acct, "@"),
-         {_, true} <- {:hosts_match, acct_host == request_host} do
+         {_, true} <- {:hosts_match_or_subdomain, String.ends_with?(request_host, acct_host)} do
       {:ok, data}
     else
       _ -> {:error, {:webfinger_invalid, url, data}}

From 239c9c3f1ce60a95b389c2f4ee1e717f4907c381 Mon Sep 17 00:00:00 2001
From: Lain Soykaf <lain@lain.com>
Date: Wed, 22 May 2024 17:40:20 +0400
Subject: [PATCH 080/192] Mix: Update version

---
 mix.exs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mix.exs b/mix.exs
index c95c2a82f..d0ee061c8 100644
--- a/mix.exs
+++ b/mix.exs
@@ -4,7 +4,7 @@ defmodule Pleroma.Mixfile do
   def project do
     [
       app: :pleroma,
-      version: version("2.6.2"),
+      version: version("2.6.3"),
       elixir: "~> 1.11",
       elixirc_paths: elixirc_paths(Mix.env()),
       compilers: [:phoenix] ++ Mix.compilers(),

From 7b4e6d4c16a246ef4ae958a1536b00320441b63e Mon Sep 17 00:00:00 2001
From: Lain Soykaf <lain@lain.com>
Date: Wed, 22 May 2024 17:44:10 +0400
Subject: [PATCH 081/192] Collect changelog

---
 CHANGELOG.md                                | 5 +++++
 changelog.d/fix-webfinger-spoofing.security | 1 -
 2 files changed, 5 insertions(+), 1 deletion(-)
 delete mode 100644 changelog.d/fix-webfinger-spoofing.security

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 92e5e6134..75d2aa415 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
+## 2.6.3
+
+### Security
+- Fix webfinger spoofing.
+
 ## 2.6.2
 
 ### Security
diff --git a/changelog.d/fix-webfinger-spoofing.security b/changelog.d/fix-webfinger-spoofing.security
deleted file mode 100644
index 7b3c9490a..000000000
--- a/changelog.d/fix-webfinger-spoofing.security
+++ /dev/null
@@ -1 +0,0 @@
-Fix webfinger spoofing.

From 1f2f7e044d1be1e56789ce01ce4e54dd86a74f36 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= <git@mkljczk.pl>
Date: Wed, 22 May 2024 15:52:10 +0200
Subject: [PATCH 082/192] Revert "Webfinger: Allow managing account for
 subdomain"

This reverts commit 84bb854056e406d5235dd442c28127891a8a8a86.
---
 lib/pleroma/web/web_finger.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pleroma/web/web_finger.ex b/lib/pleroma/web/web_finger.ex
index e149d9247..a84a4351b 100644
--- a/lib/pleroma/web/web_finger.ex
+++ b/lib/pleroma/web/web_finger.ex
@@ -230,7 +230,7 @@ defmodule Pleroma.Web.WebFinger do
   defp validate_webfinger(url, %{"subject" => "acct:" <> acct} = data) do
     with %URI{host: request_host} <- URI.parse(url),
          [_name, acct_host] <- String.split(acct, "@"),
-         {_, true} <- {:hosts_match_or_subdomain, String.ends_with?(request_host, acct_host)} do
+         {_, true} <- {:hosts_match, acct_host == request_host} do
       {:ok, data}
     else
       _ -> {:error, {:webfinger_invalid, url, data}}

From d0b18e338bfed05c6b2c4a8f5c63d865d9eb669c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= <git@mkljczk.pl>
Date: Thu, 24 Aug 2023 00:37:39 +0200
Subject: [PATCH 083/192] Fix validate_webfinger when running a different
 domain for Webfinger
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: marcin mikołajczak <git@mkljczk.pl>
---
 lib/pleroma/application.ex    |  3 ++-
 lib/pleroma/web/web_finger.ex | 30 ++++++++++++++++++++++--------
 test/pleroma/user_test.exs    |  4 ++--
 3 files changed, 26 insertions(+), 11 deletions(-)

diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex
index 75154f94c..649bb11c8 100644
--- a/lib/pleroma/application.ex
+++ b/lib/pleroma/application.ex
@@ -162,7 +162,8 @@ defmodule Pleroma.Application do
         expiration: chat_message_id_idempotency_key_expiration(),
         limit: 500_000
       ),
-      build_cachex("rel_me", limit: 2500)
+      build_cachex("rel_me", limit: 2500),
+      build_cachex("host_meta", default_ttl: :timer.minutes(120), limit: 5000)
     ]
   end
 
diff --git a/lib/pleroma/web/web_finger.ex b/lib/pleroma/web/web_finger.ex
index a84a4351b..e653b3338 100644
--- a/lib/pleroma/web/web_finger.ex
+++ b/lib/pleroma/web/web_finger.ex
@@ -155,7 +155,16 @@ defmodule Pleroma.Web.WebFinger do
     end
   end
 
+  @cachex Pleroma.Config.get([:cachex, :provider], Cachex)
   def find_lrdd_template(domain) do
+    @cachex.fetch!(:host_meta_cache, domain, fn _ ->
+      {:commit, fetch_lrdd_template(domain)}
+    end)
+  rescue
+    e -> {:error, "Cachex error: #{inspect(e)}"}
+  end
+
+  defp fetch_lrdd_template(domain) do
     # WebFinger is restricted to HTTPS - https://tools.ietf.org/html/rfc7033#section-9.1
     meta_url = "https://#{domain}/.well-known/host-meta"
 
@@ -168,7 +177,7 @@ defmodule Pleroma.Web.WebFinger do
     end
   end
 
-  defp get_address_from_domain(domain, encoded_account) when is_binary(domain) do
+  defp get_address_from_domain(domain, "acct:" <> _ = encoded_account) when is_binary(domain) do
     case find_lrdd_template(domain) do
       {:ok, template} ->
         String.replace(template, "{uri}", encoded_account)
@@ -178,6 +187,11 @@ defmodule Pleroma.Web.WebFinger do
     end
   end
 
+  defp get_address_from_domain(domain, account) when is_binary(domain) do
+    encoded_account = URI.encode("acct:#{account}")
+    get_address_from_domain(domain, encoded_account)
+  end
+
   defp get_address_from_domain(_, _), do: {:error, :webfinger_no_domain}
 
   @spec finger(String.t()) :: {:ok, map()} | {:error, any()}
@@ -192,9 +206,7 @@ defmodule Pleroma.Web.WebFinger do
           URI.parse(account).host
       end
 
-    encoded_account = URI.encode("acct:#{account}")
-
-    with address when is_binary(address) <- get_address_from_domain(domain, encoded_account),
+    with address when is_binary(address) <- get_address_from_domain(domain, account),
          {:ok, %{status: status, body: body, headers: headers}} when status in 200..299 <-
            HTTP.get(
              address,
@@ -227,13 +239,15 @@ defmodule Pleroma.Web.WebFinger do
     end
   end
 
-  defp validate_webfinger(url, %{"subject" => "acct:" <> acct} = data) do
-    with %URI{host: request_host} <- URI.parse(url),
-         [_name, acct_host] <- String.split(acct, "@"),
+  defp validate_webfinger(request_url, %{"subject" => "acct:" <> acct = subject} = data) do
+    with [_name, acct_host] <- String.split(acct, "@"),
+         {_, url} <- {:address, get_address_from_domain(acct_host, subject)},
+         %URI{host: request_host} <- URI.parse(request_url),
+         %URI{host: acct_host} <- URI.parse(url),
          {_, true} <- {:hosts_match, acct_host == request_host} do
       {:ok, data}
     else
-      _ -> {:error, {:webfinger_invalid, url, data}}
+      _ -> {:error, {:webfinger_invalid, request_url, data}}
     end
   end
 
diff --git a/test/pleroma/user_test.exs b/test/pleroma/user_test.exs
index 48391d871..7f1a8d893 100644
--- a/test/pleroma/user_test.exs
+++ b/test/pleroma/user_test.exs
@@ -877,7 +877,7 @@ defmodule Pleroma.UserTest do
     setup do: clear_config([Pleroma.Web.WebFinger, :update_nickname_on_user_fetch], true)
 
     test "for mastodon" do
-      Tesla.Mock.mock(fn
+      Tesla.Mock.mock_global(fn
         %{url: "https://example.com/.well-known/host-meta"} ->
           %Tesla.Env{
             status: 302,
@@ -935,7 +935,7 @@ defmodule Pleroma.UserTest do
     end
 
     test "for pleroma" do
-      Tesla.Mock.mock(fn
+      Tesla.Mock.mock_global(fn
         %{url: "https://example.com/.well-known/host-meta"} ->
           %Tesla.Env{
             status: 302,

From 70cabbf6dc2f8440484f1e56d3aa2d27f65ee88f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= <git@mkljczk.pl>
Date: Thu, 24 Aug 2023 01:09:00 +0200
Subject: [PATCH 084/192] Fix tests
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: marcin mikołajczak <git@mkljczk.pl>
---
 test/pleroma/user_test.exs                    | 102 +---------------
 .../web_finger/web_finger_controller_test.exs |   5 +
 test/support/http_request_mock.ex             | 114 ++++++++++++++++++
 3 files changed, 125 insertions(+), 96 deletions(-)

diff --git a/test/pleroma/user_test.exs b/test/pleroma/user_test.exs
index 7f1a8d893..5b7a65658 100644
--- a/test/pleroma/user_test.exs
+++ b/test/pleroma/user_test.exs
@@ -877,109 +877,19 @@ defmodule Pleroma.UserTest do
     setup do: clear_config([Pleroma.Web.WebFinger, :update_nickname_on_user_fetch], true)
 
     test "for mastodon" do
-      Tesla.Mock.mock_global(fn
-        %{url: "https://example.com/.well-known/host-meta"} ->
-          %Tesla.Env{
-            status: 302,
-            headers: [{"location", "https://sub.example.com/.well-known/host-meta"}]
-          }
-
-        %{url: "https://sub.example.com/.well-known/host-meta"} ->
-          %Tesla.Env{
-            status: 200,
-            body:
-              "test/fixtures/webfinger/masto-host-meta.xml"
-              |> File.read!()
-              |> String.replace("{{domain}}", "sub.example.com")
-          }
-
-        %{url: "https://sub.example.com/.well-known/webfinger?resource=acct:a@example.com"} ->
-          %Tesla.Env{
-            status: 200,
-            body:
-              "test/fixtures/webfinger/masto-webfinger.json"
-              |> File.read!()
-              |> String.replace("{{nickname}}", "a")
-              |> String.replace("{{domain}}", "example.com")
-              |> String.replace("{{subdomain}}", "sub.example.com"),
-            headers: [{"content-type", "application/jrd+json"}]
-          }
-
-        %{url: "https://sub.example.com/users/a"} ->
-          %Tesla.Env{
-            status: 200,
-            body:
-              "test/fixtures/webfinger/masto-user.json"
-              |> File.read!()
-              |> String.replace("{{nickname}}", "a")
-              |> String.replace("{{domain}}", "sub.example.com"),
-            headers: [{"content-type", "application/activity+json"}]
-          }
-
-        %{url: "https://sub.example.com/users/a/collections/featured"} ->
-          %Tesla.Env{
-            status: 200,
-            body:
-              File.read!("test/fixtures/users_mock/masto_featured.json")
-              |> String.replace("{{domain}}", "sub.example.com")
-              |> String.replace("{{nickname}}", "a"),
-            headers: [{"content-type", "application/activity+json"}]
-          }
-      end)
-
-      ap_id = "a@example.com"
+      ap_id = "a@mastodon.example"
       {:ok, fetched_user} = User.get_or_fetch(ap_id)
 
-      assert fetched_user.ap_id == "https://sub.example.com/users/a"
-      assert fetched_user.nickname == "a@example.com"
+      assert fetched_user.ap_id == "https://sub.mastodon.example/users/a"
+      assert fetched_user.nickname == "a@mastodon.example"
     end
 
     test "for pleroma" do
-      Tesla.Mock.mock_global(fn
-        %{url: "https://example.com/.well-known/host-meta"} ->
-          %Tesla.Env{
-            status: 302,
-            headers: [{"location", "https://sub.example.com/.well-known/host-meta"}]
-          }
-
-        %{url: "https://sub.example.com/.well-known/host-meta"} ->
-          %Tesla.Env{
-            status: 200,
-            body:
-              "test/fixtures/webfinger/pleroma-host-meta.xml"
-              |> File.read!()
-              |> String.replace("{{domain}}", "sub.example.com")
-          }
-
-        %{url: "https://sub.example.com/.well-known/webfinger?resource=acct:a@example.com"} ->
-          %Tesla.Env{
-            status: 200,
-            body:
-              "test/fixtures/webfinger/pleroma-webfinger.json"
-              |> File.read!()
-              |> String.replace("{{nickname}}", "a")
-              |> String.replace("{{domain}}", "example.com")
-              |> String.replace("{{subdomain}}", "sub.example.com"),
-            headers: [{"content-type", "application/jrd+json"}]
-          }
-
-        %{url: "https://sub.example.com/users/a"} ->
-          %Tesla.Env{
-            status: 200,
-            body:
-              "test/fixtures/webfinger/pleroma-user.json"
-              |> File.read!()
-              |> String.replace("{{nickname}}", "a")
-              |> String.replace("{{domain}}", "sub.example.com"),
-            headers: [{"content-type", "application/activity+json"}]
-          }
-      end)
-
-      ap_id = "a@example.com"
+      ap_id = "a@pleroma.example"
       {:ok, fetched_user} = User.get_or_fetch(ap_id)
 
-      assert fetched_user.ap_id == "https://sub.example.com/users/a"
-      assert fetched_user.nickname == "a@example.com"
+      assert fetched_user.ap_id == "https://sub.pleroma.example/users/a"
+      assert fetched_user.nickname == "a@pleroma.example"
     end
   end
 
diff --git a/test/pleroma/web/web_finger/web_finger_controller_test.exs b/test/pleroma/web/web_finger/web_finger_controller_test.exs
index 80e072163..f501c6e44 100644
--- a/test/pleroma/web/web_finger/web_finger_controller_test.exs
+++ b/test/pleroma/web/web_finger/web_finger_controller_test.exs
@@ -56,6 +56,11 @@ defmodule Pleroma.Web.WebFinger.WebFingerControllerTest do
   end
 
   test "reach user on tld, while pleroma is running on subdomain" do
+    Pleroma.Web.Endpoint.config_change(
+      [{Pleroma.Web.Endpoint, url: [host: "sub.example.com"]}],
+      []
+    )
+
     clear_config([Pleroma.Web.Endpoint, :url, :host], "sub.example.com")
 
     clear_config([Pleroma.Web.WebFinger, :domain], "example.com")
diff --git a/test/support/http_request_mock.ex b/test/support/http_request_mock.ex
index f656c9412..20e410424 100644
--- a/test/support/http_request_mock.ex
+++ b/test/support/http_request_mock.ex
@@ -1521,6 +1521,120 @@ defmodule HttpRequestMock do
      }}
   end
 
+  def get("https://mastodon.example/.well-known/host-meta", _, _, _) do
+    {:ok,
+     %Tesla.Env{
+       status: 302,
+       headers: [{"location", "https://sub.mastodon.example/.well-known/host-meta"}]
+     }}
+  end
+
+  def get("https://sub.mastodon.example/.well-known/host-meta", _, _, _) do
+    {:ok,
+     %Tesla.Env{
+       status: 200,
+       body:
+         "test/fixtures/webfinger/masto-host-meta.xml"
+         |> File.read!()
+         |> String.replace("{{domain}}", "sub.mastodon.example")
+     }}
+  end
+
+  def get(
+        "https://sub.mastodon.example/.well-known/webfinger?resource=acct:a@mastodon.example",
+        _,
+        _,
+        _
+      ) do
+    {:ok,
+     %Tesla.Env{
+       status: 200,
+       body:
+         "test/fixtures/webfinger/masto-webfinger.json"
+         |> File.read!()
+         |> String.replace("{{nickname}}", "a")
+         |> String.replace("{{domain}}", "mastodon.example")
+         |> String.replace("{{subdomain}}", "sub.mastodon.example"),
+       headers: [{"content-type", "application/jrd+json"}]
+     }}
+  end
+
+  def get("https://sub.mastodon.example/users/a", _, _, _) do
+    {:ok,
+     %Tesla.Env{
+       status: 200,
+       body:
+         "test/fixtures/webfinger/masto-user.json"
+         |> File.read!()
+         |> String.replace("{{nickname}}", "a")
+         |> String.replace("{{domain}}", "sub.mastodon.example"),
+       headers: [{"content-type", "application/activity+json"}]
+     }}
+  end
+
+  def get("https://sub.mastodon.example/users/a/collections/featured", _, _, _) do
+    {:ok,
+     %Tesla.Env{
+       status: 200,
+       body:
+         File.read!("test/fixtures/users_mock/masto_featured.json")
+         |> String.replace("{{domain}}", "sub.mastodon.example")
+         |> String.replace("{{nickname}}", "a"),
+       headers: [{"content-type", "application/activity+json"}]
+     }}
+  end
+
+  def get("https://pleroma.example/.well-known/host-meta", _, _, _) do
+    {:ok,
+     %Tesla.Env{
+       status: 302,
+       headers: [{"location", "https://sub.pleroma.example/.well-known/host-meta"}]
+     }}
+  end
+
+  def get("https://sub.pleroma.example/.well-known/host-meta", _, _, _) do
+    {:ok,
+     %Tesla.Env{
+       status: 200,
+       body:
+         "test/fixtures/webfinger/pleroma-host-meta.xml"
+         |> File.read!()
+         |> String.replace("{{domain}}", "sub.pleroma.example")
+     }}
+  end
+
+  def get(
+        "https://sub.pleroma.example/.well-known/webfinger?resource=acct:a@pleroma.example",
+        _,
+        _,
+        _
+      ) do
+    {:ok,
+     %Tesla.Env{
+       status: 200,
+       body:
+         "test/fixtures/webfinger/pleroma-webfinger.json"
+         |> File.read!()
+         |> String.replace("{{nickname}}", "a")
+         |> String.replace("{{domain}}", "pleroma.example")
+         |> String.replace("{{subdomain}}", "sub.pleroma.example"),
+       headers: [{"content-type", "application/jrd+json"}]
+     }}
+  end
+
+  def get("https://sub.pleroma.example/users/a", _, _, _) do
+    {:ok,
+     %Tesla.Env{
+       status: 200,
+       body:
+         "test/fixtures/webfinger/pleroma-user.json"
+         |> File.read!()
+         |> String.replace("{{nickname}}", "a")
+         |> String.replace("{{domain}}", "sub.pleroma.example"),
+       headers: [{"content-type", "application/activity+json"}]
+     }}
+  end
+
   def get(url, query, body, headers) do
     {:error,
      "Mock response not implemented for GET #{inspect(url)}, #{query}, #{inspect(body)}, #{inspect(headers)}"}

From d536d58080d68598ca282263159f9d565a048642 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= <git@mkljczk.pl>
Date: Wed, 22 May 2024 15:53:32 +0200
Subject: [PATCH 085/192] changelog
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: marcin mikołajczak <git@mkljczk.pl>
---
 changelog.d/webfinger-validation.fix | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/webfinger-validation.fix

diff --git a/changelog.d/webfinger-validation.fix b/changelog.d/webfinger-validation.fix
new file mode 100644
index 000000000..e64312666
--- /dev/null
+++ b/changelog.d/webfinger-validation.fix
@@ -0,0 +1 @@
+Fix validate_webfinger when running a different domain for Webfinger
\ No newline at end of file

From 5f1f574f01ea18170a228a8cb273e143d2f05ab4 Mon Sep 17 00:00:00 2001
From: Lain Soykaf <lain@lain.com>
Date: Wed, 22 May 2024 18:45:34 +0400
Subject: [PATCH 086/192] WebFingerControllerTest: Restore host after test.

---
 test/pleroma/web/web_finger/web_finger_controller_test.exs | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/test/pleroma/web/web_finger/web_finger_controller_test.exs b/test/pleroma/web/web_finger/web_finger_controller_test.exs
index f501c6e44..80e072163 100644
--- a/test/pleroma/web/web_finger/web_finger_controller_test.exs
+++ b/test/pleroma/web/web_finger/web_finger_controller_test.exs
@@ -56,11 +56,6 @@ defmodule Pleroma.Web.WebFinger.WebFingerControllerTest do
   end
 
   test "reach user on tld, while pleroma is running on subdomain" do
-    Pleroma.Web.Endpoint.config_change(
-      [{Pleroma.Web.Endpoint, url: [host: "sub.example.com"]}],
-      []
-    )
-
     clear_config([Pleroma.Web.Endpoint, :url, :host], "sub.example.com")
 
     clear_config([Pleroma.Web.WebFinger, :domain], "example.com")

From 50ffbd980e8f9aee48788cea90b723c2dcca017d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= <git@mkljczk.pl>
Date: Wed, 22 May 2024 15:52:10 +0200
Subject: [PATCH 087/192] Revert "Webfinger: Allow managing account for
 subdomain"

This reverts commit 84bb854056e406d5235dd442c28127891a8a8a86.
---
 lib/pleroma/web/web_finger.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pleroma/web/web_finger.ex b/lib/pleroma/web/web_finger.ex
index 668d7d576..0d6a686c3 100644
--- a/lib/pleroma/web/web_finger.ex
+++ b/lib/pleroma/web/web_finger.ex
@@ -230,7 +230,7 @@ defmodule Pleroma.Web.WebFinger do
   defp validate_webfinger(url, %{"subject" => "acct:" <> acct} = data) do
     with %URI{host: request_host} <- URI.parse(url),
          [_name, acct_host] <- String.split(acct, "@"),
-         {_, true} <- {:hosts_match_or_subdomain, String.ends_with?(request_host, acct_host)} do
+         {_, true} <- {:hosts_match, acct_host == request_host} do
       {:ok, data}
     else
       _ -> {:error, {:webfinger_invalid, url, data}}

From b245a5c8c2a554b18f9e22c050abf59e41eda5b5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= <git@mkljczk.pl>
Date: Thu, 24 Aug 2023 00:37:39 +0200
Subject: [PATCH 088/192] Fix validate_webfinger when running a different
 domain for Webfinger
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: marcin mikołajczak <git@mkljczk.pl>
---
 lib/pleroma/application.ex    |  3 ++-
 lib/pleroma/web/web_finger.ex | 30 ++++++++++++++++++++++--------
 test/pleroma/user_test.exs    |  4 ++--
 3 files changed, 26 insertions(+), 11 deletions(-)

diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex
index e68a3c57e..385e3872d 100644
--- a/lib/pleroma/application.ex
+++ b/lib/pleroma/application.ex
@@ -210,7 +210,8 @@ defmodule Pleroma.Application do
         expiration: chat_message_id_idempotency_key_expiration(),
         limit: 500_000
       ),
-      build_cachex("rel_me", limit: 2500)
+      build_cachex("rel_me", limit: 2500),
+      build_cachex("host_meta", default_ttl: :timer.minutes(120), limit: 5000)
     ]
   end
 
diff --git a/lib/pleroma/web/web_finger.ex b/lib/pleroma/web/web_finger.ex
index 0d6a686c3..398742200 100644
--- a/lib/pleroma/web/web_finger.ex
+++ b/lib/pleroma/web/web_finger.ex
@@ -155,7 +155,16 @@ defmodule Pleroma.Web.WebFinger do
     end
   end
 
+  @cachex Pleroma.Config.get([:cachex, :provider], Cachex)
   def find_lrdd_template(domain) do
+    @cachex.fetch!(:host_meta_cache, domain, fn _ ->
+      {:commit, fetch_lrdd_template(domain)}
+    end)
+  rescue
+    e -> {:error, "Cachex error: #{inspect(e)}"}
+  end
+
+  defp fetch_lrdd_template(domain) do
     # WebFinger is restricted to HTTPS - https://tools.ietf.org/html/rfc7033#section-9.1
     meta_url = "https://#{domain}/.well-known/host-meta"
 
@@ -168,7 +177,7 @@ defmodule Pleroma.Web.WebFinger do
     end
   end
 
-  defp get_address_from_domain(domain, encoded_account) when is_binary(domain) do
+  defp get_address_from_domain(domain, "acct:" <> _ = encoded_account) when is_binary(domain) do
     case find_lrdd_template(domain) do
       {:ok, template} ->
         String.replace(template, "{uri}", encoded_account)
@@ -178,6 +187,11 @@ defmodule Pleroma.Web.WebFinger do
     end
   end
 
+  defp get_address_from_domain(domain, account) when is_binary(domain) do
+    encoded_account = URI.encode("acct:#{account}")
+    get_address_from_domain(domain, encoded_account)
+  end
+
   defp get_address_from_domain(_, _), do: {:error, :webfinger_no_domain}
 
   @spec finger(String.t()) :: {:ok, map()} | {:error, any()}
@@ -192,9 +206,7 @@ defmodule Pleroma.Web.WebFinger do
           URI.parse(account).host
       end
 
-    encoded_account = URI.encode("acct:#{account}")
-
-    with address when is_binary(address) <- get_address_from_domain(domain, encoded_account),
+    with address when is_binary(address) <- get_address_from_domain(domain, account),
          {:ok, %{status: status, body: body, headers: headers}} when status in 200..299 <-
            HTTP.get(
              address,
@@ -227,13 +239,15 @@ defmodule Pleroma.Web.WebFinger do
     end
   end
 
-  defp validate_webfinger(url, %{"subject" => "acct:" <> acct} = data) do
-    with %URI{host: request_host} <- URI.parse(url),
-         [_name, acct_host] <- String.split(acct, "@"),
+  defp validate_webfinger(request_url, %{"subject" => "acct:" <> acct = subject} = data) do
+    with [_name, acct_host] <- String.split(acct, "@"),
+         {_, url} <- {:address, get_address_from_domain(acct_host, subject)},
+         %URI{host: request_host} <- URI.parse(request_url),
+         %URI{host: acct_host} <- URI.parse(url),
          {_, true} <- {:hosts_match, acct_host == request_host} do
       {:ok, data}
     else
-      _ -> {:error, {:webfinger_invalid, url, data}}
+      _ -> {:error, {:webfinger_invalid, request_url, data}}
     end
   end
 
diff --git a/test/pleroma/user_test.exs b/test/pleroma/user_test.exs
index 7f60b959a..f64299370 100644
--- a/test/pleroma/user_test.exs
+++ b/test/pleroma/user_test.exs
@@ -872,7 +872,7 @@ defmodule Pleroma.UserTest do
     setup do: clear_config([Pleroma.Web.WebFinger, :update_nickname_on_user_fetch], true)
 
     test "for mastodon" do
-      Tesla.Mock.mock(fn
+      Tesla.Mock.mock_global(fn
         %{url: "https://example.com/.well-known/host-meta"} ->
           %Tesla.Env{
             status: 302,
@@ -930,7 +930,7 @@ defmodule Pleroma.UserTest do
     end
 
     test "for pleroma" do
-      Tesla.Mock.mock(fn
+      Tesla.Mock.mock_global(fn
         %{url: "https://example.com/.well-known/host-meta"} ->
           %Tesla.Env{
             status: 302,

From 45b5e6ecd8e647026bbdcdb454d75e5e586f5bb5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= <git@mkljczk.pl>
Date: Thu, 24 Aug 2023 01:09:00 +0200
Subject: [PATCH 089/192] Fix tests
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: marcin mikołajczak <git@mkljczk.pl>
---
 test/pleroma/user_test.exs                    | 102 +----------
 .../web_finger/web_finger_controller_test.exs |   2 +-
 test/support/http_request_mock.ex             | 171 ++++++++++++++++++
 3 files changed, 178 insertions(+), 97 deletions(-)

diff --git a/test/pleroma/user_test.exs b/test/pleroma/user_test.exs
index f64299370..b1ff52768 100644
--- a/test/pleroma/user_test.exs
+++ b/test/pleroma/user_test.exs
@@ -872,109 +872,19 @@ defmodule Pleroma.UserTest do
     setup do: clear_config([Pleroma.Web.WebFinger, :update_nickname_on_user_fetch], true)
 
     test "for mastodon" do
-      Tesla.Mock.mock_global(fn
-        %{url: "https://example.com/.well-known/host-meta"} ->
-          %Tesla.Env{
-            status: 302,
-            headers: [{"location", "https://sub.example.com/.well-known/host-meta"}]
-          }
-
-        %{url: "https://sub.example.com/.well-known/host-meta"} ->
-          %Tesla.Env{
-            status: 200,
-            body:
-              "test/fixtures/webfinger/masto-host-meta.xml"
-              |> File.read!()
-              |> String.replace("{{domain}}", "sub.example.com")
-          }
-
-        %{url: "https://sub.example.com/.well-known/webfinger?resource=acct:a@example.com"} ->
-          %Tesla.Env{
-            status: 200,
-            body:
-              "test/fixtures/webfinger/masto-webfinger.json"
-              |> File.read!()
-              |> String.replace("{{nickname}}", "a")
-              |> String.replace("{{domain}}", "example.com")
-              |> String.replace("{{subdomain}}", "sub.example.com"),
-            headers: [{"content-type", "application/jrd+json"}]
-          }
-
-        %{url: "https://sub.example.com/users/a"} ->
-          %Tesla.Env{
-            status: 200,
-            body:
-              "test/fixtures/webfinger/masto-user.json"
-              |> File.read!()
-              |> String.replace("{{nickname}}", "a")
-              |> String.replace("{{domain}}", "sub.example.com"),
-            headers: [{"content-type", "application/activity+json"}]
-          }
-
-        %{url: "https://sub.example.com/users/a/collections/featured"} ->
-          %Tesla.Env{
-            status: 200,
-            body:
-              File.read!("test/fixtures/users_mock/masto_featured.json")
-              |> String.replace("{{domain}}", "sub.example.com")
-              |> String.replace("{{nickname}}", "a"),
-            headers: [{"content-type", "application/activity+json"}]
-          }
-      end)
-
-      ap_id = "a@example.com"
+      ap_id = "a@mastodon.example"
       {:ok, fetched_user} = User.get_or_fetch(ap_id)
 
-      assert fetched_user.ap_id == "https://sub.example.com/users/a"
-      assert fetched_user.nickname == "a@example.com"
+      assert fetched_user.ap_id == "https://sub.mastodon.example/users/a"
+      assert fetched_user.nickname == "a@mastodon.example"
     end
 
     test "for pleroma" do
-      Tesla.Mock.mock_global(fn
-        %{url: "https://example.com/.well-known/host-meta"} ->
-          %Tesla.Env{
-            status: 302,
-            headers: [{"location", "https://sub.example.com/.well-known/host-meta"}]
-          }
-
-        %{url: "https://sub.example.com/.well-known/host-meta"} ->
-          %Tesla.Env{
-            status: 200,
-            body:
-              "test/fixtures/webfinger/pleroma-host-meta.xml"
-              |> File.read!()
-              |> String.replace("{{domain}}", "sub.example.com")
-          }
-
-        %{url: "https://sub.example.com/.well-known/webfinger?resource=acct:a@example.com"} ->
-          %Tesla.Env{
-            status: 200,
-            body:
-              "test/fixtures/webfinger/pleroma-webfinger.json"
-              |> File.read!()
-              |> String.replace("{{nickname}}", "a")
-              |> String.replace("{{domain}}", "example.com")
-              |> String.replace("{{subdomain}}", "sub.example.com"),
-            headers: [{"content-type", "application/jrd+json"}]
-          }
-
-        %{url: "https://sub.example.com/users/a"} ->
-          %Tesla.Env{
-            status: 200,
-            body:
-              "test/fixtures/webfinger/pleroma-user.json"
-              |> File.read!()
-              |> String.replace("{{nickname}}", "a")
-              |> String.replace("{{domain}}", "sub.example.com"),
-            headers: [{"content-type", "application/activity+json"}]
-          }
-      end)
-
-      ap_id = "a@example.com"
+      ap_id = "a@pleroma.example"
       {:ok, fetched_user} = User.get_or_fetch(ap_id)
 
-      assert fetched_user.ap_id == "https://sub.example.com/users/a"
-      assert fetched_user.nickname == "a@example.com"
+      assert fetched_user.ap_id == "https://sub.pleroma.example/users/a"
+      assert fetched_user.nickname == "a@pleroma.example"
     end
   end
 
diff --git a/test/pleroma/web/web_finger/web_finger_controller_test.exs b/test/pleroma/web/web_finger/web_finger_controller_test.exs
index 5e3ac26f9..e01cec5e4 100644
--- a/test/pleroma/web/web_finger/web_finger_controller_test.exs
+++ b/test/pleroma/web/web_finger/web_finger_controller_test.exs
@@ -48,7 +48,7 @@ defmodule Pleroma.Web.WebFinger.WebFingerControllerTest do
            ]
   end
 
-  test "reach user on tld, while pleroma is runned on subdomain" do
+  test "reach user on tld, while pleroma is running on subdomain" do
     Pleroma.Web.Endpoint.config_change(
       [{Pleroma.Web.Endpoint, url: [host: "sub.example.com"]}],
       []
diff --git a/test/support/http_request_mock.ex b/test/support/http_request_mock.ex
index 78a367024..82d8c38d7 100644
--- a/test/support/http_request_mock.ex
+++ b/test/support/http_request_mock.ex
@@ -1464,6 +1464,177 @@ defmodule HttpRequestMock do
      }}
   end
 
+  def get("https://google.com/", _, _, _) do
+    {:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/google.html")}}
+  end
+
+  def get("https://yahoo.com/", _, _, _) do
+    {:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/yahoo.html")}}
+  end
+
+  def get("https://example.com/error", _, _, _), do: {:error, :overload}
+
+  def get("https://example.com/ogp-missing-title", _, _, _) do
+    {:ok,
+     %Tesla.Env{
+       status: 200,
+       body: File.read!("test/fixtures/rich_media/ogp-missing-title.html")
+     }}
+  end
+
+  def get("https://example.com/oembed", _, _, _) do
+    {:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/oembed.html")}}
+  end
+
+  def get("https://example.com/oembed.json", _, _, _) do
+    {:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/oembed.json")}}
+  end
+
+  def get("https://example.com/twitter-card", _, _, _) do
+    {:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/twitter_card.html")}}
+  end
+
+  def get("https://example.com/non-ogp", _, _, _) do
+    {:ok,
+     %Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/non_ogp_embed.html")}}
+  end
+
+  def get("https://example.com/empty", _, _, _) do
+    {:ok, %Tesla.Env{status: 200, body: "hello"}}
+  end
+
+  def get("https://friends.grishka.me/posts/54642", _, _, _) do
+    {:ok,
+     %Tesla.Env{
+       status: 200,
+       body: File.read!("test/fixtures/tesla_mock/smithereen_non_anonymous_poll.json"),
+       headers: activitypub_object_headers()
+     }}
+  end
+
+  def get("https://friends.grishka.me/users/1", _, _, _) do
+    {:ok,
+     %Tesla.Env{
+       status: 200,
+       body: File.read!("test/fixtures/tesla_mock/smithereen_user.json"),
+       headers: activitypub_object_headers()
+     }}
+  end
+
+  def get("https://mastodon.example/.well-known/host-meta", _, _, _) do
+    {:ok,
+     %Tesla.Env{
+       status: 302,
+       headers: [{"location", "https://sub.mastodon.example/.well-known/host-meta"}]
+     }}
+  end
+
+  def get("https://sub.mastodon.example/.well-known/host-meta", _, _, _) do
+    {:ok,
+     %Tesla.Env{
+       status: 200,
+       body:
+         "test/fixtures/webfinger/masto-host-meta.xml"
+         |> File.read!()
+         |> String.replace("{{domain}}", "sub.mastodon.example")
+     }}
+  end
+
+  def get(
+        "https://sub.mastodon.example/.well-known/webfinger?resource=acct:a@mastodon.example",
+        _,
+        _,
+        _
+      ) do
+    {:ok,
+     %Tesla.Env{
+       status: 200,
+       body:
+         "test/fixtures/webfinger/masto-webfinger.json"
+         |> File.read!()
+         |> String.replace("{{nickname}}", "a")
+         |> String.replace("{{domain}}", "mastodon.example")
+         |> String.replace("{{subdomain}}", "sub.mastodon.example"),
+       headers: [{"content-type", "application/jrd+json"}]
+     }}
+  end
+
+  def get("https://sub.mastodon.example/users/a", _, _, _) do
+    {:ok,
+     %Tesla.Env{
+       status: 200,
+       body:
+         "test/fixtures/webfinger/masto-user.json"
+         |> File.read!()
+         |> String.replace("{{nickname}}", "a")
+         |> String.replace("{{domain}}", "sub.mastodon.example"),
+       headers: [{"content-type", "application/activity+json"}]
+     }}
+  end
+
+  def get("https://sub.mastodon.example/users/a/collections/featured", _, _, _) do
+    {:ok,
+     %Tesla.Env{
+       status: 200,
+       body:
+         File.read!("test/fixtures/users_mock/masto_featured.json")
+         |> String.replace("{{domain}}", "sub.mastodon.example")
+         |> String.replace("{{nickname}}", "a"),
+       headers: [{"content-type", "application/activity+json"}]
+     }}
+  end
+
+  def get("https://pleroma.example/.well-known/host-meta", _, _, _) do
+    {:ok,
+     %Tesla.Env{
+       status: 302,
+       headers: [{"location", "https://sub.pleroma.example/.well-known/host-meta"}]
+     }}
+  end
+
+  def get("https://sub.pleroma.example/.well-known/host-meta", _, _, _) do
+    {:ok,
+     %Tesla.Env{
+       status: 200,
+       body:
+         "test/fixtures/webfinger/pleroma-host-meta.xml"
+         |> File.read!()
+         |> String.replace("{{domain}}", "sub.pleroma.example")
+     }}
+  end
+
+  def get(
+        "https://sub.pleroma.example/.well-known/webfinger?resource=acct:a@pleroma.example",
+        _,
+        _,
+        _
+      ) do
+    {:ok,
+     %Tesla.Env{
+       status: 200,
+       body:
+         "test/fixtures/webfinger/pleroma-webfinger.json"
+         |> File.read!()
+         |> String.replace("{{nickname}}", "a")
+         |> String.replace("{{domain}}", "pleroma.example")
+         |> String.replace("{{subdomain}}", "sub.pleroma.example"),
+       headers: [{"content-type", "application/jrd+json"}]
+     }}
+  end
+
+  def get("https://sub.pleroma.example/users/a", _, _, _) do
+    {:ok,
+     %Tesla.Env{
+       status: 200,
+       body:
+         "test/fixtures/webfinger/pleroma-user.json"
+         |> File.read!()
+         |> String.replace("{{nickname}}", "a")
+         |> String.replace("{{domain}}", "sub.pleroma.example"),
+       headers: [{"content-type", "application/activity+json"}]
+     }}
+  end
+
   def get(url, query, body, headers) do
     {:error,
      "Mock response not implemented for GET #{inspect(url)}, #{query}, #{inspect(body)}, #{inspect(headers)}"}

From c42527dc2efe6d25310e44cfec7396c51ced5cec Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= <git@mkljczk.pl>
Date: Wed, 22 May 2024 15:53:32 +0200
Subject: [PATCH 090/192] changelog
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: marcin mikołajczak <git@mkljczk.pl>
---
 changelog.d/webfinger-validation.fix | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/webfinger-validation.fix

diff --git a/changelog.d/webfinger-validation.fix b/changelog.d/webfinger-validation.fix
new file mode 100644
index 000000000..e64312666
--- /dev/null
+++ b/changelog.d/webfinger-validation.fix
@@ -0,0 +1 @@
+Fix validate_webfinger when running a different domain for Webfinger
\ No newline at end of file

From 53a3176d2414bf4af523f1d9d13fc082fd23ea43 Mon Sep 17 00:00:00 2001
From: Lain Soykaf <lain@lain.com>
Date: Wed, 22 May 2024 18:45:34 +0400
Subject: [PATCH 091/192] WebFingerControllerTest: Restore host after test.

---
 test/pleroma/web/web_finger/web_finger_controller_test.exs | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/test/pleroma/web/web_finger/web_finger_controller_test.exs b/test/pleroma/web/web_finger/web_finger_controller_test.exs
index e01cec5e4..cc7125ce4 100644
--- a/test/pleroma/web/web_finger/web_finger_controller_test.exs
+++ b/test/pleroma/web/web_finger/web_finger_controller_test.exs
@@ -49,11 +49,6 @@ defmodule Pleroma.Web.WebFinger.WebFingerControllerTest do
   end
 
   test "reach user on tld, while pleroma is running on subdomain" do
-    Pleroma.Web.Endpoint.config_change(
-      [{Pleroma.Web.Endpoint, url: [host: "sub.example.com"]}],
-      []
-    )
-
     clear_config([Pleroma.Web.Endpoint, :url, :host], "sub.example.com")
 
     clear_config([Pleroma.Web.WebFinger, :domain], "example.com")

From 818712f99f165011aaaad5fd82c40304004ace23 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Thu, 23 May 2024 00:35:38 +0200
Subject: [PATCH 092/192] pleroma_ctl: Use realpath(1) instead of readlink(1)

From realpath(1) in POSIX 202x Draft 4.1:
> If file does not name a symbolic link, readlink shall write a diagnostic
> message to standard error and exit with non-zero status.

Which also doesn't includes `-f`, in preference of `realpath`.
---
 changelog.d/realpath-over-readlink.fix | 1 +
 rel/files/bin/pleroma_ctl              | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)
 create mode 100644 changelog.d/realpath-over-readlink.fix

diff --git a/changelog.d/realpath-over-readlink.fix b/changelog.d/realpath-over-readlink.fix
new file mode 100644
index 000000000..479561b95
--- /dev/null
+++ b/changelog.d/realpath-over-readlink.fix
@@ -0,0 +1 @@
+pleroma_ctl: Use realpath(1) instead of readlink(1)
diff --git a/rel/files/bin/pleroma_ctl b/rel/files/bin/pleroma_ctl
index 87c486514..6f0dba3a8 100755
--- a/rel/files/bin/pleroma_ctl
+++ b/rel/files/bin/pleroma_ctl
@@ -134,7 +134,7 @@ if [ -z "$1" ] || [ "$1" = "help" ]; then
 
 "
 else
-	SCRIPT=$(readlink -f "$0")
+	SCRIPT=$(realpath "$0")
 	SCRIPTPATH=$(dirname "$SCRIPT")
 
 	FULL_ARGS="$*"

From 94e4f215896dc7976a54fd146daf3e286602925a Mon Sep 17 00:00:00 2001
From: Lain Soykaf <lain@lain.com>
Date: Thu, 23 May 2024 14:38:30 +0400
Subject: [PATCH 093/192] QdrantSearch: Deal with actor restrictions

---
 lib/pleroma/search/qdrant_search.ex        | 22 ++++-
 test/pleroma/search/qdrant_search_test.exs | 95 +++++++++++++++++++++-
 2 files changed, 114 insertions(+), 3 deletions(-)

diff --git a/lib/pleroma/search/qdrant_search.ex b/lib/pleroma/search/qdrant_search.ex
index 283c43075..9cb34ef71 100644
--- a/lib/pleroma/search/qdrant_search.ex
+++ b/lib/pleroma/search/qdrant_search.ex
@@ -43,23 +43,41 @@ defmodule Pleroma.Search.QdrantSearch do
     end
   end
 
+  defp actor_from_activity(%{data: %{"actor" => actor}}) do
+    actor
+  end
+
+  defp actor_from_activity(_), do: nil
+
   defp build_index_payload(activity, embedding) do
+    actor = actor_from_activity(activity)
+    published_at = activity.data["published"]
+
     %{
       points: [
         %{
           id: activity.id |> FlakeId.from_string() |> Ecto.UUID.cast!(),
-          vector: embedding
+          vector: embedding,
+          payload: %{actor: actor, published_at: published_at}
         }
       ]
     }
   end
 
   defp build_search_payload(embedding, options) do
-    %{
+    base = %{
       vector: embedding,
       limit: options[:limit] || 20,
       offset: options[:offset] || 0
     }
+
+    if options[:actor] do
+      Map.put(base, :filter, %{
+        must: [%{key: "actor", match: %{value: options[:actor].ap_id}}]
+      })
+    else
+      base
+    end
   end
 
   @impl true
diff --git a/test/pleroma/search/qdrant_search_test.exs b/test/pleroma/search/qdrant_search_test.exs
index a2f9cc7ec..371074dcf 100644
--- a/test/pleroma/search/qdrant_search_test.exs
+++ b/test/pleroma/search/qdrant_search_test.exs
@@ -15,6 +15,94 @@ defmodule Pleroma.Search.QdrantSearchTest do
   alias Pleroma.Workers.SearchIndexingWorker
 
   describe "Qdrant search" do
+    test "searches for a term by encoding it and sending it to qdrant" do
+      user = insert(:user)
+
+      {:ok, activity} =
+        CommonAPI.post(user, %{
+          status: "guys i just don't wanna leave the swamp",
+          visibility: "public"
+        })
+
+      Config
+      |> expect(:get, 3, fn
+        [Pleroma.Search, :module], nil ->
+          QdrantSearch
+
+        [Pleroma.Search.QdrantSearch, key], nil ->
+          %{
+            openai_model: "a_model",
+            openai_url: "https://openai.url",
+            qdrant_url: "https://qdrant.url"
+          }[key]
+      end)
+
+      Tesla.Mock.mock(fn
+        %{url: "https://openai.url/v1/embeddings", method: :post} ->
+          Tesla.Mock.json(%{
+            data: [%{embedding: [1, 2, 3]}]
+          })
+
+        %{url: "https://qdrant.url/collections/posts/points/search", method: :post, body: body} ->
+          data = Jason.decode!(body)
+          refute data["filter"]
+
+          Tesla.Mock.json(%{
+            result: [%{"id" => activity.id |> FlakeId.from_string() |> Ecto.UUID.cast!()}]
+          })
+      end)
+
+      results = QdrantSearch.search(nil, "guys i just don't wanna leave the swamp", %{})
+
+      assert results == [activity]
+    end
+
+    test "for a given actor, ask for only relevant matches" do
+      user = insert(:user)
+
+      {:ok, activity} =
+        CommonAPI.post(user, %{
+          status: "guys i just don't wanna leave the swamp",
+          visibility: "public"
+        })
+
+      Config
+      |> expect(:get, 3, fn
+        [Pleroma.Search, :module], nil ->
+          QdrantSearch
+
+        [Pleroma.Search.QdrantSearch, key], nil ->
+          %{
+            openai_model: "a_model",
+            openai_url: "https://openai.url",
+            qdrant_url: "https://qdrant.url"
+          }[key]
+      end)
+
+      Tesla.Mock.mock(fn
+        %{url: "https://openai.url/v1/embeddings", method: :post} ->
+          Tesla.Mock.json(%{
+            data: [%{embedding: [1, 2, 3]}]
+          })
+
+        %{url: "https://qdrant.url/collections/posts/points/search", method: :post, body: body} ->
+          data = Jason.decode!(body)
+
+          assert data["filter"] == %{
+                   "must" => [%{"key" => "actor", "match" => %{"value" => user.ap_id}}]
+                 }
+
+          Tesla.Mock.json(%{
+            result: [%{"id" => activity.id |> FlakeId.from_string() |> Ecto.UUID.cast!()}]
+          })
+      end)
+
+      results =
+        QdrantSearch.search(nil, "guys i just don't wanna leave the swamp", %{actor: user})
+
+      assert results == [activity]
+    end
+
     test "indexes a public post on creation, deletes from the index on deletion" do
       user = insert(:user)
 
@@ -29,7 +117,12 @@ defmodule Pleroma.Search.QdrantSearchTest do
         %{method: :put, url: "https://qdrant.url/collections/posts/points", body: body} ->
           send(self(), "posted_to_qdrant")
 
-          assert match?(%{"points" => [%{"vector" => [1, 2, 3]}]}, Jason.decode!(body))
+          data = Jason.decode!(body)
+          %{"points" => [%{"vector" => vector, "payload" => payload}]} = data
+
+          assert vector == [1, 2, 3]
+          assert payload["actor"]
+          assert payload["published_at"]
 
           Tesla.Mock.json("ok")
 

From a566ad56e1434715d00067b1e49be66b6787f5ba Mon Sep 17 00:00:00 2001
From: Lain Soykaf <lain@lain.com>
Date: Thu, 23 May 2024 18:55:16 +0400
Subject: [PATCH 094/192] QdrantSearch: Fix actor / author restriction

---
 lib/pleroma/search/qdrant_search.ex        | 4 ++--
 test/pleroma/search/qdrant_search_test.exs | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/pleroma/search/qdrant_search.ex b/lib/pleroma/search/qdrant_search.ex
index 9cb34ef71..19e8cd4bf 100644
--- a/lib/pleroma/search/qdrant_search.ex
+++ b/lib/pleroma/search/qdrant_search.ex
@@ -71,9 +71,9 @@ defmodule Pleroma.Search.QdrantSearch do
       offset: options[:offset] || 0
     }
 
-    if options[:actor] do
+    if author = options[:author] do
       Map.put(base, :filter, %{
-        must: [%{key: "actor", match: %{value: options[:actor].ap_id}}]
+        must: [%{key: "actor", match: %{value: author.ap_id}}]
       })
     else
       base
diff --git a/test/pleroma/search/qdrant_search_test.exs b/test/pleroma/search/qdrant_search_test.exs
index 371074dcf..46485392e 100644
--- a/test/pleroma/search/qdrant_search_test.exs
+++ b/test/pleroma/search/qdrant_search_test.exs
@@ -98,7 +98,7 @@ defmodule Pleroma.Search.QdrantSearchTest do
       end)
 
       results =
-        QdrantSearch.search(nil, "guys i just don't wanna leave the swamp", %{actor: user})
+        QdrantSearch.search(nil, "guys i just don't wanna leave the swamp", %{author: user})
 
       assert results == [activity]
     end

From 618b77071afb480b763a493bfcd9b376effedaaf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= <git@mkljczk.pl>
Date: Sat, 25 May 2024 09:10:11 +0200
Subject: [PATCH 095/192] Update pleroma_api.md
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: marcin mikołajczak <git@mkljczk.pl>
---
 changelog.d/api-docs-2.skip         | 0
 docs/development/API/pleroma_api.md | 4 +---
 2 files changed, 1 insertion(+), 3 deletions(-)
 create mode 100644 changelog.d/api-docs-2.skip

diff --git a/changelog.d/api-docs-2.skip b/changelog.d/api-docs-2.skip
new file mode 100644
index 000000000..e69de29bb
diff --git a/docs/development/API/pleroma_api.md b/docs/development/API/pleroma_api.md
index 267dfc1ec..57d333ffe 100644
--- a/docs/development/API/pleroma_api.md
+++ b/docs/development/API/pleroma_api.md
@@ -295,9 +295,7 @@ See [Admin-API](admin_api.md)
         "id": "9umDrYheeY451cQnEe",
         "name": "Read later",
         "emoji": "🕓",
-        "source": {
-          "emoji": "🕓"
-        }
+        "emoji_url": null
     }
 ]
 ```

From 61a3b793165e92d6f23a2e59fb9b95e06737cf25 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Sat, 25 May 2024 14:20:47 -0400
Subject: [PATCH 096/192] Search backend healthcheck process

---
 changelog.d/search-healthcheck.add    |  1 +
 config/config.exs                     |  2 +-
 lib/pleroma/application.ex            |  3 +-
 lib/pleroma/search.ex                 |  5 ++
 lib/pleroma/search/database_search.ex |  3 +
 lib/pleroma/search/healthcheck.ex     | 85 +++++++++++++++++++++++++++
 lib/pleroma/search/meilisearch.ex     | 11 ++++
 lib/pleroma/search/search_backend.ex  |  8 +++
 8 files changed, 116 insertions(+), 2 deletions(-)
 create mode 100644 changelog.d/search-healthcheck.add
 create mode 100644 lib/pleroma/search/healthcheck.ex

diff --git a/changelog.d/search-healthcheck.add b/changelog.d/search-healthcheck.add
new file mode 100644
index 000000000..4974925e7
--- /dev/null
+++ b/changelog.d/search-healthcheck.add
@@ -0,0 +1 @@
+Monitoring of search backend health to control the processing of jobs in the search indexing Oban queue
diff --git a/config/config.exs b/config/config.exs
index b69044a2b..8b9a588b7 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -579,7 +579,7 @@ config :pleroma, Oban,
     attachments_cleanup: 1,
     new_users_digest: 1,
     mute_expire: 5,
-    search_indexing: 10,
+    search_indexing: [limit: 10, paused: true],
     rich_media_expiration: 2
   ],
   plugins: [Oban.Plugins.Pruner],
diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex
index 649bb11c8..d266d1836 100644
--- a/lib/pleroma/application.ex
+++ b/lib/pleroma/application.ex
@@ -109,7 +109,8 @@ defmodule Pleroma.Application do
         streamer_registry() ++
         background_migrators() ++
         shout_child(shout_enabled?()) ++
-        [Pleroma.Gopher.Server]
+        [Pleroma.Gopher.Server] ++
+        [Pleroma.Search.Healthcheck]
 
     # See http://elixir-lang.org/docs/stable/elixir/Supervisor.html
     # for other strategies and supported options
diff --git a/lib/pleroma/search.ex b/lib/pleroma/search.ex
index 3b266e59b..e8dbcca1f 100644
--- a/lib/pleroma/search.ex
+++ b/lib/pleroma/search.ex
@@ -14,4 +14,9 @@ defmodule Pleroma.Search do
 
     search_module.search(options[:for_user], query, options)
   end
+
+  def healthcheck_endpoints do
+    search_module = Pleroma.Config.get([Pleroma.Search, :module], Pleroma.Activity)
+    search_module.healthcheck_endpoints
+  end
 end
diff --git a/lib/pleroma/search/database_search.ex b/lib/pleroma/search/database_search.ex
index 31bfc7e33..11e99e7f1 100644
--- a/lib/pleroma/search/database_search.ex
+++ b/lib/pleroma/search/database_search.ex
@@ -48,6 +48,9 @@ defmodule Pleroma.Search.DatabaseSearch do
   @impl true
   def remove_from_index(_object), do: :ok
 
+  @impl true
+  def healthcheck_endpoints, do: nil
+
   def maybe_restrict_author(query, %User{} = author) do
     Activity.Queries.by_author(query, author)
   end
diff --git a/lib/pleroma/search/healthcheck.ex b/lib/pleroma/search/healthcheck.ex
new file mode 100644
index 000000000..495aee930
--- /dev/null
+++ b/lib/pleroma/search/healthcheck.ex
@@ -0,0 +1,85 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2024 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+defmodule Pleroma.Search.Healthcheck do
+  @doc """
+  Monitors health of search backend to control processing of events based on health and availability.
+  """
+  use GenServer
+  require Logger
+
+  @tick :timer.seconds(60)
+  @queue :search_indexing
+
+  def start_link(_) do
+    GenServer.start_link(__MODULE__, [], name: __MODULE__)
+  end
+
+  @impl true
+  def init(_) do
+    state = %{healthy: false}
+    {:ok, state, {:continue, :start}}
+  end
+
+  @impl true
+  def handle_continue(:start, state) do
+    tick()
+    {:noreply, state}
+  end
+
+  @impl true
+  def handle_info(:check, state) do
+    urls = Pleroma.Search.healthcheck_endpoints()
+
+    new_state =
+      if healthy?(urls) do
+        Oban.resume_queue(queue: @queue)
+        Map.put(state, :healthy, true)
+      else
+        Oban.pause_queue(queue: @queue)
+        Map.put(state, :healthy, false)
+      end
+
+    maybe_log_state_change(state, new_state)
+
+    tick()
+    {:noreply, new_state}
+  end
+
+  @impl true
+  def handle_call(:check, _from, state) do
+    status = Map.get(state, :healthy)
+
+    {:reply, status, state, :hibernate}
+  end
+
+  defp healthy?([]), do: true
+
+  defp healthy?(urls) when is_list(urls) do
+    Enum.all?(
+      urls,
+      fn url ->
+        case Pleroma.HTTP.get(url) do
+          {:ok, %{status: 200}} -> true
+          _ -> false
+        end
+      end
+    )
+  end
+
+  defp healthy?(_), do: true
+
+  defp tick do
+    Process.send_after(self(), :check, @tick)
+  end
+
+  defp maybe_log_state_change(%{healthy: true}, %{healthy: false}) do
+    Logger.error("Pausing Oban queue #{@queue} due to search backend healthcheck failure")
+  end
+
+  defp maybe_log_state_change(%{healthy: false}, %{healthy: true}) do
+    Logger.info("Resuming Oban queue #{@queue} due to search backend healthcheck pass")
+  end
+
+  defp maybe_log_state_change(_, _), do: :ok
+end
diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex
index 2bff663e8..08c2f3d86 100644
--- a/lib/pleroma/search/meilisearch.ex
+++ b/lib/pleroma/search/meilisearch.ex
@@ -178,4 +178,15 @@ defmodule Pleroma.Search.Meilisearch do
   def remove_from_index(object) do
     meili_delete("/indexes/objects/documents/#{object.id}")
   end
+
+  @impl true
+  def healthcheck_endpoints do
+    endpoint =
+      Config.get([Pleroma.Search.Meilisearch, :url])
+      |> URI.parse()
+      |> Map.put(:path, "/health")
+      |> URI.to_string()
+
+    [endpoint]
+  end
 end
diff --git a/lib/pleroma/search/search_backend.ex b/lib/pleroma/search/search_backend.ex
index 68bc48cec..13c887bc2 100644
--- a/lib/pleroma/search/search_backend.ex
+++ b/lib/pleroma/search/search_backend.ex
@@ -21,4 +21,12 @@ defmodule Pleroma.Search.SearchBackend do
   from index.
   """
   @callback remove_from_index(object :: Pleroma.Object.t()) :: :ok | {:error, any()}
+
+  @doc """
+  Healthcheck endpoints of search backend infrastructure to monitor for controlling
+  processing of jobs in the Oban queue.
+
+  It is expected a 200 response is healthy and other responses are unhealthy.
+  """
+  @callback healthcheck_endpoints :: list() | nil
 end

From 3474b42ce396150b21f26ed35bea46ad61f57d5f Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Sat, 25 May 2024 16:55:29 -0400
Subject: [PATCH 097/192] Drop TTL to 5 seconds

---
 lib/pleroma/search/healthcheck.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pleroma/search/healthcheck.ex b/lib/pleroma/search/healthcheck.ex
index 495aee930..9a2d9fdd6 100644
--- a/lib/pleroma/search/healthcheck.ex
+++ b/lib/pleroma/search/healthcheck.ex
@@ -8,7 +8,7 @@ defmodule Pleroma.Search.Healthcheck do
   use GenServer
   require Logger
 
-  @tick :timer.seconds(60)
+  @tick :timer.seconds(5)
   @queue :search_indexing
 
   def start_link(_) do

From 354b700bedf8ad6e9187245977165ebd7bc2fa1c Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Sun, 26 May 2024 14:01:00 -0400
Subject: [PATCH 098/192] Assert that AWS URLs without query parameters do not
 crash

---
 .../web/rich_media/parser/ttl/aws_signed_url_test.exs      | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/test/pleroma/web/rich_media/parser/ttl/aws_signed_url_test.exs b/test/pleroma/web/rich_media/parser/ttl/aws_signed_url_test.exs
index cd8be8675..cc28aa7f3 100644
--- a/test/pleroma/web/rich_media/parser/ttl/aws_signed_url_test.exs
+++ b/test/pleroma/web/rich_media/parser/ttl/aws_signed_url_test.exs
@@ -10,6 +10,7 @@ defmodule Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrlTest do
 
   alias Pleroma.UnstubbedConfigMock, as: ConfigMock
   alias Pleroma.Web.RichMedia.Card
+  alias Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl
 
   setup do
     ConfigMock
@@ -82,6 +83,12 @@ defmodule Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrlTest do
     assert DateTime.diff(scheduled_at, timestamp_dt) == valid_till
   end
 
+  test "AWS URL for an image without expiration works" do
+    og_data = %{"image" => "https://amazonaws.com/image.png"}
+
+    assert is_nil(AwsSignedUrl.ttl(og_data, ""))
+  end
+
   defp construct_s3_url(timestamp, valid_till) do
     "https://pleroma.s3.ap-southeast-1.amazonaws.com/sachin%20%281%29%20_a%20-%25%2Aasdasd%20BNN%20bnnn%20.png?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAIBLWWK6RGDQXDLJQ%2F20190716%2Fap-southeast-1%2Fs3%2Faws4_request&X-Amz-Date=#{timestamp}&X-Amz-Expires=#{valid_till}&X-Amz-Signature=04ffd6b98634f4b1bbabc62e0fac4879093cd54a6eed24fe8eb38e8369526bbf&X-Amz-SignedHeaders=host"
   end

From 807782b7f96ee0e053ad59b464766d750f8a8800 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Sat, 25 May 2024 16:27:59 -0400
Subject: [PATCH 099/192] Fix rich media parsing some Amazon URLs

---
 changelog.d/richmediattl.fix                            | 1 +
 lib/pleroma/web/rich_media/parser/ttl/aws_signed_url.ex | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)
 create mode 100644 changelog.d/richmediattl.fix

diff --git a/changelog.d/richmediattl.fix b/changelog.d/richmediattl.fix
new file mode 100644
index 000000000..98de63015
--- /dev/null
+++ b/changelog.d/richmediattl.fix
@@ -0,0 +1 @@
+Parsing of RichMedia TTLs for Amazon URLs when query parameters are nil
diff --git a/lib/pleroma/web/rich_media/parser/ttl/aws_signed_url.ex b/lib/pleroma/web/rich_media/parser/ttl/aws_signed_url.ex
index 948c727e1..1172a120a 100644
--- a/lib/pleroma/web/rich_media/parser/ttl/aws_signed_url.ex
+++ b/lib/pleroma/web/rich_media/parser/ttl/aws_signed_url.ex
@@ -23,7 +23,7 @@ defmodule Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl do
     %URI{host: host, query: query} = URI.parse(image)
 
     is_binary(host) and String.contains?(host, "amazonaws.com") and
-      String.contains?(query, "X-Amz-Expires")
+      is_binary(query) and String.contains?(query, "X-Amz-Expires")
   end
 
   defp aws_signed_url?(_), do: nil

From f2b0d5f1d02e243a7a1a6f339b59e5abcb8e1bd8 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Sun, 26 May 2024 14:11:41 -0400
Subject: [PATCH 100/192] Make it easier to read the state for debugging
 purposes and expose functions for testing

---
 lib/pleroma/search/healthcheck.ex | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/lib/pleroma/search/healthcheck.ex b/lib/pleroma/search/healthcheck.ex
index 9a2d9fdd6..170f29344 100644
--- a/lib/pleroma/search/healthcheck.ex
+++ b/lib/pleroma/search/healthcheck.ex
@@ -32,7 +32,7 @@ defmodule Pleroma.Search.Healthcheck do
     urls = Pleroma.Search.healthcheck_endpoints()
 
     new_state =
-      if healthy?(urls) do
+      if check(urls) do
         Oban.resume_queue(queue: @queue)
         Map.put(state, :healthy, true)
       else
@@ -47,15 +47,15 @@ defmodule Pleroma.Search.Healthcheck do
   end
 
   @impl true
-  def handle_call(:check, _from, state) do
-    status = Map.get(state, :healthy)
-
-    {:reply, status, state, :hibernate}
+  def handle_call(:state, _from, state) do
+    {:reply, state, state, :hibernate}
   end
 
-  defp healthy?([]), do: true
+  def state, do: GenServer.call(__MODULE__, :state)
 
-  defp healthy?(urls) when is_list(urls) do
+  def check([]), do: true
+
+  def check(urls) when is_list(urls) do
     Enum.all?(
       urls,
       fn url ->
@@ -67,7 +67,7 @@ defmodule Pleroma.Search.Healthcheck do
     )
   end
 
-  defp healthy?(_), do: true
+  def check(_), do: true
 
   defp tick do
     Process.send_after(self(), :check, @tick)

From 03f4b461895802259c895c81462a3e9d0d31c1e5 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Sun, 26 May 2024 14:21:24 -0400
Subject: [PATCH 101/192] Test that healthchecks behave correctly for the
 expected HTTP responses

---
 test/pleroma/search/healthcheck_test.exs | 49 ++++++++++++++++++++++++
 1 file changed, 49 insertions(+)
 create mode 100644 test/pleroma/search/healthcheck_test.exs

diff --git a/test/pleroma/search/healthcheck_test.exs b/test/pleroma/search/healthcheck_test.exs
new file mode 100644
index 000000000..e7649d949
--- /dev/null
+++ b/test/pleroma/search/healthcheck_test.exs
@@ -0,0 +1,49 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2024 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Search.HealthcheckTest do
+  use Pleroma.DataCase
+
+  import Tesla.Mock
+
+  alias Pleroma.Search.Healthcheck
+
+  @good1 "http://good1.example.com/healthz"
+  @good2 "http://good2.example.com/health"
+  @bad "http://bad.example.com/healthy"
+
+  setup do
+    mock(fn
+      %{method: :get, url: @good1} ->
+        %Tesla.Env{
+          status: 200,
+          body: ""
+        }
+
+      %{method: :get, url: @good2} ->
+        %Tesla.Env{
+          status: 200,
+          body: ""
+        }
+
+      %{method: :get, url: @bad} ->
+        %Tesla.Env{
+          status: 503,
+          body: ""
+        }
+    end)
+
+    :ok
+  end
+
+  test "true for 200 responses" do
+    assert Healthcheck.check([@good1])
+    assert Healthcheck.check([@good1, @good2])
+  end
+
+  test "false if any response is not a 200" do
+    refute Healthcheck.check([@bad])
+    refute Healthcheck.check([@good1, @bad])
+  end
+end

From d4769b076a95ce2281dba5673c410eb098445bba Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Sun, 26 May 2024 15:13:59 -0400
Subject: [PATCH 102/192] Return a 422 when trying to reply to a deleted status

---
 changelog.d/reply-to-deleted.change            |  1 +
 lib/pleroma/web/common_api/activity_draft.ex   | 18 ++++++++++++++++--
 .../controllers/status_controller_test.exs     | 10 ++++++++++
 3 files changed, 27 insertions(+), 2 deletions(-)
 create mode 100644 changelog.d/reply-to-deleted.change

diff --git a/changelog.d/reply-to-deleted.change b/changelog.d/reply-to-deleted.change
new file mode 100644
index 000000000..8b952ee7a
--- /dev/null
+++ b/changelog.d/reply-to-deleted.change
@@ -0,0 +1 @@
+A 422 error is returned when attempting to reply to a deleted status
diff --git a/lib/pleroma/web/common_api/activity_draft.ex b/lib/pleroma/web/common_api/activity_draft.ex
index bc46a8a36..8aa1e258d 100644
--- a/lib/pleroma/web/common_api/activity_draft.ex
+++ b/lib/pleroma/web/common_api/activity_draft.ex
@@ -129,8 +129,22 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do
 
   defp in_reply_to(%{params: %{in_reply_to_status_id: ""}} = draft), do: draft
 
-  defp in_reply_to(%{params: %{in_reply_to_status_id: id}} = draft) when is_binary(id) do
-    %__MODULE__{draft | in_reply_to: Activity.get_by_id(id)}
+  defp in_reply_to(%{params: %{in_reply_to_status_id: :deleted}} = draft) do
+    add_error(draft, dgettext("errors", "Cannot reply to a deleted status"))
+  end
+
+  defp in_reply_to(%{params: %{in_reply_to_status_id: id} = params} = draft) when is_binary(id) do
+    activity = Activity.get_by_id(id)
+
+    params =
+      if is_nil(activity) do
+        # Deleted activities are returned as nil
+        Map.put(params, :in_reply_to_status_id, :deleted)
+      else
+        Map.put(params, :in_reply_to_status_id, activity)
+      end
+
+    in_reply_to(%{draft | params: params})
   end
 
   defp in_reply_to(%{params: %{in_reply_to_status_id: %Activity{} = in_reply_to}} = draft) do
diff --git a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs
index 80c1ed099..f34911e5b 100644
--- a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs
+++ b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs
@@ -235,6 +235,16 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do
       assert Activity.get_in_reply_to_activity(activity).id == replied_to.id
     end
 
+    test "replying to a deleted status", %{user: user, conn: conn} do
+      {:ok, status} = CommonAPI.post(user, %{status: "cofe"})
+      {:ok, _deleted_status} = CommonAPI.delete(status.id, user)
+
+      conn
+      |> put_req_header("content-type", "application/json")
+      |> post("/api/v1/statuses", %{"status" => "xD", "in_reply_to_id" => status.id})
+      |> json_response_and_validate_schema(422)
+    end
+
     test "replying to a direct message with visibility other than direct", %{
       user: user,
       conn: conn

From d9b82255b9cf49176f8ef1d5a87abf7d80769a47 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Sun, 26 May 2024 15:23:12 -0400
Subject: [PATCH 103/192] Add an HTTP timeout for the healthcheck

---
 lib/pleroma/search/healthcheck.ex | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/lib/pleroma/search/healthcheck.ex b/lib/pleroma/search/healthcheck.ex
index 170f29344..e562c8478 100644
--- a/lib/pleroma/search/healthcheck.ex
+++ b/lib/pleroma/search/healthcheck.ex
@@ -8,8 +8,9 @@ defmodule Pleroma.Search.Healthcheck do
   use GenServer
   require Logger
 
-  @tick :timer.seconds(5)
   @queue :search_indexing
+  @tick :timer.seconds(5)
+  @timeout :timer.seconds(2)
 
   def start_link(_) do
     GenServer.start_link(__MODULE__, [], name: __MODULE__)
@@ -59,7 +60,7 @@ defmodule Pleroma.Search.Healthcheck do
     Enum.all?(
       urls,
       fn url ->
-        case Pleroma.HTTP.get(url) do
+        case Pleroma.HTTP.get(url, [], recv_timeout: @timeout) do
           {:ok, %{status: 200}} -> true
           _ -> false
         end

From d35b69d2686e62cc5076bd7a33449f98f8a11a85 Mon Sep 17 00:00:00 2001
From: Lain Soykaf <lain@lain.com>
Date: Mon, 27 May 2024 13:18:02 +0400
Subject: [PATCH 104/192] Pleroma.Search: Remove wrong (but irrelevant) results

---
 lib/pleroma/search.ex | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/lib/pleroma/search.ex b/lib/pleroma/search.ex
index e8dbcca1f..fd0218cb8 100644
--- a/lib/pleroma/search.ex
+++ b/lib/pleroma/search.ex
@@ -10,13 +10,12 @@ defmodule Pleroma.Search do
   end
 
   def search(query, options) do
-    search_module = Pleroma.Config.get([Pleroma.Search, :module], Pleroma.Activity)
-
+    search_module = Pleroma.Config.get([Pleroma.Search, :module])
     search_module.search(options[:for_user], query, options)
   end
 
   def healthcheck_endpoints do
-    search_module = Pleroma.Config.get([Pleroma.Search, :module], Pleroma.Activity)
+    search_module = Pleroma.Config.get([Pleroma.Search, :module])
     search_module.healthcheck_endpoints
   end
 end

From 8b76f56050a609bf562053cb7201a9204901490e Mon Sep 17 00:00:00 2001
From: Lain Soykaf <lain@lain.com>
Date: Mon, 27 May 2024 13:57:42 +0400
Subject: [PATCH 105/192] QdrantSearch: Add healthcheck for qdrant

---
 lib/pleroma/search/qdrant_search.ex        | 11 +++++++++++
 test/pleroma/search/qdrant_search_test.exs | 12 ++++++++++++
 2 files changed, 23 insertions(+)

diff --git a/lib/pleroma/search/qdrant_search.ex b/lib/pleroma/search/qdrant_search.ex
index 19e8cd4bf..3c3ffce16 100644
--- a/lib/pleroma/search/qdrant_search.ex
+++ b/lib/pleroma/search/qdrant_search.ex
@@ -139,6 +139,17 @@ defmodule Pleroma.Search.QdrantSearch do
         []
     end
   end
+
+  @impl true
+  def healthcheck_endpoints do
+    qdrant_health =
+      Config.get([Pleroma.Search.QdrantSearch, :qdrant_url])
+      |> URI.parse()
+      |> Map.put(:path, "/healthz")
+      |> URI.to_string()
+
+    [qdrant_health]
+  end
 end
 
 defmodule Pleroma.Search.QdrantSearch.OpenAIClient do
diff --git a/test/pleroma/search/qdrant_search_test.exs b/test/pleroma/search/qdrant_search_test.exs
index 46485392e..b389aa816 100644
--- a/test/pleroma/search/qdrant_search_test.exs
+++ b/test/pleroma/search/qdrant_search_test.exs
@@ -15,6 +15,18 @@ defmodule Pleroma.Search.QdrantSearchTest do
   alias Pleroma.Workers.SearchIndexingWorker
 
   describe "Qdrant search" do
+    test "returns the correct healthcheck endpoints" do
+      Config
+      |> expect(:get, 1, fn
+        [Pleroma.Search.QdrantSearch, key], nil ->
+          %{qdrant_url: "https://qdrant.url"}[key]
+      end)
+
+      health_endpoints = QdrantSearch.healthcheck_endpoints()
+
+      assert "https://qdrant.url/healthz" in health_endpoints
+    end
+
     test "searches for a term by encoding it and sending it to qdrant" do
       user = insert(:user)
 

From ec3f3fef7798111641f08020d5fd7ae16e407b89 Mon Sep 17 00:00:00 2001
From: Lain Soykaf <lain@lain.com>
Date: Mon, 27 May 2024 14:15:04 +0400
Subject: [PATCH 106/192] Fastembed Server: Add health check endpoint

---
 supplemental/search/fastembed-api/fastembed-server.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/supplemental/search/fastembed-api/fastembed-server.py b/supplemental/search/fastembed-api/fastembed-server.py
index dd4a7a9c8..02da69db2 100644
--- a/supplemental/search/fastembed-api/fastembed-server.py
+++ b/supplemental/search/fastembed-api/fastembed-server.py
@@ -17,6 +17,10 @@ def embeddings(request: EmbeddingRequest):
     embeddings = next(model.embed(request.input)).tolist()
     return {"data": [{"embedding": embeddings}]}
 
+@app.get("/health")
+def health():
+    return {"status": "ok"}
+
 if __name__ == "__main__":
     import uvicorn
 

From f4c04e6b2dce6d75d148ca520aaef27005ecaa82 Mon Sep 17 00:00:00 2001
From: Lain Soykaf <lain@lain.com>
Date: Mon, 27 May 2024 14:21:55 +0400
Subject: [PATCH 107/192] QdrantSearch: Add health checks.

---
 config/config.exs                          |  3 +++
 lib/pleroma/search/qdrant_search.ex        |  4 +++-
 test/pleroma/search/qdrant_search_test.exs | 20 +++++++++++++++++---
 3 files changed, 23 insertions(+), 4 deletions(-)

diff --git a/config/config.exs b/config/config.exs
index d891a5218..f388dfe52 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -919,6 +919,9 @@ config :pleroma, Pleroma.Search.QdrantSearch,
   qdrant_url: "http://127.0.0.1:6333/",
   qdrant_api_key: "",
   openai_url: "http://127.0.0.1:11345",
+  # The healthcheck url has to be set to nil when used with the real openai
+  # API, as it doesn't have a healthcheck endpoint.
+  openai_healthcheck_url: "http://127.0.0.1:11345/health",
   openai_model: "snowflake/snowflake-arctic-embed-xs",
   openai_api_key: "",
   qdrant_index_configuration: %{
diff --git a/lib/pleroma/search/qdrant_search.ex b/lib/pleroma/search/qdrant_search.ex
index 3c3ffce16..429ae05b8 100644
--- a/lib/pleroma/search/qdrant_search.ex
+++ b/lib/pleroma/search/qdrant_search.ex
@@ -148,7 +148,9 @@ defmodule Pleroma.Search.QdrantSearch do
       |> Map.put(:path, "/healthz")
       |> URI.to_string()
 
-    [qdrant_health]
+    openai_health = Config.get([Pleroma.Search.QdrantSearch, :openai_healthcheck_url])
+
+    [qdrant_health, openai_health] |> Enum.filter(& &1)
   end
 end
 
diff --git a/test/pleroma/search/qdrant_search_test.exs b/test/pleroma/search/qdrant_search_test.exs
index b389aa816..47a77a391 100644
--- a/test/pleroma/search/qdrant_search_test.exs
+++ b/test/pleroma/search/qdrant_search_test.exs
@@ -16,15 +16,29 @@ defmodule Pleroma.Search.QdrantSearchTest do
 
   describe "Qdrant search" do
     test "returns the correct healthcheck endpoints" do
+      # No openai healthcheck URL
       Config
-      |> expect(:get, 1, fn
+      |> expect(:get, 2, fn
         [Pleroma.Search.QdrantSearch, key], nil ->
           %{qdrant_url: "https://qdrant.url"}[key]
       end)
 
-      health_endpoints = QdrantSearch.healthcheck_endpoints()
+      [health_endpoint] = QdrantSearch.healthcheck_endpoints()
 
-      assert "https://qdrant.url/healthz" in health_endpoints
+      assert "https://qdrant.url/healthz" == health_endpoint
+
+      # Set openai healthcheck URL
+      Config
+      |> expect(:get, 2, fn
+        [Pleroma.Search.QdrantSearch, key], nil ->
+          %{qdrant_url: "https://qdrant.url", openai_healthcheck_url: "https://openai.url/health"}[
+            key
+          ]
+      end)
+
+      [_, health_endpoint] = QdrantSearch.healthcheck_endpoints()
+
+      assert "https://openai.url/health" == health_endpoint
     end
 
     test "searches for a term by encoding it and sending it to qdrant" do

From ddf103eca04c9571ba8310915556cc51cd4a9af8 Mon Sep 17 00:00:00 2001
From: Lain Soykaf <lain@lain.com>
Date: Mon, 27 May 2024 14:35:08 +0400
Subject: [PATCH 108/192] QdrantSearch: Fetch a post in search if possible.

---
 lib/pleroma/search/qdrant_search.ex | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/lib/pleroma/search/qdrant_search.ex b/lib/pleroma/search/qdrant_search.ex
index 429ae05b8..b659bb682 100644
--- a/lib/pleroma/search/qdrant_search.ex
+++ b/lib/pleroma/search/qdrant_search.ex
@@ -9,6 +9,7 @@ defmodule Pleroma.Search.QdrantSearch do
   alias __MODULE__.QdrantClient
 
   import Pleroma.Search.Meilisearch, only: [object_to_search_data: 1]
+  import Pleroma.Search.DatabaseSearch, only: [maybe_fetch: 3]
 
   @impl true
   def create_index do
@@ -115,8 +116,8 @@ defmodule Pleroma.Search.QdrantSearch do
   end
 
   @impl true
-  def search(_user, query, options) do
-    query = "Represent this sentence for searching relevant passages: #{query}"
+  def search(user, original_query, options) do
+    query = "Represent this sentence for searching relevant passages: #{original_query}"
 
     with {:ok, embedding} <- get_embedding(query),
          {:ok, %{body: %{"result" => result}}} <-
@@ -134,6 +135,7 @@ defmodule Pleroma.Search.QdrantSearch do
       |> Activity.restrict_deactivated_users()
       |> Ecto.Query.order_by([a], fragment("array_position(?, ?)", ^ids, a.id))
       |> Pleroma.Repo.all()
+      |> maybe_fetch(user, original_query)
     else
       _ ->
         []

From f214c2cdac4a94fae51e7679223df9557c6a1827 Mon Sep 17 00:00:00 2001
From: Lain Soykaf <lain@lain.com>
Date: Mon, 27 May 2024 15:23:33 +0400
Subject: [PATCH 109/192] NotificationTest: Remove impossible case.

---
 test/pleroma/notification_test.exs | 16 ----------------
 1 file changed, 16 deletions(-)

diff --git a/test/pleroma/notification_test.exs b/test/pleroma/notification_test.exs
index ecdb32e32..2c582c708 100644
--- a/test/pleroma/notification_test.exs
+++ b/test/pleroma/notification_test.exs
@@ -859,22 +859,6 @@ defmodule Pleroma.NotificationTest do
       assert Enum.empty?(Notification.for_user(user))
     end
 
-    test "replying to a deleted post without tagging does not generate a notification" do
-      user = insert(:user)
-      other_user = insert(:user)
-
-      {:ok, activity} = CommonAPI.post(user, %{status: "test post"})
-      {:ok, _deletion_activity} = CommonAPI.delete(activity.id, user)
-
-      {:ok, _reply_activity} =
-        CommonAPI.post(other_user, %{
-          status: "test reply",
-          in_reply_to_status_id: activity.id
-        })
-
-      assert Enum.empty?(Notification.for_user(user))
-    end
-
     test "notifications are deleted if a local user is deleted" do
       user = insert(:user)
       other_user = insert(:user)

From 3055c1598b43ee9460b88880e2752c68e9cf6edb Mon Sep 17 00:00:00 2001
From: Lain Soykaf <lain@lain.com>
Date: Mon, 27 May 2024 17:22:18 +0400
Subject: [PATCH 110/192] IPFSTest: Fix configuration mocking

---
 config/test.exs                      |  1 +
 lib/pleroma/upload.ex                |  2 +-
 lib/pleroma/uploaders/ipfs.ex        |  7 +--
 test/pleroma/uploaders/ipfs_test.exs | 70 +++++++++++++++++++---------
 4 files changed, 55 insertions(+), 25 deletions(-)

diff --git a/config/test.exs b/config/test.exs
index 9b4113dd5..3345bb3a9 100644
--- a/config/test.exs
+++ b/config/test.exs
@@ -153,6 +153,7 @@ config :pleroma, Pleroma.Uploaders.S3, config_impl: Pleroma.UnstubbedConfigMock
 config :pleroma, Pleroma.Upload, config_impl: Pleroma.UnstubbedConfigMock
 config :pleroma, Pleroma.ScheduledActivity, config_impl: Pleroma.UnstubbedConfigMock
 config :pleroma, Pleroma.Web.RichMedia.Helpers, config_impl: Pleroma.StaticStubbedConfigMock
+config :pleroma, Pleroma.Uploaders.IPFS, config_impl: Pleroma.UnstubbedConfigMock
 
 peer_module =
   if String.to_integer(System.otp_release()) >= 25 do
diff --git a/lib/pleroma/upload.ex b/lib/pleroma/upload.ex
index 2c6b23c39..35c7c02a5 100644
--- a/lib/pleroma/upload.ex
+++ b/lib/pleroma/upload.ex
@@ -282,7 +282,7 @@ defmodule Pleroma.Upload do
         end
 
       Pleroma.Uploaders.IPFS ->
-        Config.get([Pleroma.Uploaders.IPFS, :get_gateway_url])
+        @config_impl.get([Pleroma.Uploaders.IPFS, :get_gateway_url])
 
       _ ->
         public_endpoint || upload_base_url || Pleroma.Web.Endpoint.url() <> "/media/"
diff --git a/lib/pleroma/uploaders/ipfs.ex b/lib/pleroma/uploaders/ipfs.ex
index 32e06c5cf..d171e4652 100644
--- a/lib/pleroma/uploaders/ipfs.ex
+++ b/lib/pleroma/uploaders/ipfs.ex
@@ -6,11 +6,12 @@ defmodule Pleroma.Uploaders.IPFS do
   @behaviour Pleroma.Uploaders.Uploader
   require Logger
 
-  alias Pleroma.Config
   alias Tesla.Multipart
 
+  @config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config)
+
   defp get_final_url(method) do
-    config = Config.get([__MODULE__])
+    config = @config_impl.get([__MODULE__])
     post_base_url = Keyword.get(config, :post_gateway_url)
 
     Path.join([post_base_url, method])
@@ -69,7 +70,7 @@ defmodule Pleroma.Uploaders.IPFS do
   @impl true
   def delete_file(file) do
     case Pleroma.HTTP.post(delete_file_endpoint(), "", [], params: [arg: file]) do
-      {:ok, %{status_code: 204}} -> :ok
+      {:ok, %{status: 204}} -> :ok
       error -> {:error, inspect(error)}
     end
   end
diff --git a/test/pleroma/uploaders/ipfs_test.exs b/test/pleroma/uploaders/ipfs_test.exs
index 853d185e5..cf325b54f 100644
--- a/test/pleroma/uploaders/ipfs_test.exs
+++ b/test/pleroma/uploaders/ipfs_test.exs
@@ -8,22 +8,22 @@ defmodule Pleroma.Uploaders.IPFSTest do
   alias Pleroma.Uploaders.IPFS
   alias Tesla.Multipart
 
-  import Mock
   import ExUnit.CaptureLog
+  import Mock
+  import Mox
 
-  setup do
-    clear_config([Pleroma.Upload, :uploader], Pleroma.Uploaders.IPFS)
-    clear_config([Pleroma.Uploaders.IPFS])
-
-    clear_config(
-      [Pleroma.Uploaders.IPFS, :get_gateway_url],
-      "https://{CID}.ipfs.mydomain.com"
-    )
-
-    clear_config([Pleroma.Uploaders.IPFS, :post_gateway_url], "http://localhost:5001")
-  end
+  alias Pleroma.UnstubbedConfigMock, as: Config
 
   describe "get_final_url" do
+    setup do
+      Config
+      |> expect(:get, fn [Pleroma.Uploaders.IPFS] ->
+        [post_gateway_url: "http://localhost:5001"]
+      end)
+
+      :ok
+    end
+
     test "it returns the final url for put_file" do
       assert IPFS.put_file_endpoint() == "http://localhost:5001/api/v0/add"
     end
@@ -34,7 +34,21 @@ defmodule Pleroma.Uploaders.IPFSTest do
   end
 
   describe "get_file/1" do
+    setup do
+      Config
+      |> expect(:get, fn [Pleroma.Upload, :uploader] -> Pleroma.Uploaders.IPFS end)
+      |> expect(:get, fn [Pleroma.Upload, :base_url] -> nil end)
+      |> expect(:get, fn [Pleroma.Uploaders.IPFS, :public_endpoint] -> nil end)
+
+      :ok
+    end
+
     test "it returns path to ipfs file with cid as subdomain" do
+      Config
+      |> expect(:get, fn [Pleroma.Uploaders.IPFS, :get_gateway_url] ->
+        "https://{CID}.ipfs.mydomain.com"
+      end)
+
       assert IPFS.get_file("testcid") == {
                :ok,
                {:url, "https://testcid.ipfs.mydomain.com"}
@@ -42,10 +56,10 @@ defmodule Pleroma.Uploaders.IPFSTest do
     end
 
     test "it returns path to ipfs file with cid as path" do
-      clear_config(
-        [Pleroma.Uploaders.IPFS, :get_gateway_url],
+      Config
+      |> expect(:get, fn [Pleroma.Uploaders.IPFS, :get_gateway_url] ->
         "https://ipfs.mydomain.com/ipfs/{CID}"
-      )
+      end)
 
       assert IPFS.get_file("testcid") == {
                :ok,
@@ -56,6 +70,11 @@ defmodule Pleroma.Uploaders.IPFSTest do
 
   describe "put_file/1" do
     setup do
+      Config
+      |> expect(:get, fn [Pleroma.Uploaders.IPFS] ->
+        [post_gateway_url: "http://localhost:5001"]
+      end)
+
       file_upload = %Pleroma.Upload{
         name: "image-tet.jpg",
         content_type: "image/jpeg",
@@ -73,7 +92,7 @@ defmodule Pleroma.Uploaders.IPFSTest do
 
     test "save file", %{file_upload: file_upload} do
       with_mock Pleroma.HTTP,
-        post: fn "http://localhost:5001/api/v0/add", mp, [], params: ["cid-version": "1"] ->
+        post: fn "http://localhost:5001/api/v0/add", _mp, [], params: ["cid-version": "1"] ->
           {:ok,
            %Tesla.Env{
              status: 200,
@@ -88,7 +107,7 @@ defmodule Pleroma.Uploaders.IPFSTest do
 
     test "returns error", %{file_upload: file_upload} do
       with_mock Pleroma.HTTP,
-        post: fn "http://localhost:5001/api/v0/add", mp, [], params: ["cid-version": "1"] ->
+        post: fn "http://localhost:5001/api/v0/add", _mp, [], params: ["cid-version": "1"] ->
           {:error, "IPFS Gateway upload failed"}
         end do
         assert capture_log(fn ->
@@ -99,19 +118,19 @@ defmodule Pleroma.Uploaders.IPFSTest do
 
     test "returns error if JSON decode fails", %{file_upload: file_upload} do
       with_mock Pleroma.HTTP, [],
-        post: fn "http://localhost:5001/api/v0/add", mp, [], params: ["cid-version": "1"] ->
+        post: fn "http://localhost:5001/api/v0/add", _mp, [], params: ["cid-version": "1"] ->
           {:ok, %Tesla.Env{status: 200, body: "invalid"}}
         end do
         assert capture_log(fn ->
                  assert IPFS.put_file(file_upload) == {:error, "JSON decode failed"}
                end) =~
-                 "Elixir.Pleroma.Uploaders.IPFS: {:error, %Jason.DecodeError{data: \"invalid\", position: 0, token: nil}}"
+                 "Elixir.Pleroma.Uploaders.IPFS: {:error, %Jason.DecodeError"
       end
     end
 
     test "returns error if JSON body doesn't contain Hash key", %{file_upload: file_upload} do
       with_mock Pleroma.HTTP, [],
-        post: fn "http://localhost:5001/api/v0/add", mp, [], params: ["cid-version": "1"] ->
+        post: fn "http://localhost:5001/api/v0/add", _mp, [], params: ["cid-version": "1"] ->
           {:ok, %Tesla.Env{status: 200, body: "{\"key\": \"value\"}"}}
         end do
         assert IPFS.put_file(file_upload) == {:error, "JSON doesn't contain Hash key"}
@@ -120,9 +139,18 @@ defmodule Pleroma.Uploaders.IPFSTest do
   end
 
   describe "delete_file/1" do
+    setup do
+      Config
+      |> expect(:get, fn [Pleroma.Uploaders.IPFS] ->
+        [post_gateway_url: "http://localhost:5001"]
+      end)
+
+      :ok
+    end
+
     test_with_mock "deletes file", Pleroma.HTTP,
       post: fn "http://localhost:5001/api/v0/files/rm", "", [], params: [arg: "image.jpg"] ->
-        {:ok, %{status_code: 204}}
+        {:ok, %{status: 204}}
       end do
       assert :ok = IPFS.delete_file("image.jpg")
     end

From ed93af64e14e1e82cf4840b1a160df8eddecc55c Mon Sep 17 00:00:00 2001
From: Lain Soykaf <lain@lain.com>
Date: Mon, 27 May 2024 17:50:34 +0400
Subject: [PATCH 111/192] Add changelog

---
 changelog.d/add-nsfw-mrf.add | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/add-nsfw-mrf.add

diff --git a/changelog.d/add-nsfw-mrf.add b/changelog.d/add-nsfw-mrf.add
new file mode 100644
index 000000000..ce62c7ed0
--- /dev/null
+++ b/changelog.d/add-nsfw-mrf.add
@@ -0,0 +1 @@
+Add NSFW-detecting MRF

From a50c657427a2dfe9d48c25529a179fe634d30e48 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Mon, 27 May 2024 11:17:02 -0400
Subject: [PATCH 112/192] Add a dedicated connection pool for Rich Media

Sharing this pool with regular Media is problematic as Rich Media will connect to many different
domains and thrash the pool, but regular Media will have predictable connections to the webservers
hosting media for the fediverse servers you peer with.
---
 config/config.exs                     | 9 +++++++++
 lib/pleroma/web/rich_media/helpers.ex | 2 +-
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/config/config.exs b/config/config.exs
index 8b9a588b7..b8030651f 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -827,6 +827,11 @@ config :pleroma, :pools,
     max_waiting: 20,
     recv_timeout: 15_000
   ],
+  rich_media: [
+    size: 25,
+    max_waiting: 20,
+     recv_timeout: 15_000
+   ],
   upload: [
     size: 25,
     max_waiting: 5,
@@ -847,6 +852,10 @@ config :pleroma, :hackney_pools,
     max_connections: 50,
     timeout: 150_000
   ],
+  rich_media: [
+    max_connections: 50,
+    timeout: 150_000
+  ],
   upload: [
     max_connections: 25,
     timeout: 300_000
diff --git a/lib/pleroma/web/rich_media/helpers.ex b/lib/pleroma/web/rich_media/helpers.ex
index 119994458..ea41bd285 100644
--- a/lib/pleroma/web/rich_media/helpers.ex
+++ b/lib/pleroma/web/rich_media/helpers.ex
@@ -58,7 +58,7 @@ defmodule Pleroma.Web.RichMedia.Helpers do
 
   defp http_options do
     [
-      pool: :media,
+      pool: :rich_media,
       max_body: Config.get([:rich_media, :max_body], 5_000_000)
     ]
   end

From 6708f154a4f7ad46b4637d4d566b8cf81e3ebb7b Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Mon, 27 May 2024 11:18:58 -0400
Subject: [PATCH 113/192] Rework Gun connection pool sizes to make better use
 of the default 250 connections

---
 config/config.exs | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/config/config.exs b/config/config.exs
index b8030651f..a025defeb 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -818,27 +818,27 @@ config :pleroma, :connections_pool,
 
 config :pleroma, :pools,
   federation: [
-    size: 50,
-    max_waiting: 10,
+    size: 75,
+    max_waiting: 20,
     recv_timeout: 10_000
   ],
   media: [
-    size: 50,
+    size: 75,
     max_waiting: 20,
     recv_timeout: 15_000
   ],
   rich_media: [
     size: 25,
     max_waiting: 20,
-     recv_timeout: 15_000
-   ],
+    recv_timeout: 15_000
+  ],
   upload: [
     size: 25,
-    max_waiting: 5,
+    max_waiting: 20,
     recv_timeout: 15_000
   ],
   default: [
-    size: 10,
+    size: 50,
     max_waiting: 2,
     recv_timeout: 5_000
   ]

From d272eb62cd4da45e5ef82fcc0126d2cf799d292a Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Mon, 27 May 2024 11:22:45 -0400
Subject: [PATCH 114/192] Trust the connection pools to enforce the concurrency
 limitations

---
 .../web/activity_pub/mrf/media_proxy_warming_policy.ex        | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex b/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex
index c95d35bb9..f10dc3ce5 100644
--- a/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex
@@ -30,9 +30,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do
       if Pleroma.Config.get(:env) == :test do
         fetch(prefetch_url)
       else
-        ConcurrentLimiter.limit(__MODULE__, fn ->
-          Task.start(fn -> fetch(prefetch_url) end)
-        end)
+        Task.start(fn -> fetch(prefetch_url) end)
       end
     end
   end

From 37d79b76bb770cb294c0a54777b435dc49c042ab Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Mon, 27 May 2024 11:24:54 -0400
Subject: [PATCH 115/192] Use the configured http client options for mediaproxy

---
 .../web/activity_pub/mrf/media_proxy_warming_policy.ex | 10 ++++------
 lib/pleroma/web/media_proxy/media_proxy_controller.ex  |  3 ++-
 2 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex b/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex
index f10dc3ce5..e5eb6896a 100644
--- a/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex
@@ -11,11 +11,6 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do
 
   require Logger
 
-  @adapter_options [
-    pool: :media,
-    recv_timeout: 10_000
-  ]
-
   @impl true
   def history_awareness, do: :auto
 
@@ -35,7 +30,10 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do
     end
   end
 
-  defp fetch(url), do: HTTP.get(url, [], @adapter_options)
+  defp fetch(url) do
+    http_client_opts = Pleroma.Config.get([:media_proxy, :proxy_opts, :http], pool: :media)
+    HTTP.get(url, [], http_client_opts)
+  end
 
   defp preload(%{"object" => %{"attachment" => attachments}} = _message) do
     Enum.each(attachments, fn
diff --git a/lib/pleroma/web/media_proxy/media_proxy_controller.ex b/lib/pleroma/web/media_proxy/media_proxy_controller.ex
index c11484ecb..0b446e0a6 100644
--- a/lib/pleroma/web/media_proxy/media_proxy_controller.ex
+++ b/lib/pleroma/web/media_proxy/media_proxy_controller.ex
@@ -54,9 +54,10 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
 
   defp handle_preview(conn, url) do
     media_proxy_url = MediaProxy.url(url)
+    http_client_opts = Pleroma.Config.get([:media_proxy, :proxy_opts, :http], pool: :media)
 
     with {:ok, %{status: status} = head_response} when status in 200..299 <-
-           Pleroma.HTTP.request(:head, media_proxy_url, "", [], pool: :media) do
+           Pleroma.HTTP.request(:head, media_proxy_url, "", [], http_client_opts) do
       content_type = Tesla.get_header(head_response, "content-type")
       content_length = Tesla.get_header(head_response, "content-length")
       content_length = content_length && String.to_integer(content_length)

From 8b61d4e3e124c4fafeaabe58a8c7673f767b2871 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Mon, 27 May 2024 11:28:31 -0400
Subject: [PATCH 116/192] Changelogs

---
 changelog.d/mediaproxy-http.fix | 1 +
 changelog.d/pools.change        | 1 +
 2 files changed, 2 insertions(+)
 create mode 100644 changelog.d/mediaproxy-http.fix
 create mode 100644 changelog.d/pools.change

diff --git a/changelog.d/mediaproxy-http.fix b/changelog.d/mediaproxy-http.fix
new file mode 100644
index 000000000..4ff6430e0
--- /dev/null
+++ b/changelog.d/mediaproxy-http.fix
@@ -0,0 +1 @@
+Ensure MediaProxy HTTP requests obey all the defined connection settings
diff --git a/changelog.d/pools.change b/changelog.d/pools.change
new file mode 100644
index 000000000..3c689195a
--- /dev/null
+++ b/changelog.d/pools.change
@@ -0,0 +1 @@
+HTTP connection pool adjustments

From e4f1325f78e9be9fb200358d73794f15794c39bd Mon Sep 17 00:00:00 2001
From: Lain Soykaf <lain@lain.com>
Date: Mon, 27 May 2024 19:44:41 +0400
Subject: [PATCH 117/192] InetHelper: Don't use deprecated function.

---
 lib/pleroma/helpers/inet_helper.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pleroma/helpers/inet_helper.ex b/lib/pleroma/helpers/inet_helper.ex
index 3500fc679..00e18649e 100644
--- a/lib/pleroma/helpers/inet_helper.ex
+++ b/lib/pleroma/helpers/inet_helper.ex
@@ -25,6 +25,6 @@ defmodule Pleroma.Helpers.InetHelper do
         InetCidr.v6?(InetCidr.parse_address!(proxy)) -> proxy <> "/128"
       end
 
-    InetCidr.parse(proxy, true)
+    InetCidr.parse_cidr!(proxy, true)
   end
 end

From 284cd0abe5fd34d0bb31281614a7dc9249731b40 Mon Sep 17 00:00:00 2001
From: Lain Soykaf <lain@lain.com>
Date: Mon, 27 May 2024 20:04:12 +0400
Subject: [PATCH 118/192] Add changelog

---
 changelog.d/support-honk-image-summaries.add | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/support-honk-image-summaries.add

diff --git a/changelog.d/support-honk-image-summaries.add b/changelog.d/support-honk-image-summaries.add
new file mode 100644
index 000000000..052c03f95
--- /dev/null
+++ b/changelog.d/support-honk-image-summaries.add
@@ -0,0 +1 @@
+Support honk-style attachment summaries as alt-text.

From f4693dc6710c8c8ac878c2845793c7d138f90c04 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Wed, 27 Dec 2023 22:32:42 -0500
Subject: [PATCH 119/192] Update Prometheus/Grafana docs for PromEx

---
 changelog.d/prometheus-docs.change |  1 +
 docs/development/API/prometheus.md | 73 ++++++++++++++++--------------
 2 files changed, 39 insertions(+), 35 deletions(-)
 create mode 100644 changelog.d/prometheus-docs.change

diff --git a/changelog.d/prometheus-docs.change b/changelog.d/prometheus-docs.change
new file mode 100644
index 000000000..a9bd1e2e9
--- /dev/null
+++ b/changelog.d/prometheus-docs.change
@@ -0,0 +1 @@
+Update the documentation for configuring Prometheus metrics.
diff --git a/docs/development/API/prometheus.md b/docs/development/API/prometheus.md
index a5158d905..140291fe0 100644
--- a/docs/development/API/prometheus.md
+++ b/docs/development/API/prometheus.md
@@ -1,44 +1,47 @@
-# Prometheus Metrics
+# Prometheus / OpenTelemetry Metrics
 
-Pleroma includes support for exporting metrics via the [prometheus_ex](https://github.com/deadtrickster/prometheus.ex) library.
+Pleroma includes support for exporting metrics via the [prom_ex](https://github.com/akoutmos/prom_ex) library.
+The metrics are exposed by a dedicated webserver/port to improve privacy and security.
 
 Config example:
 
 ```
-config :prometheus, Pleroma.Web.Endpoint.MetricsExporter,
-  enabled: true,
-  auth: {:basic, "myusername", "mypassword"},
-  ip_whitelist: ["127.0.0.1"],
-  path: "/api/pleroma/app_metrics",
-  format: :text
-```
-
-* `enabled` (Pleroma extension) enables the endpoint
-* `ip_whitelist` (Pleroma extension) could be used to restrict access only to specified IPs
-* `auth` sets the authentication (`false` for no auth; configurable to HTTP Basic Auth, see [prometheus-plugs](https://github.com/deadtrickster/prometheus-plugs#exporting) documentation)
-* `format` sets the output format (`:text` or `:protobuf`)
-* `path` sets the path to app metrics page 
-
-
-## `/api/pleroma/app_metrics`
-
-### Exports Prometheus application metrics
-
-* Method: `GET`
-* Authentication: not required by default (see configuration options above)
-* Params: none
-* Response: text
-
-## Grafana
-
-### Config example
-
-The following is a config example to use with [Grafana](https://grafana.com)
+config :pleroma, Pleroma.PromEx,
+  disabled: false,
+  manual_metrics_start_delay: :no_delay,
+  drop_metrics_groups: [],
+  grafana: [
+    host: System.get_env("GRAFANA_HOST", "http://localhost:3000"),
+    auth_token: System.get_env("GRAFANA_TOKEN"),
+    upload_dashboards_on_start: false,
+    folder_name: "BEAM",
+    annotate_app_lifecycle: true
+  ],
+  metrics_server: [
+    port: 4021,
+    path: "/metrics",
+    protocol: :http,
+    pool_size: 5,
+    cowboy_opts: [],
+    auth_strategy: :none
+  ],
+  datasource: "Prometheus"
 
 ```
-  - job_name: 'beam'
-    metrics_path: /api/pleroma/app_metrics
-    scheme: https
+
+PromEx supports the ability to automatically publish dashboards to your Grafana server as well as register Annotations. If you do not wish to configure this capability you must generate the dashboard JSON files and import them directly. You can find the mix commands in the upstream [documentation](https://hexdocs.pm/prom_ex/Mix.Tasks.PromEx.Dashboard.Export.html). You can find the list of modules enabled in Pleroma for which you should generate dashboards for by examining the contents of the `lib/pleroma/prom_ex.ex` module.
+
+## prometheus.yml
+
+The following is a bare minimum config example to use with [Prometheus](https://prometheus.io) or Prometheus-compatible software like [VictoriaMetrics](https://victoriametrics.com).
+
+```
+global:
+  scrape_interval: 15s
+
+scrape_configs:
+  - job_name: 'pleroma'
+    scheme: http
     static_configs:
-    - targets: ['pleroma.soykaf.com']
+    - targets: ['pleroma.soykaf.com:4021']
 ```

From 7258ab1aed53da796e24bdab81f39ea1d358a549 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Mon, 27 May 2024 12:20:00 -0400
Subject: [PATCH 120/192] Changelog

---
 changelog.d/promexdocs.add | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/promexdocs.add

diff --git a/changelog.d/promexdocs.add b/changelog.d/promexdocs.add
new file mode 100644
index 000000000..dda972994
--- /dev/null
+++ b/changelog.d/promexdocs.add
@@ -0,0 +1 @@
+PromEx documentation

From 0bddca361d12f347ca9907c5ddb5d1464a17b32a Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Sun, 24 Jan 2021 14:56:45 -0600
Subject: [PATCH 121/192] DNSRBL in an MRF

---
 changelog.d/add-rbl-mrf.add                   |   1 +
 config/config.exs                             |   5 +
 .../web/activity_pub/mrf/dnsrbl_policy.ex     | 142 ++++++++++++++++++
 3 files changed, 148 insertions(+)
 create mode 100644 changelog.d/add-rbl-mrf.add
 create mode 100644 lib/pleroma/web/activity_pub/mrf/dnsrbl_policy.ex

diff --git a/changelog.d/add-rbl-mrf.add b/changelog.d/add-rbl-mrf.add
new file mode 100644
index 000000000..363270fb9
--- /dev/null
+++ b/changelog.d/add-rbl-mrf.add
@@ -0,0 +1 @@
+Add DNSRBL MRF
diff --git a/config/config.exs b/config/config.exs
index b93de52e1..1fb0f3911 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -410,6 +410,11 @@ config :pleroma, :mrf_vocabulary,
   accept: [],
   reject: []
 
+config :pleroma, :mrf_dnsrbl,
+  nameserver: "127.0.0.1",
+  port: 53,
+  zone: "bl.pleroma.com"
+
 # threshold of 7 days
 config :pleroma, :mrf_object_age,
   threshold: 604_800,
diff --git a/lib/pleroma/web/activity_pub/mrf/dnsrbl_policy.ex b/lib/pleroma/web/activity_pub/mrf/dnsrbl_policy.ex
new file mode 100644
index 000000000..9543cc545
--- /dev/null
+++ b/lib/pleroma/web/activity_pub/mrf/dnsrbl_policy.ex
@@ -0,0 +1,142 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2024 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.MRF.DNSRBLPolicy do
+  @moduledoc """
+  Dynamic activity filtering based on an RBL database
+
+  This MRF makes queries to a custom DNS server which will
+  respond with values indicating the classification of the domain
+  the activity originated from. This method has been widely used
+  in the email anti-spam industry for very fast reputation checks.
+
+  e.g., if the DNS response is 127.0.0.1 or empty, the domain is OK
+  Other values such as 127.0.0.2 may be used for specific classifications.
+
+  Information for why the host is blocked can be stored in a corresponding TXT record.
+
+  This method is fail-open so if the queries fail the activites are accepted.
+
+  An example of software meant for this purpsoe is rbldnsd which can be found
+  at http://www.corpit.ru/mjt/rbldnsd.html or mirrored at
+  https://git.pleroma.social/feld/rbldnsd
+
+  It is highly recommended that you run your own copy of rbldnsd and use an
+  external mechanism to sync/share the contents of the zone file. This is
+  important to keep the latency on the queries as low as possible and prevent
+  your DNS server from being attacked so it fails and content is permitted.
+  """
+
+  @behaviour Pleroma.Web.ActivityPub.MRF.Policy
+
+  alias Pleroma.Config
+
+  require Logger
+
+  @query_retries 1
+  @query_timeout 500
+
+  @impl true
+  def filter(%{"actor" => actor} = object) do
+    actor_info = URI.parse(actor)
+
+    with {:ok, object} <- check_rbl(actor_info, object) do
+      {:ok, object}
+    else
+      _ -> {:reject, "[DNSRBLPolicy]"}
+    end
+  end
+
+  @impl true
+  def filter(object), do: {:ok, object}
+
+  @impl true
+  def describe do
+    mrf_dnsrbl =
+      Config.get(:mrf_dnsrbl)
+      |> Enum.into(%{})
+
+    {:ok, %{mrf_dnsrbl: mrf_dnsrbl}}
+  end
+
+  @impl true
+  def config_description do
+    %{
+      key: :mrf_dnsrbl,
+      related_policy: "Pleroma.Web.ActivityPub.MRF.DNSRBLPolicy",
+      label: "MRF DNSRBL",
+      description: "DNS RealTime Blackhole Policy",
+      children: [
+        %{
+          key: :nameserver,
+          type: {:string},
+          description: "DNSRBL Nameserver to Query (IP or hostame)",
+          suggestions: ["127.0.0.1"]
+        },
+        %{
+          key: :port,
+          type: {:string},
+          description: "Nameserver port",
+          suggestions: ["53"]
+        },
+        %{
+          key: :zone,
+          type: {:string},
+          description: "Root zone for querying",
+          suggestions: ["bl.pleroma.com"]
+        }
+      ]
+    }
+  end
+
+  defp check_rbl(%{host: actor_host}, object) do
+    with false <- match?(^actor_host, Pleroma.Web.Endpoint.host()),
+         zone when not is_nil(zone) <- Keyword.get(Config.get([:mrf_dnsrbl]), :zone) do
+      query =
+        Enum.join([actor_host, zone], ".")
+        |> String.to_charlist()
+
+      rbl_response = rblquery(query)
+
+      if Enum.empty?(rbl_response) do
+        {:ok, object}
+      else
+        Task.start(fn ->
+          reason = rblquery(query, :txt) || "undefined"
+
+          Logger.warning(
+            "DNSRBL Rejected activity from #{actor_host} for reason: #{inspect(reason)}"
+          )
+        end)
+
+        :error
+      end
+    else
+      _ -> {:ok, object}
+    end
+  end
+
+  defp get_rblhost_ip(rblhost) do
+    case rblhost |> String.to_charlist() |> :inet_parse.address() do
+      {:ok, _} -> rblhost |> String.to_charlist() |> :inet_parse.address()
+      _ -> {:ok, rblhost |> String.to_charlist() |> :inet_res.lookup(:in, :a) |> Enum.random()}
+    end
+  end
+
+  defp rblquery(query, type \\ :a) do
+    config = Config.get([:mrf_dnsrbl])
+
+    case get_rblhost_ip(config[:nameserver]) do
+      {:ok, rblnsip} ->
+        :inet_res.lookup(query, :in, type,
+          nameservers: [{rblnsip, config[:port]}],
+          timeout: @query_timeout,
+          retry: @query_retries
+        )
+
+      _ ->
+        []
+    end
+  end
+end

From 5e963736cee55aa8f4bb9d9fba451ff3864ddaa8 Mon Sep 17 00:00:00 2001
From: Alex Gleason <alex@alexgleason.me>
Date: Sun, 21 May 2023 15:26:02 -0500
Subject: [PATCH 122/192] Add AntiMentionSpamPolicy

---
 .../mrf/anti_mention_spam_policy.ex           | 87 +++++++++++++++++++
 .../mrf/anti_mention_spam_policy_test.exs     | 65 ++++++++++++++
 2 files changed, 152 insertions(+)
 create mode 100644 lib/pleroma/web/activity_pub/mrf/anti_mention_spam_policy.ex
 create mode 100644 test/pleroma/web/activity_pub/mrf/anti_mention_spam_policy_test.exs

diff --git a/lib/pleroma/web/activity_pub/mrf/anti_mention_spam_policy.ex b/lib/pleroma/web/activity_pub/mrf/anti_mention_spam_policy.ex
new file mode 100644
index 000000000..ad97a1552
--- /dev/null
+++ b/lib/pleroma/web/activity_pub/mrf/anti_mention_spam_policy.ex
@@ -0,0 +1,87 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.MRF.AntiMentionSpamPolicy do
+  alias Pleroma.User
+  require Pleroma.Constants
+
+  @behaviour Pleroma.Web.ActivityPub.MRF.Policy
+
+  defp user_has_followers?(%User{} = u), do: u.follower_count > 0
+  defp user_has_posted?(%User{} = u), do: u.note_count > 0
+
+  defp user_has_age?(%User{} = u) do
+    now = NaiveDateTime.utc_now()
+    diff = u.inserted_at |> NaiveDateTime.diff(now, :second)
+    diff > :timer.seconds(30)
+  end
+
+  defp good_reputation?(%User{} = u) do
+    user_has_age?(u) and user_has_followers?(u) and user_has_posted?(u)
+  end
+
+  # copied from HellthreadPolicy
+  defp get_recipient_count(message) do
+    recipients = (message["to"] || []) ++ (message["cc"] || [])
+
+    follower_collection =
+      User.get_cached_by_ap_id(message["actor"] || message["attributedTo"]).follower_address
+
+    if Enum.member?(recipients, Pleroma.Constants.as_public()) do
+      recipients =
+        recipients
+        |> List.delete(Pleroma.Constants.as_public())
+        |> List.delete(follower_collection)
+
+      {:public, length(recipients)}
+    else
+      recipients =
+        recipients
+        |> List.delete(follower_collection)
+
+      {:not_public, length(recipients)}
+    end
+  end
+
+  defp object_has_recipients?(%{"object" => object} = activity) do
+    {_, object_count} = get_recipient_count(object)
+    {_, activity_count} = get_recipient_count(activity)
+    object_count + activity_count > 0
+  end
+
+  defp object_has_recipients?(object) do
+    {_, count} = get_recipient_count(object)
+    count > 0
+  end
+
+  @impl true
+  def filter(%{"type" => "Create", "actor" => actor} = activity) do
+    with {:ok, %User{local: false} = u} <- User.get_or_fetch_by_ap_id(actor),
+         {:has_mentions, true} <- {:has_mentions, object_has_recipients?(activity)},
+         {:good_reputation, true} <- {:good_reputation, good_reputation?(u)} do
+      {:ok, activity}
+    else
+      {:ok, %User{local: true}} ->
+        {:ok, activity}
+
+      {:has_mentions, false} ->
+        {:ok, activity}
+
+      {:good_reputation, false} ->
+        {:reject, "[AntiMentionSpamPolicy] User rejected"}
+
+      {:error, _} ->
+        {:reject, "[AntiMentionSpamPolicy] Failed to get or fetch user by ap_id"}
+
+      e ->
+        {:reject, "[AntiMentionSpamPolicy] Unhandled error #{inspect(e)}"}
+    end
+  end
+
+  # in all other cases, pass through
+  def filter(message), do: {:ok, message}
+
+  @impl true
+  def describe, do: {:ok, %{}}
+end
diff --git a/test/pleroma/web/activity_pub/mrf/anti_mention_spam_policy_test.exs b/test/pleroma/web/activity_pub/mrf/anti_mention_spam_policy_test.exs
new file mode 100644
index 000000000..63947858c
--- /dev/null
+++ b/test/pleroma/web/activity_pub/mrf/anti_mention_spam_policy_test.exs
@@ -0,0 +1,65 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.MRF.AntiMentionSpamPolicyTest do
+  use Pleroma.DataCase
+  import Pleroma.Factory
+  alias Pleroma.Web.ActivityPub.MRF.AntiMentionSpamPolicy
+
+  test "it allows posts without mentions" do
+    user = insert(:user, local: false)
+    assert user.note_count == 0
+
+    message = %{
+      "type" => "Create",
+      "actor" => user.ap_id
+    }
+
+    {:ok, _message} = AntiMentionSpamPolicy.filter(message)
+  end
+
+  test "it allows posts from users with followers, posts, and age" do
+    user =
+      insert(:user,
+        local: false,
+        follower_count: 1,
+        note_count: 1,
+        inserted_at: ~N[1970-01-01 00:00:00]
+      )
+
+    message = %{
+      "type" => "Create",
+      "actor" => user.ap_id
+    }
+
+    {:ok, _message} = AntiMentionSpamPolicy.filter(message)
+  end
+
+  test "it allows posts from local users" do
+    user = insert(:user, local: true)
+
+    message = %{
+      "type" => "Create",
+      "actor" => user.ap_id
+    }
+
+    {:ok, _message} = AntiMentionSpamPolicy.filter(message)
+  end
+
+  test "it rejects posts with mentions from users without followers" do
+    user = insert(:user, local: false, follower_count: 0)
+
+    message = %{
+      "type" => "Create",
+      "actor" => user.ap_id,
+      "object" => %{
+        "to" => ["https://pleroma.soykaf.com/users/1"],
+        "cc" => ["https://pleroma.soykaf.com/users/1"],
+        "actor" => user.ap_id
+      }
+    }
+
+    {:reject, _message} = AntiMentionSpamPolicy.filter(message)
+  end
+end

From 64cacc3694c0441d3f3f5886b301bbf93f590cb6 Mon Sep 17 00:00:00 2001
From: Alex Gleason <alex@alexgleason.me>
Date: Sun, 21 May 2023 19:31:56 -0500
Subject: [PATCH 123/192] AntiMentionSpamPolicy: fix user age check

---
 lib/pleroma/web/activity_pub/mrf/anti_mention_spam_policy.ex | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/mrf/anti_mention_spam_policy.ex b/lib/pleroma/web/activity_pub/mrf/anti_mention_spam_policy.ex
index ad97a1552..0cb3313b2 100644
--- a/lib/pleroma/web/activity_pub/mrf/anti_mention_spam_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/anti_mention_spam_policy.ex
@@ -12,9 +12,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.AntiMentionSpamPolicy do
   defp user_has_posted?(%User{} = u), do: u.note_count > 0
 
   defp user_has_age?(%User{} = u) do
-    now = NaiveDateTime.utc_now()
-    diff = u.inserted_at |> NaiveDateTime.diff(now, :second)
-    diff > :timer.seconds(30)
+    diff = NaiveDateTime.utc_now() |> NaiveDateTime.diff(u.inserted_at, :second)
+    diff >= :timer.seconds(30)
   end
 
   defp good_reputation?(%User{} = u) do

From 02d8ce8f0ba615fa0946064052113fb05dd0b6a2 Mon Sep 17 00:00:00 2001
From: Alex Gleason <alex@alexgleason.me>
Date: Mon, 22 May 2023 15:50:34 -0500
Subject: [PATCH 124/192] AntiMentionSpamPolicy: remove followers check

---
 lib/pleroma/web/activity_pub/mrf/anti_mention_spam_policy.ex | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/mrf/anti_mention_spam_policy.ex b/lib/pleroma/web/activity_pub/mrf/anti_mention_spam_policy.ex
index 0cb3313b2..9cdb2077f 100644
--- a/lib/pleroma/web/activity_pub/mrf/anti_mention_spam_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/anti_mention_spam_policy.ex
@@ -8,7 +8,6 @@ defmodule Pleroma.Web.ActivityPub.MRF.AntiMentionSpamPolicy do
 
   @behaviour Pleroma.Web.ActivityPub.MRF.Policy
 
-  defp user_has_followers?(%User{} = u), do: u.follower_count > 0
   defp user_has_posted?(%User{} = u), do: u.note_count > 0
 
   defp user_has_age?(%User{} = u) do
@@ -17,7 +16,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.AntiMentionSpamPolicy do
   end
 
   defp good_reputation?(%User{} = u) do
-    user_has_age?(u) and user_has_followers?(u) and user_has_posted?(u)
+    user_has_age?(u) and user_has_posted?(u)
   end
 
   # copied from HellthreadPolicy

From 0d092a3d4fd89a7f8df30f080087bd24ce53c597 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Mon, 27 May 2024 12:26:55 -0400
Subject: [PATCH 125/192] Changelog

---
 changelog.d/anti-mentionspam-mrf.add | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/anti-mentionspam-mrf.add

diff --git a/changelog.d/anti-mentionspam-mrf.add b/changelog.d/anti-mentionspam-mrf.add
new file mode 100644
index 000000000..9466f85f4
--- /dev/null
+++ b/changelog.d/anti-mentionspam-mrf.add
@@ -0,0 +1 @@
+Add Anti-mention Spam MRF backported from Rebased

From cab6372d7a1bdf50436eff1b4023fd6e05586dbc Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Mon, 27 May 2024 12:31:29 -0400
Subject: [PATCH 126/192] Make user age limit configurable

Switch to milliseconds for consistency with other configuration options in codebase
---
 config/config.exs                                           | 2 ++
 .../web/activity_pub/mrf/anti_mention_spam_policy.ex        | 6 ++++--
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/config/config.exs b/config/config.exs
index b93de52e1..51773d830 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -430,6 +430,8 @@ config :pleroma, :mrf_force_mention,
   mention_parent: true,
   mention_quoted: true
 
+config :pleroma, :mrf_antimentionspam, user_age_limit: 30_000
+
 config :pleroma, :rich_media,
   enabled: true,
   ignore_hosts: [],
diff --git a/lib/pleroma/web/activity_pub/mrf/anti_mention_spam_policy.ex b/lib/pleroma/web/activity_pub/mrf/anti_mention_spam_policy.ex
index 9cdb2077f..531e75ce8 100644
--- a/lib/pleroma/web/activity_pub/mrf/anti_mention_spam_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/anti_mention_spam_policy.ex
@@ -3,6 +3,7 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.ActivityPub.MRF.AntiMentionSpamPolicy do
+  alias Pleroma.Config
   alias Pleroma.User
   require Pleroma.Constants
 
@@ -11,8 +12,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.AntiMentionSpamPolicy do
   defp user_has_posted?(%User{} = u), do: u.note_count > 0
 
   defp user_has_age?(%User{} = u) do
-    diff = NaiveDateTime.utc_now() |> NaiveDateTime.diff(u.inserted_at, :second)
-    diff >= :timer.seconds(30)
+    user_age_limit = Config.get([:mrf_antimentionspam, :user_age_limit], 30_000)
+    diff = NaiveDateTime.utc_now() |> NaiveDateTime.diff(u.inserted_at, :millisecond)
+    diff >= user_age_limit
   end
 
   defp good_reputation?(%User{} = u) do

From 1c699144d23aa4a86ff8b6ebef7d760ce9e3a4e2 Mon Sep 17 00:00:00 2001
From: Lain Soykaf <lain@lain.com>
Date: Mon, 27 May 2024 21:26:40 +0400
Subject: [PATCH 127/192] HttpSecurityPlug: Don't allow unsafe-eval by default

---
 config/config.exs                             |   3 +-
 config/test.exs                               |   1 +
 lib/pleroma/application.ex                    |   7 +-
 lib/pleroma/web/plugs/http_security_plug.ex   |  49 +++--
 .../web/plugs/http_security_plug_test.exs     | 208 ++++++++++++++----
 5 files changed, 204 insertions(+), 64 deletions(-)

diff --git a/config/config.exs b/config/config.exs
index 4752bbbde..f861daf04 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -519,7 +519,8 @@ config :pleroma, :http_security,
   sts: false,
   sts_max_age: 31_536_000,
   ct_max_age: 2_592_000,
-  referrer_policy: "same-origin"
+  referrer_policy: "same-origin",
+  allow_unsafe_eval: false
 
 config :cors_plug,
   max_age: 86_400,
diff --git a/config/test.exs b/config/test.exs
index 3345bb3a9..b5c9c6e4a 100644
--- a/config/test.exs
+++ b/config/test.exs
@@ -154,6 +154,7 @@ config :pleroma, Pleroma.Upload, config_impl: Pleroma.UnstubbedConfigMock
 config :pleroma, Pleroma.ScheduledActivity, config_impl: Pleroma.UnstubbedConfigMock
 config :pleroma, Pleroma.Web.RichMedia.Helpers, config_impl: Pleroma.StaticStubbedConfigMock
 config :pleroma, Pleroma.Uploaders.IPFS, config_impl: Pleroma.UnstubbedConfigMock
+config :pleroma, Pleroma.Web.Plugs.HTTPSecurityPlug, config_impl: Pleroma.UnstubbedConfigMock
 
 peer_module =
   if String.to_integer(System.otp_release()) >= 25 do
diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex
index d266d1836..0d9757b44 100644
--- a/lib/pleroma/application.ex
+++ b/lib/pleroma/application.ex
@@ -14,6 +14,7 @@ defmodule Pleroma.Application do
   @name Mix.Project.config()[:name]
   @version Mix.Project.config()[:version]
   @repository Mix.Project.config()[:source_url]
+  @compile_env Mix.env()
 
   def name, do: @name
   def version, do: @version
@@ -51,7 +52,11 @@ defmodule Pleroma.Application do
     Pleroma.HTML.compile_scrubbers()
     Pleroma.Config.Oban.warn()
     Config.DeprecationWarnings.warn()
-    Pleroma.Web.Plugs.HTTPSecurityPlug.warn_if_disabled()
+
+    if @compile_env != :test do
+      Pleroma.Web.Plugs.HTTPSecurityPlug.warn_if_disabled()
+    end
+
     Pleroma.ApplicationRequirements.verify!()
     load_custom_modules()
     Pleroma.Docs.JSON.compile()
diff --git a/lib/pleroma/web/plugs/http_security_plug.ex b/lib/pleroma/web/plugs/http_security_plug.ex
index a27dcd0ab..a1dc6c02a 100644
--- a/lib/pleroma/web/plugs/http_security_plug.ex
+++ b/lib/pleroma/web/plugs/http_security_plug.ex
@@ -3,26 +3,27 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do
-  alias Pleroma.Config
   import Plug.Conn
 
   require Logger
 
+  @config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config)
+
   def init(opts), do: opts
 
   def call(conn, _options) do
-    if Config.get([:http_security, :enabled]) do
+    if @config_impl.get([:http_security, :enabled]) do
       conn
       |> merge_resp_headers(headers())
-      |> maybe_send_sts_header(Config.get([:http_security, :sts]))
+      |> maybe_send_sts_header(@config_impl.get([:http_security, :sts]))
     else
       conn
     end
   end
 
   def primary_frontend do
-    with %{"name" => frontend} <- Config.get([:frontends, :primary]),
-         available <- Config.get([:frontends, :available]),
+    with %{"name" => frontend} <- @config_impl.get([:frontends, :primary]),
+         available <- @config_impl.get([:frontends, :available]),
          %{} = primary_frontend <- Map.get(available, frontend) do
       {:ok, primary_frontend}
     end
@@ -37,8 +38,8 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do
   end
 
   def headers do
-    referrer_policy = Config.get([:http_security, :referrer_policy])
-    report_uri = Config.get([:http_security, :report_uri])
+    referrer_policy = @config_impl.get([:http_security, :referrer_policy])
+    report_uri = @config_impl.get([:http_security, :report_uri])
     custom_http_frontend_headers = custom_http_frontend_headers()
 
     headers = [
@@ -86,10 +87,10 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do
   @csp_start [Enum.join(static_csp_rules, ";") <> ";"]
 
   defp csp_string do
-    scheme = Config.get([Pleroma.Web.Endpoint, :url])[:scheme]
+    scheme = @config_impl.get([Pleroma.Web.Endpoint, :url])[:scheme]
     static_url = Pleroma.Web.Endpoint.static_url()
     websocket_url = Pleroma.Web.Endpoint.websocket_url()
-    report_uri = Config.get([:http_security, :report_uri])
+    report_uri = @config_impl.get([:http_security, :report_uri])
 
     img_src = "img-src 'self' data: blob:"
     media_src = "media-src 'self'"
@@ -97,8 +98,8 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do
 
     # Strict multimedia CSP enforcement only when MediaProxy is enabled
     {img_src, media_src, connect_src} =
-      if Config.get([:media_proxy, :enabled]) &&
-           !Config.get([:media_proxy, :proxy_opts, :redirect_on_failure]) do
+      if @config_impl.get([:media_proxy, :enabled]) &&
+           !@config_impl.get([:media_proxy, :proxy_opts, :redirect_on_failure]) do
         sources = build_csp_multimedia_source_list()
 
         {
@@ -115,17 +116,21 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do
       end
 
     connect_src =
-      if Config.get(:env) == :dev do
+      if @config_impl.get(:env) == :dev do
         [connect_src, " http://localhost:3035/"]
       else
         connect_src
       end
 
     script_src =
-      if Config.get(:env) == :dev do
-        "script-src 'self' 'unsafe-eval'"
+      if @config_impl.get([:http_security, :allow_unsafe_eval]) do
+        if @config_impl.get(:env) == :dev do
+          "script-src 'self' 'unsafe-eval'"
+        else
+          "script-src 'self' 'wasm-unsafe-eval'"
+        end
       else
-        "script-src 'self' 'wasm-unsafe-eval'"
+        "script-src 'self'"
       end
 
     report = if report_uri, do: ["report-uri ", report_uri, ";report-to csp-endpoint"]
@@ -161,11 +166,11 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do
   defp build_csp_multimedia_source_list do
     media_proxy_whitelist =
       [:media_proxy, :whitelist]
-      |> Config.get()
+      |> @config_impl.get()
       |> build_csp_from_whitelist([])
 
-    captcha_method = Config.get([Pleroma.Captcha, :method])
-    captcha_endpoint = Config.get([captcha_method, :endpoint])
+    captcha_method = @config_impl.get([Pleroma.Captcha, :method])
+    captcha_endpoint = @config_impl.get([captcha_method, :endpoint])
 
     base_endpoints =
       [
@@ -173,7 +178,7 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do
         [Pleroma.Upload, :base_url],
         [Pleroma.Uploaders.S3, :public_endpoint]
       ]
-      |> Enum.map(&Config.get/1)
+      |> Enum.map(&@config_impl.get/1)
 
     [captcha_endpoint | base_endpoints]
     |> Enum.map(&build_csp_param/1)
@@ -200,7 +205,7 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do
   end
 
   def warn_if_disabled do
-    unless Config.get([:http_security, :enabled]) do
+    unless Pleroma.Config.get([:http_security, :enabled]) do
       Logger.warning("
                                  .i;;;;i.
                                iYcviii;vXY:
@@ -245,8 +250,8 @@ your instance and your users via malicious posts:
   end
 
   defp maybe_send_sts_header(conn, true) do
-    max_age_sts = Config.get([:http_security, :sts_max_age])
-    max_age_ct = Config.get([:http_security, :ct_max_age])
+    max_age_sts = @config_impl.get([:http_security, :sts_max_age])
+    max_age_ct = @config_impl.get([:http_security, :ct_max_age])
 
     merge_resp_headers(conn, [
       {"strict-transport-security", "max-age=#{max_age_sts}; includeSubDomains"},
diff --git a/test/pleroma/web/plugs/http_security_plug_test.exs b/test/pleroma/web/plugs/http_security_plug_test.exs
index c79170382..80ad1fa7d 100644
--- a/test/pleroma/web/plugs/http_security_plug_test.exs
+++ b/test/pleroma/web/plugs/http_security_plug_test.exs
@@ -3,14 +3,52 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.Plugs.HTTPSecurityPlugTest do
-  use Pleroma.Web.ConnCase
+  use Pleroma.Web.ConnCase, async: true
 
   alias Plug.Conn
 
-  describe "http security enabled" do
-    setup do: clear_config([:http_security, :enabled], true)
+  import Mox
 
-    test "it sends CSP headers when enabled", %{conn: conn} do
+  setup do
+    base_config = Pleroma.Config.get([:http_security])
+    %{base_config: base_config}
+  end
+
+  defp mock_config(config, additional \\ %{}) do
+    Pleroma.UnstubbedConfigMock
+    |> stub(:get, fn
+      [:http_security, key] -> config[key]
+      key -> additional[key]
+    end)
+  end
+
+  describe "http security enabled" do
+    setup %{base_config: base_config} do
+      %{base_config: Keyword.put(base_config, :enabled, true)}
+    end
+
+    test "it does not contain unsafe-eval", %{conn: conn, base_config: base_config} do
+      mock_config(base_config)
+
+      conn = get(conn, "/api/v1/instance")
+      [header] = Conn.get_resp_header(conn, "content-security-policy")
+      refute header =~ ~r/unsafe-eval/
+    end
+
+    test "with allow_unsafe_eval set, it does contain it", %{conn: conn, base_config: base_config} do
+      base_config =
+        base_config
+        |> Keyword.put(:allow_unsafe_eval, true)
+
+      mock_config(base_config)
+
+      conn = get(conn, "/api/v1/instance")
+      [header] = Conn.get_resp_header(conn, "content-security-policy")
+      assert header =~ ~r/unsafe-eval/
+    end
+
+    test "it sends CSP headers when enabled", %{conn: conn, base_config: base_config} do
+      mock_config(base_config)
       conn = get(conn, "/api/v1/instance")
 
       refute Conn.get_resp_header(conn, "x-xss-protection") == []
@@ -22,8 +60,10 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlugTest do
       refute Conn.get_resp_header(conn, "content-security-policy") == []
     end
 
-    test "it sends STS headers when enabled", %{conn: conn} do
-      clear_config([:http_security, :sts], true)
+    test "it sends STS headers when enabled", %{conn: conn, base_config: base_config} do
+      base_config
+      |> Keyword.put(:sts, true)
+      |> mock_config()
 
       conn = get(conn, "/api/v1/instance")
 
@@ -31,8 +71,10 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlugTest do
       refute Conn.get_resp_header(conn, "expect-ct") == []
     end
 
-    test "it does not send STS headers when disabled", %{conn: conn} do
-      clear_config([:http_security, :sts], false)
+    test "it does not send STS headers when disabled", %{conn: conn, base_config: base_config} do
+      base_config
+      |> Keyword.put(:sts, false)
+      |> mock_config()
 
       conn = get(conn, "/api/v1/instance")
 
@@ -40,19 +82,30 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlugTest do
       assert Conn.get_resp_header(conn, "expect-ct") == []
     end
 
-    test "referrer-policy header reflects configured value", %{conn: conn} do
-      resp = get(conn, "/api/v1/instance")
+    test "referrer-policy header reflects configured value", %{
+      conn: conn,
+      base_config: base_config
+    } do
+      mock_config(base_config)
 
+      resp = get(conn, "/api/v1/instance")
       assert Conn.get_resp_header(resp, "referrer-policy") == ["same-origin"]
 
-      clear_config([:http_security, :referrer_policy], "no-referrer")
+      base_config
+      |> Keyword.put(:referrer_policy, "no-referrer")
+      |> mock_config
 
       resp = get(conn, "/api/v1/instance")
 
       assert Conn.get_resp_header(resp, "referrer-policy") == ["no-referrer"]
     end
 
-    test "it sends `report-to` & `report-uri` CSP response headers", %{conn: conn} do
+    test "it sends `report-to` & `report-uri` CSP response headers", %{
+      conn: conn,
+      base_config: base_config
+    } do
+      mock_config(base_config)
+
       conn = get(conn, "/api/v1/instance")
 
       [csp] = Conn.get_resp_header(conn, "content-security-policy")
@@ -65,7 +118,11 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlugTest do
                "{\"endpoints\":[{\"url\":\"https://endpoint.com\"}],\"group\":\"csp-endpoint\",\"max-age\":10886400}"
     end
 
-    test "default values for img-src and media-src with disabled media proxy", %{conn: conn} do
+    test "default values for img-src and media-src with disabled media proxy", %{
+      conn: conn,
+      base_config: base_config
+    } do
+      mock_config(base_config)
       conn = get(conn, "/api/v1/instance")
 
       [csp] = Conn.get_resp_header(conn, "content-security-policy")
@@ -73,60 +130,129 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlugTest do
       assert csp =~ "img-src 'self' data: blob: https:;"
     end
 
-    test "it sets the Service-Worker-Allowed header", %{conn: conn} do
-      clear_config([:http_security, :enabled], true)
-      clear_config([:frontends, :primary], %{"name" => "fedi-fe", "ref" => "develop"})
+    test "it sets the Service-Worker-Allowed header", %{conn: conn, base_config: base_config} do
+      base_config
+      |> Keyword.put(:enabled, true)
 
-      clear_config([:frontends, :available], %{
-        "fedi-fe" => %{
-          "name" => "fedi-fe",
-          "custom-http-headers" => [{"service-worker-allowed", "/"}]
-        }
-      })
+      additional_config =
+        %{}
+        |> Map.put([:frontends, :primary], %{"name" => "fedi-fe", "ref" => "develop"})
+        |> Map.put(
+          [:frontends, :available],
+          %{
+            "fedi-fe" => %{
+              "name" => "fedi-fe",
+              "custom-http-headers" => [{"service-worker-allowed", "/"}]
+            }
+          }
+        )
 
+      mock_config(base_config, additional_config)
       conn = get(conn, "/api/v1/instance")
       assert Conn.get_resp_header(conn, "service-worker-allowed") == ["/"]
     end
   end
 
   describe "img-src and media-src" do
-    setup do
-      clear_config([:http_security, :enabled], true)
-      clear_config([:media_proxy, :enabled], true)
-      clear_config([:media_proxy, :proxy_opts, :redirect_on_failure], false)
+    setup %{base_config: base_config} do
+      base_config =
+        base_config
+        |> Keyword.put(:enabled, true)
+
+      additional_config =
+        %{}
+        |> Map.put([:media_proxy, :enabled], true)
+        |> Map.put([:media_proxy, :proxy_opts, :redirect_on_failure], false)
+        |> Map.put([:media_proxy, :whitelist], [])
+
+      %{base_config: base_config, additional_config: additional_config}
     end
 
-    test "media_proxy with base_url", %{conn: conn} do
+    test "media_proxy with base_url", %{
+      conn: conn,
+      base_config: base_config,
+      additional_config: additional_config
+    } do
       url = "https://example.com"
-      clear_config([:media_proxy, :base_url], url)
+
+      additional_config =
+        additional_config
+        |> Map.put([:media_proxy, :base_url], url)
+
+      mock_config(base_config, additional_config)
+
       assert_media_img_src(conn, url)
     end
 
-    test "upload with base url", %{conn: conn} do
+    test "upload with base url", %{
+      conn: conn,
+      base_config: base_config,
+      additional_config: additional_config
+    } do
       url = "https://example2.com"
-      clear_config([Pleroma.Upload, :base_url], url)
+
+      additional_config =
+        additional_config
+        |> Map.put([Pleroma.Upload, :base_url], url)
+
+      mock_config(base_config, additional_config)
+
       assert_media_img_src(conn, url)
     end
 
-    test "with S3 public endpoint", %{conn: conn} do
+    test "with S3 public endpoint", %{
+      conn: conn,
+      base_config: base_config,
+      additional_config: additional_config
+    } do
       url = "https://example3.com"
-      clear_config([Pleroma.Uploaders.S3, :public_endpoint], url)
+
+      additional_config =
+        additional_config
+        |> Map.put([Pleroma.Uploaders.S3, :public_endpoint], url)
+
+      mock_config(base_config, additional_config)
       assert_media_img_src(conn, url)
     end
 
-    test "with captcha endpoint", %{conn: conn} do
-      clear_config([Pleroma.Captcha.Mock, :endpoint], "https://captcha.com")
+    test "with captcha endpoint", %{
+      conn: conn,
+      base_config: base_config,
+      additional_config: additional_config
+    } do
+      additional_config =
+        additional_config
+        |> Map.put([Pleroma.Captcha.Mock, :endpoint], "https://captcha.com")
+        |> Map.put([Pleroma.Captcha, :method], Pleroma.Captcha.Mock)
+
+      mock_config(base_config, additional_config)
       assert_media_img_src(conn, "https://captcha.com")
     end
 
-    test "with media_proxy whitelist", %{conn: conn} do
-      clear_config([:media_proxy, :whitelist], ["https://example6.com", "https://example7.com"])
+    test "with media_proxy whitelist", %{
+      conn: conn,
+      base_config: base_config,
+      additional_config: additional_config
+    } do
+      additional_config =
+        additional_config
+        |> Map.put([:media_proxy, :whitelist], ["https://example6.com", "https://example7.com"])
+
+      mock_config(base_config, additional_config)
       assert_media_img_src(conn, "https://example7.com https://example6.com")
     end
 
     # TODO: delete after removing support bare domains for media proxy whitelist
-    test "with media_proxy bare domains whitelist (deprecated)", %{conn: conn} do
-      clear_config([:media_proxy, :whitelist], ["example4.com", "example5.com"])
+    test "with media_proxy bare domains whitelist (deprecated)", %{
+      conn: conn,
+      base_config: base_config,
+      additional_config: additional_config
+    } do
+      additional_config =
+        additional_config
+        |> Map.put([:media_proxy, :whitelist], ["example4.com", "example5.com"])
+
+      mock_config(base_config, additional_config)
       assert_media_img_src(conn, "example5.com example4.com")
     end
   end
@@ -138,8 +264,10 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlugTest do
     assert csp =~ "img-src 'self' data: blob: #{url};"
   end
 
-  test "it does not send CSP headers when disabled", %{conn: conn} do
-    clear_config([:http_security, :enabled], false)
+  test "it does not send CSP headers when disabled", %{conn: conn, base_config: base_config} do
+    base_config
+    |> Keyword.put(:enabled, false)
+    |> mock_config
 
     conn = get(conn, "/api/v1/instance")
 

From fc7ce339edc40cb791d321a20f01f2568337b845 Mon Sep 17 00:00:00 2001
From: Lain Soykaf <lain@lain.com>
Date: Mon, 27 May 2024 21:28:20 +0400
Subject: [PATCH 128/192] Cheatsheet: Add allow_unsafe_eval

---
 docs/configuration/cheatsheet.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md
index ca2ce6369..78997c4db 100644
--- a/docs/configuration/cheatsheet.md
+++ b/docs/configuration/cheatsheet.md
@@ -472,6 +472,7 @@ This will make Pleroma listen on `127.0.0.1` port `8080` and generate urls start
 * ``ct_max_age``: The maximum age for the `Expect-CT` header if sent.
 * ``referrer_policy``: The referrer policy to use, either `"same-origin"` or `"no-referrer"`.
 * ``report_uri``: Adds the specified url to `report-uri` and `report-to` group in CSP header.
+* `allow_unsafe_eval`: Adds `wasm-unsafe-eval` to the CSP header. Needed for some non-essential frontend features like Flash emulation.
 
 ### Pleroma.Web.Plugs.RemoteIp
 

From c67b41415b369d67c25356205bf69de2d99a291c Mon Sep 17 00:00:00 2001
From: Lain Soykaf <lain@lain.com>
Date: Sun, 11 Jun 2023 20:24:18 +0400
Subject: [PATCH 129/192] Changelog: Add changelog entry.

---
 changelog.d/3904.security | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3904.security

diff --git a/changelog.d/3904.security b/changelog.d/3904.security
new file mode 100644
index 000000000..04836d4e8
--- /dev/null
+++ b/changelog.d/3904.security
@@ -0,0 +1 @@
+HTTP Security: By default, don't allow unsafe-eval. The setting needs to be changed to allow Flash emulation.

From 0847d9ebafa38007aeef0a6677588211994ab546 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Mon, 27 May 2024 14:35:30 +0000
Subject: [PATCH 130/192] Oban queue simplification

---
 changelog.d/oban-queues.change                |  1 +
 config/config.exs                             | 12 +------
 lib/pleroma/scheduled_activity.ex             |  2 +-
 lib/pleroma/web/federator.ex                  |  2 +-
 .../workers/attachments_cleanup_worker.ex     |  2 +-
 lib/pleroma/workers/backup_worker.ex          |  2 +-
 .../workers/cron/new_users_digest_worker.ex   |  2 +-
 lib/pleroma/workers/mailer_worker.ex          |  2 +-
 lib/pleroma/workers/mute_expire_worker.ex     |  2 +-
 lib/pleroma/workers/poll_worker.ex            |  2 +-
 lib/pleroma/workers/purge_expired_activity.ex |  4 +--
 lib/pleroma/workers/purge_expired_filter.ex   |  4 +--
 lib/pleroma/workers/purge_expired_token.ex    |  2 +-
 lib/pleroma/workers/remote_fetcher_worker.ex  |  2 +-
 .../workers/rich_media_expiration_worker.ex   |  2 +-
 .../workers/scheduled_activity_worker.ex      |  2 +-
 .../20240527144418_oban_queues_refactor.exs   | 32 +++++++++++++++++++
 17 files changed, 50 insertions(+), 27 deletions(-)
 create mode 100644 changelog.d/oban-queues.change
 create mode 100644 priv/repo/migrations/20240527144418_oban_queues_refactor.exs

diff --git a/changelog.d/oban-queues.change b/changelog.d/oban-queues.change
new file mode 100644
index 000000000..16df6409a
--- /dev/null
+++ b/changelog.d/oban-queues.change
@@ -0,0 +1 @@
+Oban queues have refactored to simplify the queue design
diff --git a/config/config.exs b/config/config.exs
index b93de52e1..b52021373 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -574,24 +574,14 @@ config :pleroma, Oban,
   log: false,
   queues: [
     activity_expiration: 10,
-    token_expiration: 5,
-    filter_expiration: 1,
-    backup: 1,
     federator_incoming: 5,
     federator_outgoing: 5,
     ingestion_queue: 50,
     web_push: 50,
-    mailer: 10,
     transmogrifier: 20,
-    scheduled_activities: 10,
-    poll_notifications: 10,
     background: 5,
-    remote_fetcher: 2,
-    attachments_cleanup: 1,
-    new_users_digest: 1,
-    mute_expire: 5,
     search_indexing: [limit: 10, paused: true],
-    rich_media_expiration: 2
+    slow: 1
   ],
   plugins: [Oban.Plugins.Pruner],
   crontab: [
diff --git a/lib/pleroma/scheduled_activity.ex b/lib/pleroma/scheduled_activity.ex
index 63c6cb45b..c361d7d89 100644
--- a/lib/pleroma/scheduled_activity.ex
+++ b/lib/pleroma/scheduled_activity.ex
@@ -204,7 +204,7 @@ defmodule Pleroma.ScheduledActivity do
 
   def job_query(scheduled_activity_id) do
     from(j in Oban.Job,
-      where: j.queue == "scheduled_activities",
+      where: j.queue == "federator_outgoing",
       where: fragment("args ->> 'activity_id' = ?::text", ^to_string(scheduled_activity_id))
     )
   end
diff --git a/lib/pleroma/web/federator.ex b/lib/pleroma/web/federator.ex
index 1f2c3835a..4b30fd21d 100644
--- a/lib/pleroma/web/federator.ex
+++ b/lib/pleroma/web/federator.ex
@@ -44,7 +44,7 @@ defmodule Pleroma.Web.Federator do
   end
 
   def incoming_ap_doc(%{"type" => "Delete"} = params) do
-    ReceiverWorker.enqueue("incoming_ap_doc", %{"params" => params}, priority: 3)
+    ReceiverWorker.enqueue("incoming_ap_doc", %{"params" => params}, priority: 3, queue: :slow)
   end
 
   def incoming_ap_doc(params) do
diff --git a/lib/pleroma/workers/attachments_cleanup_worker.ex b/lib/pleroma/workers/attachments_cleanup_worker.ex
index 4c1764053..0b570b70b 100644
--- a/lib/pleroma/workers/attachments_cleanup_worker.ex
+++ b/lib/pleroma/workers/attachments_cleanup_worker.ex
@@ -8,7 +8,7 @@ defmodule Pleroma.Workers.AttachmentsCleanupWorker do
   alias Pleroma.Object
   alias Pleroma.Repo
 
-  use Pleroma.Workers.WorkerHelper, queue: "attachments_cleanup"
+  use Pleroma.Workers.WorkerHelper, queue: "slow"
 
   @impl Oban.Worker
   def perform(%Job{
diff --git a/lib/pleroma/workers/backup_worker.ex b/lib/pleroma/workers/backup_worker.ex
index a485ddb4b..54ac31a3c 100644
--- a/lib/pleroma/workers/backup_worker.ex
+++ b/lib/pleroma/workers/backup_worker.ex
@@ -3,7 +3,7 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Workers.BackupWorker do
-  use Oban.Worker, queue: :backup, max_attempts: 1
+  use Oban.Worker, queue: :slow, max_attempts: 1
 
   alias Oban.Job
   alias Pleroma.User.Backup
diff --git a/lib/pleroma/workers/cron/new_users_digest_worker.ex b/lib/pleroma/workers/cron/new_users_digest_worker.ex
index 1c3e445aa..d2abb2d3b 100644
--- a/lib/pleroma/workers/cron/new_users_digest_worker.ex
+++ b/lib/pleroma/workers/cron/new_users_digest_worker.ex
@@ -9,7 +9,7 @@ defmodule Pleroma.Workers.Cron.NewUsersDigestWorker do
 
   import Ecto.Query
 
-  use Pleroma.Workers.WorkerHelper, queue: "mailer"
+  use Pleroma.Workers.WorkerHelper, queue: "background"
 
   @impl Oban.Worker
   def perform(_job) do
diff --git a/lib/pleroma/workers/mailer_worker.ex b/lib/pleroma/workers/mailer_worker.ex
index 940716558..652bf77e0 100644
--- a/lib/pleroma/workers/mailer_worker.ex
+++ b/lib/pleroma/workers/mailer_worker.ex
@@ -3,7 +3,7 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Workers.MailerWorker do
-  use Pleroma.Workers.WorkerHelper, queue: "mailer"
+  use Pleroma.Workers.WorkerHelper, queue: "background"
 
   @impl Oban.Worker
   def perform(%Job{args: %{"op" => "email", "encoded_email" => encoded_email, "config" => config}}) do
diff --git a/lib/pleroma/workers/mute_expire_worker.ex b/lib/pleroma/workers/mute_expire_worker.ex
index 8ce458d48..8ad287a7f 100644
--- a/lib/pleroma/workers/mute_expire_worker.ex
+++ b/lib/pleroma/workers/mute_expire_worker.ex
@@ -3,7 +3,7 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Workers.MuteExpireWorker do
-  use Pleroma.Workers.WorkerHelper, queue: "mute_expire"
+  use Pleroma.Workers.WorkerHelper, queue: "background"
 
   @impl Oban.Worker
   def perform(%Job{args: %{"op" => "unmute_user", "muter_id" => muter_id, "mutee_id" => mutee_id}}) do
diff --git a/lib/pleroma/workers/poll_worker.ex b/lib/pleroma/workers/poll_worker.ex
index 022d026f8..70df54193 100644
--- a/lib/pleroma/workers/poll_worker.ex
+++ b/lib/pleroma/workers/poll_worker.ex
@@ -6,7 +6,7 @@ defmodule Pleroma.Workers.PollWorker do
   @moduledoc """
   Generates notifications when a poll ends.
   """
-  use Pleroma.Workers.WorkerHelper, queue: "poll_notifications"
+  use Pleroma.Workers.WorkerHelper, queue: "background"
 
   alias Pleroma.Activity
   alias Pleroma.Notification
diff --git a/lib/pleroma/workers/purge_expired_activity.ex b/lib/pleroma/workers/purge_expired_activity.ex
index e554684fe..a65593b6e 100644
--- a/lib/pleroma/workers/purge_expired_activity.ex
+++ b/lib/pleroma/workers/purge_expired_activity.ex
@@ -7,7 +7,7 @@ defmodule Pleroma.Workers.PurgeExpiredActivity do
   Worker which purges expired activity.
   """
 
-  use Oban.Worker, queue: :activity_expiration, max_attempts: 1, unique: [period: :infinity]
+  use Oban.Worker, queue: :slow, max_attempts: 1, unique: [period: :infinity]
 
   import Ecto.Query
 
@@ -59,7 +59,7 @@ defmodule Pleroma.Workers.PurgeExpiredActivity do
   def get_expiration(id) do
     from(j in Oban.Job,
       where: j.state == "scheduled",
-      where: j.queue == "activity_expiration",
+      where: j.queue == "slow",
       where: fragment("?->>'activity_id' = ?", j.args, ^id)
     )
     |> Pleroma.Repo.one()
diff --git a/lib/pleroma/workers/purge_expired_filter.ex b/lib/pleroma/workers/purge_expired_filter.ex
index 9114aeb7f..1f6931e4c 100644
--- a/lib/pleroma/workers/purge_expired_filter.ex
+++ b/lib/pleroma/workers/purge_expired_filter.ex
@@ -7,7 +7,7 @@ defmodule Pleroma.Workers.PurgeExpiredFilter do
   Worker which purges expired filters
   """
 
-  use Oban.Worker, queue: :filter_expiration, max_attempts: 1, unique: [period: :infinity]
+  use Oban.Worker, queue: :background, max_attempts: 1, unique: [period: :infinity]
 
   import Ecto.Query
 
@@ -38,7 +38,7 @@ defmodule Pleroma.Workers.PurgeExpiredFilter do
   def get_expiration(id) do
     from(j in Job,
       where: j.state == "scheduled",
-      where: j.queue == "filter_expiration",
+      where: j.queue == "background",
       where: fragment("?->'filter_id' = ?", j.args, ^id)
     )
     |> Repo.one()
diff --git a/lib/pleroma/workers/purge_expired_token.ex b/lib/pleroma/workers/purge_expired_token.ex
index 2ccd9e80b..1854bf561 100644
--- a/lib/pleroma/workers/purge_expired_token.ex
+++ b/lib/pleroma/workers/purge_expired_token.ex
@@ -7,7 +7,7 @@ defmodule Pleroma.Workers.PurgeExpiredToken do
   Worker which purges expired OAuth tokens
   """
 
-  use Oban.Worker, queue: :token_expiration, max_attempts: 1
+  use Oban.Worker, queue: :background, max_attempts: 1
 
   @spec enqueue(%{token_id: integer(), valid_until: DateTime.t(), mod: module()}) ::
           {:ok, Oban.Job.t()} | {:error, Ecto.Changeset.t()}
diff --git a/lib/pleroma/workers/remote_fetcher_worker.ex b/lib/pleroma/workers/remote_fetcher_worker.ex
index c26418483..ed04c54b2 100644
--- a/lib/pleroma/workers/remote_fetcher_worker.ex
+++ b/lib/pleroma/workers/remote_fetcher_worker.ex
@@ -5,7 +5,7 @@
 defmodule Pleroma.Workers.RemoteFetcherWorker do
   alias Pleroma.Object.Fetcher
 
-  use Pleroma.Workers.WorkerHelper, queue: "remote_fetcher"
+  use Pleroma.Workers.WorkerHelper, queue: "background"
 
   @impl Oban.Worker
   def perform(%Job{args: %{"op" => "fetch_remote", "id" => id} = args}) do
diff --git a/lib/pleroma/workers/rich_media_expiration_worker.ex b/lib/pleroma/workers/rich_media_expiration_worker.ex
index d7ae497a7..0b74687cf 100644
--- a/lib/pleroma/workers/rich_media_expiration_worker.ex
+++ b/lib/pleroma/workers/rich_media_expiration_worker.ex
@@ -6,7 +6,7 @@ defmodule Pleroma.Workers.RichMediaExpirationWorker do
   alias Pleroma.Web.RichMedia.Card
 
   use Oban.Worker,
-    queue: :rich_media_expiration
+    queue: :background
 
   @impl Oban.Worker
   def perform(%Job{args: %{"url" => url} = _args}) do
diff --git a/lib/pleroma/workers/scheduled_activity_worker.ex b/lib/pleroma/workers/scheduled_activity_worker.ex
index 4df84d00f..ab62686f4 100644
--- a/lib/pleroma/workers/scheduled_activity_worker.ex
+++ b/lib/pleroma/workers/scheduled_activity_worker.ex
@@ -7,7 +7,7 @@ defmodule Pleroma.Workers.ScheduledActivityWorker do
   The worker to post scheduled activity.
   """
 
-  use Pleroma.Workers.WorkerHelper, queue: "scheduled_activities"
+  use Pleroma.Workers.WorkerHelper, queue: "federator_outgoing"
 
   alias Pleroma.Repo
   alias Pleroma.ScheduledActivity
diff --git a/priv/repo/migrations/20240527144418_oban_queues_refactor.exs b/priv/repo/migrations/20240527144418_oban_queues_refactor.exs
new file mode 100644
index 000000000..64ee28dfd
--- /dev/null
+++ b/priv/repo/migrations/20240527144418_oban_queues_refactor.exs
@@ -0,0 +1,32 @@
+defmodule Pleroma.Repo.Migrations.ObanQueuesRefactor do
+  use Ecto.Migration
+
+  @changed_queues [
+    {"attachments_cleanup", "slow"},
+    {"mailer", "background"},
+    {"mute_expire", "background"},
+    {"poll_notifications", "background"},
+    {"activity_expiration", "slow"},
+    {"filter_expiration", "background"},
+    {"token_expiration", "background"},
+    {"remote_fetcher", "background"},
+    {"rich_media_expiration", "background"}
+  ]
+
+  def up do
+    Enum.each(@changed_queues, fn {old, new} ->
+      execute("UPDATE oban_jobs SET queue = '#{new}' WHERE queue = '#{old}';")
+    end)
+
+    # Handled special as reverting this would not be ideal and leaving it is harmless
+    execute(
+      "UPDATE oban_jobs SET queue = 'federator_outgoing' WHERE queue = 'scheduled_activities';"
+    )
+  end
+
+  def down do
+    # Just move all slow queue jobs to background queue if we are reverting
+    # as the slow queue will not be processing jobs
+    execute("UPDATE oban_jobs SET queue = 'background' WHERE queue = 'slow';")
+  end
+end

From f63e44b8bc8e4e2f21fe21f1407a85d072dcab6d Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Mon, 27 May 2024 13:46:15 -0400
Subject: [PATCH 131/192] Fix Oban related tests

---
 test/pleroma/scheduled_activity_test.exs        |  3 +--
 .../scheduled_activity_controller_test.exs      | 17 ++++++++++++-----
 2 files changed, 13 insertions(+), 7 deletions(-)

diff --git a/test/pleroma/scheduled_activity_test.exs b/test/pleroma/scheduled_activity_test.exs
index 4818e8bcf..aaf643cfc 100644
--- a/test/pleroma/scheduled_activity_test.exs
+++ b/test/pleroma/scheduled_activity_test.exs
@@ -31,8 +31,7 @@ defmodule Pleroma.ScheduledActivityTest do
       {:ok, sa1} = ScheduledActivity.create(user, attrs)
       {:ok, sa2} = ScheduledActivity.create(user, attrs)
 
-      jobs =
-        Repo.all(from(j in Oban.Job, where: j.queue == "scheduled_activities", select: j.args))
+      jobs = Repo.all(from(j in Oban.Job, where: j.queue == "federator_outgoing", select: j.args))
 
       assert jobs == [%{"activity_id" => sa1.id}, %{"activity_id" => sa2.id}]
     end
diff --git a/test/pleroma/web/mastodon_api/controllers/scheduled_activity_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/scheduled_activity_controller_test.exs
index 632242221..2d6b2aee2 100644
--- a/test/pleroma/web/mastodon_api/controllers/scheduled_activity_controller_test.exs
+++ b/test/pleroma/web/mastodon_api/controllers/scheduled_activity_controller_test.exs
@@ -3,6 +3,7 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.MastodonAPI.ScheduledActivityControllerTest do
+  use Oban.Testing, repo: Pleroma.Repo
   use Pleroma.Web.ConnCase, async: true
 
   alias Pleroma.Repo
@@ -78,7 +79,7 @@ defmodule Pleroma.Web.MastodonAPI.ScheduledActivityControllerTest do
         }
       )
 
-    job = Repo.one(from(j in Oban.Job, where: j.queue == "scheduled_activities"))
+    job = Repo.one(from(j in Oban.Job, where: j.queue == "federator_outgoing"))
 
     assert job.args == %{"activity_id" => scheduled_activity.id}
     assert DateTime.truncate(job.scheduled_at, :second) == to_datetime(scheduled_at)
@@ -124,9 +125,11 @@ defmodule Pleroma.Web.MastodonAPI.ScheduledActivityControllerTest do
         }
       )
 
-    job = Repo.one(from(j in Oban.Job, where: j.queue == "scheduled_activities"))
-
-    assert job.args == %{"activity_id" => scheduled_activity.id}
+    assert_enqueued(
+      worker: Pleroma.Workers.ScheduledActivityWorker,
+      args: %{"activity_id" => scheduled_activity.id},
+      queue: :federator_outgoing
+    )
 
     res_conn =
       conn
@@ -135,7 +138,11 @@ defmodule Pleroma.Web.MastodonAPI.ScheduledActivityControllerTest do
 
     assert %{} = json_response_and_validate_schema(res_conn, 200)
     refute Repo.get(ScheduledActivity, scheduled_activity.id)
-    refute Repo.get(Oban.Job, job.id)
+
+    refute_enqueued(
+      worker: Pleroma.Workers.ScheduledActivityWorker,
+      args: %{"activity_id" => scheduled_activity.id}
+    )
 
     res_conn =
       conn

From 29eac86dc0bb246e983afe4209332194bf11bed0 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Mon, 27 May 2024 13:53:22 -0400
Subject: [PATCH 132/192] Logger metadata changelog

---
 changelog.d/logger-metadata.add | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/logger-metadata.add

diff --git a/changelog.d/logger-metadata.add b/changelog.d/logger-metadata.add
new file mode 100644
index 000000000..6c627a972
--- /dev/null
+++ b/changelog.d/logger-metadata.add
@@ -0,0 +1 @@
+Logger metadata is now attached to some logs to help with troubleshooting and analysis

From 6b8c15a4a1fdbc2a2a4ef194d9519e717470c632 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Mon, 27 May 2024 14:11:42 -0400
Subject: [PATCH 133/192] Remove MediaProxyWarmingPolicy config for
 ConcurrentLimiter as we are not using it

---
 config/config.exs                                           | 1 -
 .../web/activity_pub/mrf/media_proxy_warming_policy.ex      | 6 +-----
 2 files changed, 1 insertion(+), 6 deletions(-)

diff --git a/config/config.exs b/config/config.exs
index a025defeb..3c35f439a 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -902,7 +902,6 @@ config :pleroma, Pleroma.User.Backup,
 
 config :pleroma, ConcurrentLimiter, [
   {Pleroma.Web.RichMedia.Helpers, [max_running: 5, max_waiting: 5]},
-  {Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy, [max_running: 5, max_waiting: 5]},
   {Pleroma.Search, [max_running: 30, max_waiting: 50]}
 ]
 
diff --git a/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex b/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex
index e5eb6896a..0c5b53def 100644
--- a/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex
@@ -22,11 +22,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do
 
       Logger.debug("Prefetching #{inspect(url)} as #{inspect(prefetch_url)}")
 
-      if Pleroma.Config.get(:env) == :test do
-        fetch(prefetch_url)
-      else
-        Task.start(fn -> fetch(prefetch_url) end)
-      end
+      fetch(prefetch_url)
     end
   end
 

From ba511a30b923cc9248686702f75a4041d69c7bee Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Mon, 27 May 2024 14:12:38 -0400
Subject: [PATCH 134/192] RichMedia use of ConcurrentLimiter was removed in the
 refactor

---
 config/config.exs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/config/config.exs b/config/config.exs
index 3c35f439a..e7e751d8a 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -901,7 +901,6 @@ config :pleroma, Pleroma.User.Backup,
   process_chunk_size: 100
 
 config :pleroma, ConcurrentLimiter, [
-  {Pleroma.Web.RichMedia.Helpers, [max_running: 5, max_waiting: 5]},
   {Pleroma.Search, [max_running: 30, max_waiting: 50]}
 ]
 

From 81e44ced0c7251b5a6b585f297e1e00fad08c6d1 Mon Sep 17 00:00:00 2001
From: Lain Soykaf <lain@lain.com>
Date: Mon, 27 May 2024 22:13:20 +0400
Subject: [PATCH 135/192] HTTPSecurityPlug: Fix tests

---
 config/test.exs                                    | 2 +-
 lib/pleroma/web/plugs/http_security_plug.ex        | 4 ++--
 test/pleroma/web/plugs/http_security_plug_test.exs | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/config/test.exs b/config/test.exs
index b5c9c6e4a..6c88ad3c6 100644
--- a/config/test.exs
+++ b/config/test.exs
@@ -154,7 +154,7 @@ config :pleroma, Pleroma.Upload, config_impl: Pleroma.UnstubbedConfigMock
 config :pleroma, Pleroma.ScheduledActivity, config_impl: Pleroma.UnstubbedConfigMock
 config :pleroma, Pleroma.Web.RichMedia.Helpers, config_impl: Pleroma.StaticStubbedConfigMock
 config :pleroma, Pleroma.Uploaders.IPFS, config_impl: Pleroma.UnstubbedConfigMock
-config :pleroma, Pleroma.Web.Plugs.HTTPSecurityPlug, config_impl: Pleroma.UnstubbedConfigMock
+config :pleroma, Pleroma.Web.Plugs.HTTPSecurityPlug, config_impl: Pleroma.StaticStubbedConfigMock
 
 peer_module =
   if String.to_integer(System.otp_release()) >= 25 do
diff --git a/lib/pleroma/web/plugs/http_security_plug.ex b/lib/pleroma/web/plugs/http_security_plug.ex
index a1dc6c02a..38f6c511e 100644
--- a/lib/pleroma/web/plugs/http_security_plug.ex
+++ b/lib/pleroma/web/plugs/http_security_plug.ex
@@ -116,7 +116,7 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do
       end
 
     connect_src =
-      if @config_impl.get(:env) == :dev do
+      if @config_impl.get([:env]) == :dev do
         [connect_src, " http://localhost:3035/"]
       else
         connect_src
@@ -124,7 +124,7 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do
 
     script_src =
       if @config_impl.get([:http_security, :allow_unsafe_eval]) do
-        if @config_impl.get(:env) == :dev do
+        if @config_impl.get([:env]) == :dev do
           "script-src 'self' 'unsafe-eval'"
         else
           "script-src 'self' 'wasm-unsafe-eval'"
diff --git a/test/pleroma/web/plugs/http_security_plug_test.exs b/test/pleroma/web/plugs/http_security_plug_test.exs
index 80ad1fa7d..11a351a41 100644
--- a/test/pleroma/web/plugs/http_security_plug_test.exs
+++ b/test/pleroma/web/plugs/http_security_plug_test.exs
@@ -15,7 +15,7 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlugTest do
   end
 
   defp mock_config(config, additional \\ %{}) do
-    Pleroma.UnstubbedConfigMock
+    Pleroma.StaticStubbedConfigMock
     |> stub(:get, fn
       [:http_security, key] -> config[key]
       key -> additional[key]

From bb86a01b9b5889155f60f08143755dd101d925f0 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Mon, 27 May 2024 15:20:47 -0400
Subject: [PATCH 136/192] Credo

---
 lib/pleroma/web/activity_pub/activity_pub_controller.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex
index d2b2cae0b..e6161455d 100644
--- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex
@@ -522,7 +522,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
     conn
   end
 
-  defp log_inbox_metadata(conn = %{params: %{"actor" => actor, "type" => type}}, _) do
+  defp log_inbox_metadata(%{params: %{"actor" => actor, "type" => type}} = conn, _) do
     Logger.metadata(actor: actor, type: type)
     conn
   end

From 73d58c22d4a9a87539cf1c3a33083464fc4b8540 Mon Sep 17 00:00:00 2001
From: Lain Soykaf <lain@lain.com>
Date: Tue, 28 May 2024 08:09:19 +0400
Subject: [PATCH 137/192] Linting

---
 lib/pleroma/web/activity_pub/activity_pub_controller.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex
index d2b2cae0b..e6161455d 100644
--- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex
@@ -522,7 +522,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
     conn
   end
 
-  defp log_inbox_metadata(conn = %{params: %{"actor" => actor, "type" => type}}, _) do
+  defp log_inbox_metadata(%{params: %{"actor" => actor, "type" => type}} = conn, _) do
     Logger.metadata(actor: actor, type: type)
     conn
   end

From f5978da67633110acbc6494ff6f07b3f07424779 Mon Sep 17 00:00:00 2001
From: Lain Soykaf <lain@lain.com>
Date: Tue, 28 May 2024 14:00:25 +0400
Subject: [PATCH 138/192] HTTPSignaturePlugTest: Rewrite to use mox.

---
 config/test.exs                               |   4 +
 lib/pleroma/http_signatures_api.ex            |   4 +
 lib/pleroma/web/plugs/http_signature_plug.ex  |  19 +-
 .../web/plugs/http_signature_plug_test.exs    | 219 +++++++++---------
 test/support/data_case.ex                     |   1 +
 test/support/http_signatures_proxy.ex         |   9 +
 test/support/mocks.ex                         |   1 +
 7 files changed, 144 insertions(+), 113 deletions(-)
 create mode 100644 lib/pleroma/http_signatures_api.ex
 create mode 100644 test/support/http_signatures_proxy.ex

diff --git a/config/test.exs b/config/test.exs
index 6c88ad3c6..0d4c82e0e 100644
--- a/config/test.exs
+++ b/config/test.exs
@@ -155,6 +155,10 @@ config :pleroma, Pleroma.ScheduledActivity, config_impl: Pleroma.UnstubbedConfig
 config :pleroma, Pleroma.Web.RichMedia.Helpers, config_impl: Pleroma.StaticStubbedConfigMock
 config :pleroma, Pleroma.Uploaders.IPFS, config_impl: Pleroma.UnstubbedConfigMock
 config :pleroma, Pleroma.Web.Plugs.HTTPSecurityPlug, config_impl: Pleroma.StaticStubbedConfigMock
+config :pleroma, Pleroma.Web.Plugs.HTTPSignaturePlug, config_impl: Pleroma.StaticStubbedConfigMock
+
+config :pleroma, Pleroma.Web.Plugs.HTTPSignaturePlug,
+  http_signatures_impl: Pleroma.StubbedHTTPSignaturesMock
 
 peer_module =
   if String.to_integer(System.otp_release()) >= 25 do
diff --git a/lib/pleroma/http_signatures_api.ex b/lib/pleroma/http_signatures_api.ex
new file mode 100644
index 000000000..8e73dc98e
--- /dev/null
+++ b/lib/pleroma/http_signatures_api.ex
@@ -0,0 +1,4 @@
+defmodule Pleroma.HTTPSignaturesAPI do
+  @callback validate_conn(conn :: Plug.Conn.t()) :: boolean
+  @callback signature_for_conn(conn :: Plug.Conn.t()) :: map
+end
diff --git a/lib/pleroma/web/plugs/http_signature_plug.ex b/lib/pleroma/web/plugs/http_signature_plug.ex
index 71b2a5f51..6bf2dd432 100644
--- a/lib/pleroma/web/plugs/http_signature_plug.ex
+++ b/lib/pleroma/web/plugs/http_signature_plug.ex
@@ -8,11 +8,17 @@ defmodule Pleroma.Web.Plugs.HTTPSignaturePlug do
   import Plug.Conn
   import Phoenix.Controller, only: [get_format: 1, text: 2]
 
-  alias Pleroma.Config
   alias Pleroma.Web.ActivityPub.MRF
 
   require Logger
 
+  @config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config)
+  @http_signatures_impl Application.compile_env(
+                          :pleroma,
+                          [__MODULE__, :http_signatures_impl],
+                          HTTPSignatures
+                        )
+
   def init(options) do
     options
   end
@@ -41,7 +47,7 @@ defmodule Pleroma.Web.Plugs.HTTPSignaturePlug do
       |> put_req_header("(request-target)", request_target)
       |> put_req_header("@request-target", request_target)
 
-    HTTPSignatures.validate_conn(conn)
+    @http_signatures_impl.validate_conn(conn)
   end
 
   defp validate_signature(conn) do
@@ -108,9 +114,9 @@ defmodule Pleroma.Web.Plugs.HTTPSignaturePlug do
   defp maybe_require_signature(%{assigns: %{valid_signature: true}} = conn), do: conn
 
   defp maybe_require_signature(%{remote_ip: remote_ip} = conn) do
-    if Pleroma.Config.get([:activitypub, :authorized_fetch_mode], false) do
+    if @config_impl.get([:activitypub, :authorized_fetch_mode], false) do
       exceptions =
-        Pleroma.Config.get([:activitypub, :authorized_fetch_mode_exceptions], [])
+        @config_impl.get([:activitypub, :authorized_fetch_mode_exceptions], [])
         |> Enum.map(&InetHelper.parse_cidr/1)
 
       if Enum.any?(exceptions, fn x -> InetCidr.contains?(x, remote_ip) end) do
@@ -129,7 +135,8 @@ defmodule Pleroma.Web.Plugs.HTTPSignaturePlug do
   defp maybe_filter_requests(%{halted: true} = conn), do: conn
 
   defp maybe_filter_requests(conn) do
-    if Pleroma.Config.get([:activitypub, :authorized_fetch_mode], false) do
+    if @config_impl.get([:activitypub, :authorized_fetch_mode], false) and
+         conn.assigns[:actor_id] do
       %{host: host} = URI.parse(conn.assigns.actor_id)
 
       if MRF.subdomain_match?(rejected_domains(), host) do
@@ -145,7 +152,7 @@ defmodule Pleroma.Web.Plugs.HTTPSignaturePlug do
   end
 
   defp rejected_domains do
-    Config.get([:instance, :rejected_instances])
+    @config_impl.get([:instance, :rejected_instances])
     |> Pleroma.Web.ActivityPub.MRF.instance_list_from_tuples()
     |> Pleroma.Web.ActivityPub.MRF.subdomains_regex()
   end
diff --git a/test/pleroma/web/plugs/http_signature_plug_test.exs b/test/pleroma/web/plugs/http_signature_plug_test.exs
index b871d956e..5f049dc45 100644
--- a/test/pleroma/web/plugs/http_signature_plug_test.exs
+++ b/test/pleroma/web/plugs/http_signature_plug_test.exs
@@ -3,89 +3,88 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.Plugs.HTTPSignaturePlugTest do
-  use Pleroma.Web.ConnCase
+  use Pleroma.Web.ConnCase, async: true
   alias Pleroma.Web.Plugs.HTTPSignaturePlug
+  alias Pleroma.StubbedHTTPSignaturesMock, as: HTTPSignaturesMock
+  alias Pleroma.StaticStubbedConfigMock, as: ConfigMock
 
   import Plug.Conn
   import Phoenix.Controller, only: [put_format: 2]
-  import Mock
+  import Mox
 
-  test "it call HTTPSignatures to check validity if the actor signed it" do
+  test "it calls HTTPSignatures to check validity if the actor signed it" do
     params = %{"actor" => "http://mastodon.example.org/users/admin"}
     conn = build_conn(:get, "/doesntmattter", params)
 
-    with_mock HTTPSignatures,
-      validate_conn: fn _ -> true end,
-      signature_for_conn: fn _ ->
-        %{"keyId" => "http://mastodon.example.org/users/admin#main-key"}
-      end do
-      conn =
-        conn
-        |> put_req_header(
-          "signature",
-          "keyId=\"http://mastodon.example.org/users/admin#main-key"
-        )
-        |> put_format("activity+json")
-        |> HTTPSignaturePlug.call(%{})
+    HTTPSignaturesMock
+    |> expect(:validate_conn, fn _ -> true end)
 
-      assert conn.assigns.valid_signature == true
-      assert conn.halted == false
-      assert called(HTTPSignatures.validate_conn(:_))
-    end
+    conn =
+      conn
+      |> put_req_header(
+        "signature",
+        "keyId=\"http://mastodon.example.org/users/admin#main-key"
+      )
+      |> put_format("activity+json")
+      |> HTTPSignaturePlug.call(%{})
+
+    assert conn.assigns.valid_signature == true
+    assert conn.halted == false
   end
 
   describe "requires a signature when `authorized_fetch_mode` is enabled" do
     setup do
-      clear_config([:activitypub, :authorized_fetch_mode], true)
-
       params = %{"actor" => "http://mastodon.example.org/users/admin"}
       conn = build_conn(:get, "/doesntmattter", params) |> put_format("activity+json")
 
       [conn: conn]
     end
 
-    test "when signature header is present", %{conn: conn} do
-      with_mock HTTPSignatures,
-        validate_conn: fn _ -> false end,
-        signature_for_conn: fn _ ->
-          %{"keyId" => "http://mastodon.example.org/users/admin#main-key"}
-        end do
-        conn =
-          conn
-          |> put_req_header(
-            "signature",
-            "keyId=\"http://mastodon.example.org/users/admin#main-key"
-          )
-          |> HTTPSignaturePlug.call(%{})
+    test "when signature header is present", %{conn: orig_conn} do
+      ConfigMock
+      |> expect(:get, fn [:activitypub, :authorized_fetch_mode], false -> true end)
+      |> expect(:get, fn [:activitypub, :authorized_fetch_mode_exceptions], [] -> [] end)
 
-        assert conn.assigns.valid_signature == false
-        assert conn.halted == true
-        assert conn.status == 401
-        assert conn.state == :sent
-        assert conn.resp_body == "Request not signed"
-        assert called(HTTPSignatures.validate_conn(:_))
-      end
+      HTTPSignaturesMock
+      |> expect(:validate_conn, 2, fn _ -> false end)
 
-      with_mock HTTPSignatures,
-        validate_conn: fn _ -> true end,
-        signature_for_conn: fn _ ->
-          %{"keyId" => "http://mastodon.example.org/users/admin#main-key"}
-        end do
-        conn =
-          conn
-          |> put_req_header(
-            "signature",
-            "keyId=\"http://mastodon.example.org/users/admin#main-key"
-          )
-          |> HTTPSignaturePlug.call(%{})
+      conn =
+        orig_conn
+        |> put_req_header(
+          "signature",
+          "keyId=\"http://mastodon.example.org/users/admin#main-key"
+        )
+        |> HTTPSignaturePlug.call(%{})
 
-        assert conn.assigns.valid_signature == true
-        assert conn.halted == false
-        assert called(HTTPSignatures.validate_conn(:_))
-      end
+      assert conn.assigns.valid_signature == false
+      assert conn.halted == true
+      assert conn.status == 401
+      assert conn.state == :sent
+      assert conn.resp_body == "Request not signed"
+
+      ConfigMock
+      |> expect(:get, fn [:activitypub, :authorized_fetch_mode], false -> true end)
+
+      HTTPSignaturesMock
+      |> expect(:validate_conn, fn _ -> true end)
+
+      conn =
+        orig_conn
+        |> put_req_header(
+          "signature",
+          "keyId=\"http://mastodon.example.org/users/admin#main-key"
+        )
+        |> HTTPSignaturePlug.call(%{})
+
+      assert conn.assigns.valid_signature == true
+      assert conn.halted == false
     end
 
     test "halts the connection when `signature` header is not present", %{conn: conn} do
+      ConfigMock
+      |> expect(:get, fn [:activitypub, :authorized_fetch_mode], false -> true end)
+      |> expect(:get, fn [:activitypub, :authorized_fetch_mode_exceptions], [] -> [] end)
+
       conn = HTTPSignaturePlug.call(conn, %{})
       assert conn.assigns[:valid_signature] == nil
       assert conn.halted == true
@@ -95,65 +94,71 @@ defmodule Pleroma.Web.Plugs.HTTPSignaturePlugTest do
     end
 
     test "exempts specific IPs from `authorized_fetch_mode_exceptions`", %{conn: conn} do
-      clear_config([:activitypub, :authorized_fetch_mode_exceptions], ["192.168.0.0/24"])
+      ConfigMock
+      |> expect(:get, fn [:activitypub, :authorized_fetch_mode], false -> true end)
+      |> expect(:get, fn [:activitypub, :authorized_fetch_mode_exceptions], [] ->
+        ["192.168.0.0/24"]
+      end)
+      |> expect(:get, fn [:activitypub, :authorized_fetch_mode], false -> true end)
 
-      with_mock HTTPSignatures, validate_conn: fn _ -> false end do
-        conn =
-          conn
-          |> Map.put(:remote_ip, {192, 168, 0, 1})
-          |> put_req_header(
-            "signature",
-            "keyId=\"http://mastodon.example.org/users/admin#main-key"
-          )
-          |> HTTPSignaturePlug.call(%{})
+      HTTPSignaturesMock
+      |> expect(:validate_conn, 2, fn _ -> false end)
 
-        assert conn.remote_ip == {192, 168, 0, 1}
-        assert conn.halted == false
-        assert called(HTTPSignatures.validate_conn(:_))
-      end
-    end
-  end
-
-  test "rejects requests from `rejected_instances` when `authorized_fetch_mode` is enabled" do
-    clear_config([:activitypub, :authorized_fetch_mode], true)
-    clear_config([:instance, :rejected_instances], [{"mastodon.example.org", "no reason"}])
-
-    with_mock HTTPSignatures,
-      validate_conn: fn _ -> true end,
-      signature_for_conn: fn _ ->
-        %{"keyId" => "http://mastodon.example.org/users/admin#main-key"}
-      end do
       conn =
-        build_conn(:get, "/doesntmattter", %{"actor" => "http://mastodon.example.org/users/admin"})
+        conn
+        |> Map.put(:remote_ip, {192, 168, 0, 1})
         |> put_req_header(
           "signature",
           "keyId=\"http://mastodon.example.org/users/admin#main-key"
         )
-        |> put_format("activity+json")
         |> HTTPSignaturePlug.call(%{})
 
-      assert conn.assigns.valid_signature == true
-      assert conn.halted == true
-      assert called(HTTPSignatures.validate_conn(:_))
-    end
-
-    with_mock HTTPSignatures,
-      validate_conn: fn _ -> true end,
-      signature_for_conn: fn _ ->
-        %{"keyId" => "http://allowed.example.org/users/admin#main-key"}
-      end do
-      conn =
-        build_conn(:get, "/doesntmattter", %{"actor" => "http://allowed.example.org/users/admin"})
-        |> put_req_header(
-          "signature",
-          "keyId=\"http://allowed.example.org/users/admin#main-key"
-        )
-        |> put_format("activity+json")
-        |> HTTPSignaturePlug.call(%{})
-
-      assert conn.assigns.valid_signature == true
+      assert conn.remote_ip == {192, 168, 0, 1}
       assert conn.halted == false
-      assert called(HTTPSignatures.validate_conn(:_))
     end
   end
+
+  test "rejects requests from `rejected_instances` when `authorized_fetch_mode` is enabled" do
+    ConfigMock
+    |> expect(:get, fn [:activitypub, :authorized_fetch_mode], false -> true end)
+    |> expect(:get, fn [:instance, :rejected_instances] ->
+      [{"mastodon.example.org", "no reason"}]
+    end)
+
+    HTTPSignaturesMock
+    |> expect(:validate_conn, fn _ -> true end)
+
+    conn =
+      build_conn(:get, "/doesntmattter", %{"actor" => "http://mastodon.example.org/users/admin"})
+      |> put_req_header(
+        "signature",
+        "keyId=\"http://mastodon.example.org/users/admin#main-key"
+      )
+      |> put_format("activity+json")
+      |> HTTPSignaturePlug.call(%{})
+
+    assert conn.assigns.valid_signature == true
+    assert conn.halted == true
+
+    ConfigMock
+    |> expect(:get, fn [:activitypub, :authorized_fetch_mode], false -> true end)
+    |> expect(:get, fn [:instance, :rejected_instances] ->
+      [{"mastodon.example.org", "no reason"}]
+    end)
+
+    HTTPSignaturesMock
+    |> expect(:validate_conn, fn _ -> true end)
+
+    conn =
+      build_conn(:get, "/doesntmattter", %{"actor" => "http://allowed.example.org/users/admin"})
+      |> put_req_header(
+        "signature",
+        "keyId=\"http://allowed.example.org/users/admin#main-key"
+      )
+      |> put_format("activity+json")
+      |> HTTPSignaturePlug.call(%{})
+
+    assert conn.assigns.valid_signature == true
+    assert conn.halted == false
+  end
 end
diff --git a/test/support/data_case.ex b/test/support/data_case.ex
index 14403f0b8..52d4bef1a 100644
--- a/test/support/data_case.ex
+++ b/test/support/data_case.ex
@@ -116,6 +116,7 @@ defmodule Pleroma.DataCase do
     Mox.stub_with(Pleroma.Web.FederatorMock, Pleroma.Web.Federator)
     Mox.stub_with(Pleroma.ConfigMock, Pleroma.Config)
     Mox.stub_with(Pleroma.StaticStubbedConfigMock, Pleroma.Test.StaticConfig)
+    Mox.stub_with(Pleroma.StubbedHTTPSignaturesMock, Pleroma.Test.HTTPSignaturesProxy)
   end
 
   def ensure_local_uploader(context) do
diff --git a/test/support/http_signatures_proxy.ex b/test/support/http_signatures_proxy.ex
new file mode 100644
index 000000000..4c6b39d19
--- /dev/null
+++ b/test/support/http_signatures_proxy.ex
@@ -0,0 +1,9 @@
+defmodule Pleroma.Test.HTTPSignaturesProxy do
+  @behaviour Pleroma.HTTPSignaturesAPI
+
+  @impl true
+  defdelegate validate_conn(conn), to: HTTPSignatures
+
+  @impl true
+  defdelegate signature_for_conn(conn), to: HTTPSignatures
+end
diff --git a/test/support/mocks.ex b/test/support/mocks.ex
index d906f0e1d..63cbc49ab 100644
--- a/test/support/mocks.ex
+++ b/test/support/mocks.ex
@@ -28,6 +28,7 @@ Mox.defmock(Pleroma.Web.FederatorMock, for: Pleroma.Web.Federator.Publishing)
 Mox.defmock(Pleroma.ConfigMock, for: Pleroma.Config.Getting)
 Mox.defmock(Pleroma.UnstubbedConfigMock, for: Pleroma.Config.Getting)
 Mox.defmock(Pleroma.StaticStubbedConfigMock, for: Pleroma.Config.Getting)
+Mox.defmock(Pleroma.StubbedHTTPSignaturesMock, for: Pleroma.HTTPSignaturesAPI)
 
 Mox.defmock(Pleroma.LoggerMock, for: Pleroma.Logging)
 

From 8066645f711f38986b3d0f9c0b34d6956563da6a Mon Sep 17 00:00:00 2001
From: Lain Soykaf <lain@lain.com>
Date: Tue, 28 May 2024 14:20:48 +0400
Subject: [PATCH 139/192] Linting

---
 test/pleroma/web/plugs/http_signature_plug_test.exs | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/test/pleroma/web/plugs/http_signature_plug_test.exs b/test/pleroma/web/plugs/http_signature_plug_test.exs
index 5f049dc45..9d07270bb 100644
--- a/test/pleroma/web/plugs/http_signature_plug_test.exs
+++ b/test/pleroma/web/plugs/http_signature_plug_test.exs
@@ -4,13 +4,14 @@
 
 defmodule Pleroma.Web.Plugs.HTTPSignaturePlugTest do
   use Pleroma.Web.ConnCase, async: true
-  alias Pleroma.Web.Plugs.HTTPSignaturePlug
-  alias Pleroma.StubbedHTTPSignaturesMock, as: HTTPSignaturesMock
-  alias Pleroma.StaticStubbedConfigMock, as: ConfigMock
 
-  import Plug.Conn
-  import Phoenix.Controller, only: [put_format: 2]
+  alias Pleroma.StaticStubbedConfigMock, as: ConfigMock
+  alias Pleroma.StubbedHTTPSignaturesMock, as: HTTPSignaturesMock
+  alias Pleroma.Web.Plugs.HTTPSignaturePlug
+
   import Mox
+  import Phoenix.Controller, only: [put_format: 2]
+  import Plug.Conn
 
   test "it calls HTTPSignatures to check validity if the actor signed it" do
     params = %{"actor" => "http://mastodon.example.org/users/admin"}

From 335691bae1a002c1a3bec956884fe665114285ec Mon Sep 17 00:00:00 2001
From: Lain Soykaf <lain@lain.com>
Date: Tue, 28 May 2024 14:38:44 +0400
Subject: [PATCH 140/192] Add changelog

---
 changelog.d/authorized-fetch-rejections.add | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/authorized-fetch-rejections.add

diff --git a/changelog.d/authorized-fetch-rejections.add b/changelog.d/authorized-fetch-rejections.add
new file mode 100644
index 000000000..66e15a979
--- /dev/null
+++ b/changelog.d/authorized-fetch-rejections.add
@@ -0,0 +1 @@
+Add an option to reject certain domains when authorized fetch is enabled.

From 0b864c3696e47ba1def6047905dad9065b0bee0e Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Tue, 28 May 2024 08:49:34 -0400
Subject: [PATCH 141/192] Dialyzer: fix invalid @spec

lib/pleroma/notification.ex:492:invalid_contract
The @spec for the function does not match the success typing of the function.

Function:
Pleroma.Notification.get_notified_from_activity/2

Success typing:
@spec get_notified_from_activity(_, _) :: [any()]
---
 lib/pleroma/notification.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex
index 4f714b25f..f521a2998 100644
--- a/lib/pleroma/notification.ex
+++ b/lib/pleroma/notification.ex
@@ -489,7 +489,7 @@ defmodule Pleroma.Notification do
 
   NOTE: might be called for FAKE Activities, see ActivityPub.Utils.get_notified_from_object/1
   """
-  @spec get_notified_from_activity(Activity.t(), boolean()) :: {list(User.t()), list(User.t())}
+  @spec get_notified_from_activity(Activity.t(), boolean()) :: list(User.t())
   def get_notified_from_activity(activity, local_only \\ true)
 
   def get_notified_from_activity(%Activity{data: %{"type" => type}} = activity, local_only)

From 42c5f7c74e93b7b489456578f8285d06320c15dc Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Tue, 28 May 2024 08:55:18 -0400
Subject: [PATCH 142/192] Dialyzer: fix invalid @spec

The callback already defines the @spec and these do not match it.

lib/pleroma/upload/filter/exiftool/strip_location.ex:12:callback_spec_type_mismatch
The @spec return type does not match the expected return type
for filter/1 callback in Pleroma.Upload.Filter behaviour.

Actual:
@spec filter(...) :: {:ok, _}

Expected:
@spec filter(...) :: {:error, _} | {:ok, :filtered | :noop} | {:ok, :filtered, struct()}
---
 lib/pleroma/upload/filter/exiftool/strip_location.ex | 2 --
 lib/pleroma/upload/filter/mogrifun.ex                | 1 -
 lib/pleroma/upload/filter/mogrify.ex                 | 1 -
 3 files changed, 4 deletions(-)

diff --git a/lib/pleroma/upload/filter/exiftool/strip_location.ex b/lib/pleroma/upload/filter/exiftool/strip_location.ex
index f2bcc4622..8becee712 100644
--- a/lib/pleroma/upload/filter/exiftool/strip_location.ex
+++ b/lib/pleroma/upload/filter/exiftool/strip_location.ex
@@ -9,8 +9,6 @@ defmodule Pleroma.Upload.Filter.Exiftool.StripLocation do
   """
   @behaviour Pleroma.Upload.Filter
 
-  @spec filter(Pleroma.Upload.t()) :: {:ok, any()} | {:error, String.t()}
-
   # Formats not compatible with exiftool at this time
   def filter(%Pleroma.Upload{content_type: "image/heic"}), do: {:ok, :noop}
   def filter(%Pleroma.Upload{content_type: "image/webp"}), do: {:ok, :noop}
diff --git a/lib/pleroma/upload/filter/mogrifun.ex b/lib/pleroma/upload/filter/mogrifun.ex
index a0f247b70..9716580a8 100644
--- a/lib/pleroma/upload/filter/mogrifun.ex
+++ b/lib/pleroma/upload/filter/mogrifun.ex
@@ -38,7 +38,6 @@ defmodule Pleroma.Upload.Filter.Mogrifun do
     [{"fill", "yellow"}, {"tint", "40"}]
   ]
 
-  @spec filter(Pleroma.Upload.t()) :: {:ok, atom()} | {:error, String.t()}
   def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do
     try do
       Filter.Mogrify.do_filter(file, [Enum.random(@filters)])
diff --git a/lib/pleroma/upload/filter/mogrify.ex b/lib/pleroma/upload/filter/mogrify.ex
index 06efbf321..d1e166022 100644
--- a/lib/pleroma/upload/filter/mogrify.ex
+++ b/lib/pleroma/upload/filter/mogrify.ex
@@ -8,7 +8,6 @@ defmodule Pleroma.Upload.Filter.Mogrify do
   @type conversion :: action :: String.t() | {action :: String.t(), opts :: String.t()}
   @type conversions :: conversion() | [conversion()]
 
-  @spec filter(Pleroma.Upload.t()) :: {:ok, :atom} | {:error, String.t()}
   def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do
     try do
       do_filter(file, Pleroma.Config.get!([__MODULE__, :args]))

From f8ce639e3f76257097793c666d3ebf8f22539a30 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Tue, 28 May 2024 09:30:19 -0400
Subject: [PATCH 143/192] Dialyzer: guard clause can never succeed

lib/pleroma/web/activity_pub/mrf/dnsrbl_policy.ex:106:guard_fail
The guard clause:

when _ ::
  [
    binary()
    | [string() | char()]
    | {string() | integer(), string()}
    | {{byte(), byte(), byte(), byte()}, integer(), binary()}
    | {integer(), integer(), integer(), string() | byte()}
    | {integer(), integer(), string(), string(), string(), string()}
    | {string(), string(), integer(), integer(), integer(), integer(), integer()}
    | {char(), char(), char(), char(), char(), char(), char(), char()}
  ] === nil

can never succeed.
---
 lib/pleroma/web/activity_pub/mrf/dnsrbl_policy.ex | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/lib/pleroma/web/activity_pub/mrf/dnsrbl_policy.ex b/lib/pleroma/web/activity_pub/mrf/dnsrbl_policy.ex
index 9543cc545..7c6bb888f 100644
--- a/lib/pleroma/web/activity_pub/mrf/dnsrbl_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/dnsrbl_policy.ex
@@ -103,7 +103,11 @@ defmodule Pleroma.Web.ActivityPub.MRF.DNSRBLPolicy do
         {:ok, object}
       else
         Task.start(fn ->
-          reason = rblquery(query, :txt) || "undefined"
+          reason =
+            case rblquery(query, :txt) do
+              [[result]] -> result
+              _ -> "undefined"
+            end
 
           Logger.warning(
             "DNSRBL Rejected activity from #{actor_host} for reason: #{inspect(reason)}"

From 18835bf7012e8e234eb27456a437f4d1e8796645 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Tue, 28 May 2024 09:38:36 -0400
Subject: [PATCH 144/192] Use the configured http client options for mediaproxy

---
 lib/pleroma/helpers/media_helper.ex | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/lib/pleroma/helpers/media_helper.ex b/lib/pleroma/helpers/media_helper.ex
index e44114d9d..0ac07fa41 100644
--- a/lib/pleroma/helpers/media_helper.ex
+++ b/lib/pleroma/helpers/media_helper.ex
@@ -25,7 +25,7 @@ defmodule Pleroma.Helpers.MediaHelper do
   end
 
   def image_resize(url, options) do
-    with {:ok, env} <- HTTP.get(url, [], pool: :media),
+    with {:ok, env} <- HTTP.get(url, [], http_client_opts()),
          {:ok, resized} <-
            Operation.thumbnail_buffer(env.body, options.max_width,
              height: options.max_height,
@@ -46,7 +46,7 @@ defmodule Pleroma.Helpers.MediaHelper do
   def video_framegrab(url) do
     with executable when is_binary(executable) <- System.find_executable("ffmpeg"),
          false <- @cachex.exists?(:failed_media_helper_cache, url),
-         {:ok, env} <- HTTP.get(url, [], pool: :media),
+         {:ok, env} <- HTTP.get(url, [], http_client_opts()),
          {:ok, pid} <- StringIO.open(env.body) do
       body_stream = IO.binstream(pid, 1)
 
@@ -84,4 +84,6 @@ defmodule Pleroma.Helpers.MediaHelper do
       {:error, _} = error -> error
     end
   end
+
+  defp http_client_opts, do: Pleroma.Config.get([:media_proxy, :proxy_opts, :http], pool: :media)
 end

From 17ebb2df8404474ef66c6a38e974143166b5e49b Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Tue, 28 May 2024 09:43:35 -0400
Subject: [PATCH 145/192] Dialyzer: fix pattern matches preventing video
 thumbnailing from working

lib/pleroma/web/media_proxy/media_proxy_controller.ex:154:pattern_match
The pattern can never match the type.

Pattern:
{:ok, _thumbnail_binary}

Type:
{:error, boolean() | {:ffmpeg, :command_not_found}}
---
 changelog.d/video-thumbs.fix        |  1 +
 lib/pleroma/helpers/media_helper.ex | 10 +++++-----
 2 files changed, 6 insertions(+), 5 deletions(-)
 create mode 100644 changelog.d/video-thumbs.fix

diff --git a/changelog.d/video-thumbs.fix b/changelog.d/video-thumbs.fix
new file mode 100644
index 000000000..03e862f3d
--- /dev/null
+++ b/changelog.d/video-thumbs.fix
@@ -0,0 +1 @@
+Video thumbnails were not being generated due to a negative cache lookup logic error
diff --git a/lib/pleroma/helpers/media_helper.ex b/lib/pleroma/helpers/media_helper.ex
index 0ac07fa41..8566ab3ea 100644
--- a/lib/pleroma/helpers/media_helper.ex
+++ b/lib/pleroma/helpers/media_helper.ex
@@ -45,7 +45,7 @@ defmodule Pleroma.Helpers.MediaHelper do
   @spec video_framegrab(String.t()) :: {:ok, binary()} | {:error, any()}
   def video_framegrab(url) do
     with executable when is_binary(executable) <- System.find_executable("ffmpeg"),
-         false <- @cachex.exists?(:failed_media_helper_cache, url),
+         {:ok, false} <- @cachex.exists?(:failed_media_helper_cache, url),
          {:ok, env} <- HTTP.get(url, [], http_client_opts()),
          {:ok, pid} <- StringIO.open(env.body) do
       body_stream = IO.binstream(pid, 1)
@@ -71,13 +71,13 @@ defmodule Pleroma.Helpers.MediaHelper do
         end)
 
       case Task.yield(task, 5_000) do
-        nil ->
+        {:ok, result} ->
+          {:ok, result}
+
+        _ ->
           Task.shutdown(task)
           @cachex.put(:failed_media_helper_cache, url, nil)
           {:error, {:ffmpeg, :timeout}}
-
-        result ->
-          {:ok, result}
       end
     else
       nil -> {:error, {:ffmpeg, :command_not_found}}

From 1b3c84e241f8ed1066d113346365ce489971ac14 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Tue, 28 May 2024 09:58:44 -0400
Subject: [PATCH 146/192] Dialyzer: no_local_return

WebPushEncryption.send_web_push/4 was written to raise on erroroneus input, so we must guard against that.

lib/pleroma/web/push/impl.ex:65:no_return Function push_message/4 has no local return.
---
 lib/pleroma/web/push/impl.ex | 26 ++++++++++++++++----------
 1 file changed, 16 insertions(+), 10 deletions(-)

diff --git a/lib/pleroma/web/push/impl.ex b/lib/pleroma/web/push/impl.ex
index 9e68d827b..53334e72c 100644
--- a/lib/pleroma/web/push/impl.ex
+++ b/lib/pleroma/web/push/impl.ex
@@ -63,19 +63,25 @@ defmodule Pleroma.Web.Push.Impl do
 
   @doc "Push message to web"
   def push_message(body, sub, api_key, subscription) do
-    case WebPushEncryption.send_web_push(body, sub, api_key) do
-      {:ok, %{status: code}} when code in 400..499 ->
-        Logger.debug("Removing subscription record")
-        Repo.delete!(subscription)
-        :ok
+    try do
+      case WebPushEncryption.send_web_push(body, sub, api_key) do
+        {:ok, %{status: code}} when code in 400..499 ->
+          Logger.debug("Removing subscription record")
+          Repo.delete!(subscription)
+          :ok
 
-      {:ok, %{status: code}} when code in 200..299 ->
-        :ok
+        {:ok, %{status: code}} when code in 200..299 ->
+          :ok
 
-      {:ok, %{status: code}} ->
-        Logger.error("Web Push Notification failed with code: #{code}")
-        :error
+        {:ok, %{status: code}} ->
+          Logger.error("Web Push Notification failed with code: #{code}")
+          :error
 
+        error ->
+          Logger.error("Web Push Notification failed with #{inspect(error)}")
+          :error
+      end
+    rescue
       error ->
         Logger.error("Web Push Notification failed with #{inspect(error)}")
         :error

From a041879eaaa7ca8ca421a89e303f58b5b68bc6da Mon Sep 17 00:00:00 2001
From: Lain Soykaf <lain@lain.com>
Date: Tue, 28 May 2024 17:04:43 +0400
Subject: [PATCH 147/192] Linting

---
 test/mix/tasks/pleroma/database_test.exs | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/test/mix/tasks/pleroma/database_test.exs b/test/mix/tasks/pleroma/database_test.exs
index 01fd97e2d..5f5ab5195 100644
--- a/test/mix/tasks/pleroma/database_test.exs
+++ b/test/mix/tasks/pleroma/database_test.exs
@@ -352,12 +352,17 @@ defmodule Mix.Tasks.Pleroma.DatabaseTest do
     end
 
     test "We don't have unexpected tables which may contain objects that are referenced by activities" do
-      # We can delete orphaned activities. For that we look for the objects they reference in the 'objects', 'activities', and 'users' table.
-      # If someone adds another table with objects (idk, maybe with separate relations, or collections or w/e), then we need to make sure we
-      # add logic for that in the 'prune_objects' task so that we don't wrongly delete their corresponding activities.
+      # We can delete orphaned activities. For that we look for the objects
+      # they reference in the 'objects', 'activities', and 'users' table.
+      # If someone adds another table with objects (idk, maybe with separate
+      # relations, or collections or w/e), then we need to make sure we
+      # add logic for that in the 'prune_objects' task so that we don't
+      # wrongly delete their corresponding activities.
       # So when someone adds (or removes) a table, this test will fail.
-      # Either the table contains objects which can be referenced from the activities table
-      # => in that case the prune_objects job should be adapted so we don't delete activities who still have the referenced object.
+      # Either the table contains objects which can be referenced from the
+      # activities table
+      # => in that case the prune_objects job should be adapted so we don't
+      #    delete activities who still have the referenced object.
       # Or it doesn't contain objects which can be referenced from the activities table
       # => in that case you can add/remove the table to/from this (sorted) list.
 
@@ -463,7 +468,8 @@ defmodule Mix.Tasks.Pleroma.DatabaseTest do
       })
       |> Repo.insert()
 
-      # The remote activities without existing reference, and only the remote activities without existing reference, are deleted
+      # The remote activities without existing reference,
+      # and only the remote activities without existing reference, are deleted
       # if, and only if, we provide the --prune-orphaned-activities option
       assert length(Repo.all(Activity)) == 5
       Mix.Tasks.Pleroma.Database.run(["prune_objects"])

From 8743c6c640d395ff6d7d268df1e382ba0fb0ca96 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Tue, 28 May 2024 10:36:00 -0400
Subject: [PATCH 148/192] Dialyzer: The pattern can never match the type

We will never pass :plain to query_with/4, so remove that match and change it to query_with/3

lib/pleroma/search/database_search.ex:127:pattern_match
The pattern can never match the type.

Pattern:
_q, :rum, _search_query, :plain

Type:

  %Ecto.Query{
    :aliases => _,
    :assocs => _,
    :combinations => _,
    :distinct => _,
    :from => _,
    :group_bys => _,
    :havings => _,
    :joins => _,
    :limit => _,
    :lock => _,
    :offset => _,
    :order_bys => _,
    :prefix => _,
    :preloads => _,
    :select => _,
    :sources => _,
    :updates => _,
    :wheres => _,
    :windows => _,
    :with_ctes => _
  },
  :rum,
  _,
  :websearch
---
 lib/pleroma/search/database_search.ex | 36 +++------------------------
 1 file changed, 3 insertions(+), 33 deletions(-)

diff --git a/lib/pleroma/search/database_search.ex b/lib/pleroma/search/database_search.ex
index c6fe8a9bd..aef5d1e74 100644
--- a/lib/pleroma/search/database_search.ex
+++ b/lib/pleroma/search/database_search.ex
@@ -28,7 +28,7 @@ defmodule Pleroma.Search.DatabaseSearch do
       |> Activity.with_preloaded_object()
       |> Activity.restrict_deactivated_users()
       |> restrict_public(user)
-      |> query_with(index_type, search_query, :websearch)
+      |> query_with(index_type, search_query)
       |> maybe_restrict_local(user)
       |> maybe_restrict_author(author)
       |> maybe_restrict_blocked(user)
@@ -88,25 +88,7 @@ defmodule Pleroma.Search.DatabaseSearch do
     )
   end
 
-  defp query_with(q, :gin, search_query, :plain) do
-    %{rows: [[tsc]]} =
-      Ecto.Adapters.SQL.query!(
-        Pleroma.Repo,
-        "select current_setting('default_text_search_config')::regconfig::oid;"
-      )
-
-    from([a, o] in q,
-      where:
-        fragment(
-          "to_tsvector(?::oid::regconfig, ?->>'content') @@ plainto_tsquery(?)",
-          ^tsc,
-          o.data,
-          ^search_query
-        )
-    )
-  end
-
-  defp query_with(q, :gin, search_query, :websearch) do
+  defp query_with(q, :gin, search_query) do
     %{rows: [[tsc]]} =
       Ecto.Adapters.SQL.query!(
         Pleroma.Repo,
@@ -124,19 +106,7 @@ defmodule Pleroma.Search.DatabaseSearch do
     )
   end
 
-  defp query_with(q, :rum, search_query, :plain) do
-    from([a, o] in q,
-      where:
-        fragment(
-          "? @@ plainto_tsquery(?)",
-          o.fts_content,
-          ^search_query
-        ),
-      order_by: [fragment("? <=> now()::date", o.inserted_at)]
-    )
-  end
-
-  defp query_with(q, :rum, search_query, :websearch) do
+  defp query_with(q, :rum, search_query) do
     from([a, o] in q,
       where:
         fragment(

From 6551ca2db7a0907252bbc649c7d082b3edf92a93 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Tue, 28 May 2024 10:40:54 -0400
Subject: [PATCH 149/192] Dialyzer: overlapping_contract

Wrong @spec name for remove_from_block/2

lib/pleroma/user.ex:2721:overlapping_contract
Overloaded contract for Pleroma.User.add_to_block/2 has
overlapping domains; such contracts are currently unsupported and
are simply ignored.
---
 lib/pleroma/user.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index 6d6aa98b5..d94b68ce0 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -2727,7 +2727,7 @@ defmodule Pleroma.User do
     end
   end
 
-  @spec add_to_block(User.t(), User.t()) ::
+  @spec remove_from_block(User.t(), User.t()) ::
           {:ok, UserRelationship.t()} | {:ok, nil} | {:error, Ecto.Changeset.t()}
   defp remove_from_block(%User{} = user, %User{} = blocked) do
     with {:ok, relationship} <- UserRelationship.delete_block(user, blocked) do

From 6b6a2adb07c3b9a52cd0a5adf435a916088bb4d7 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Tue, 28 May 2024 10:49:43 -0400
Subject: [PATCH 150/192] Dialyzer: The function call will not succeed.

:idna.encode/1 expects a charlist even though it will accept a binary string. That functionality is undocumented / not part of its typespec, so we should turn it into a charlist first. Also switch to using match?/2

lib/pleroma/user.ex:2056:call
The function call will not succeed.

:idna.encode(_host :: binary())

will never return since the success typing is:
(string()) :: string()

and the contract is
(string()) :: string()
---
 lib/pleroma/user.ex | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index d94b68ce0..884c1f302 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -2053,7 +2053,8 @@ defmodule Pleroma.User do
            %{scheme: scheme, userinfo: nil, host: host}
            when not_empty_string(host) and scheme in ["http", "https"] <-
              URI.parse(value),
-           {:not_idn, true} <- {:not_idn, to_string(:idna.encode(host)) == host},
+           {:not_idn, true} <-
+             {:not_idn, match?(^host, to_string(:idna.encode(to_charlist(host))))},
            "me" <- Pleroma.Web.RelMe.maybe_put_rel_me(value, profile_urls) do
         CommonUtils.to_masto_date(NaiveDateTime.utc_now())
       else

From f663135724fac2ef12107369f94d8e030bc1b4a5 Mon Sep 17 00:00:00 2001
From: Lain Soykaf <lain@lain.com>
Date: Tue, 28 May 2024 18:54:36 +0400
Subject: [PATCH 151/192] DatabaseTest: Fix test.

---
 test/mix/tasks/pleroma/database_test.exs | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/test/mix/tasks/pleroma/database_test.exs b/test/mix/tasks/pleroma/database_test.exs
index 5f5ab5195..d773038cb 100644
--- a/test/mix/tasks/pleroma/database_test.exs
+++ b/test/mix/tasks/pleroma/database_test.exs
@@ -375,6 +375,7 @@ defmodule Mix.Tasks.Pleroma.DatabaseTest do
                ["announcements"],
                ["apps"],
                ["backups"],
+               ["bookmark_folders"],
                ["bookmarks"],
                ["chat_message_references"],
                ["chats"],
@@ -405,6 +406,8 @@ defmodule Mix.Tasks.Pleroma.DatabaseTest do
                ["push_subscriptions"],
                ["registrations"],
                ["report_notes"],
+               ["rich_media_card"],
+               ["rules"],
                ["scheduled_activities"],
                ["schema_migrations"],
                ["thread_mutes"],

From 79c418bcb7534ae3645ee0b7728f8e65a031f7a4 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Tue, 28 May 2024 11:07:28 -0400
Subject: [PATCH 152/192] Dialyzer: fix invalid @spec

---
 lib/pleroma/web/o_auth/token.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pleroma/web/o_auth/token.ex b/lib/pleroma/web/o_auth/token.ex
index a5ad2e909..9b1198b42 100644
--- a/lib/pleroma/web/o_auth/token.ex
+++ b/lib/pleroma/web/o_auth/token.ex
@@ -96,7 +96,7 @@ defmodule Pleroma.Web.OAuth.Token do
     |> validate_required([:valid_until])
   end
 
-  @spec create(App.t(), User.t(), map()) :: {:ok, Token} | {:error, Ecto.Changeset.t()}
+  @spec create(App.t(), User.t(), map()) :: {:ok, Token.t()} | {:error, Ecto.Changeset.t()}
   def create(%App{} = app, %User{} = user, attrs \\ %{}) do
     with {:ok, token} <- do_create(app, user, attrs) do
       if Pleroma.Config.get([:oauth2, :clean_expired_tokens]) do

From c16ef40f134e69ad509a52dd3470672ca297f084 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Wed, 29 May 2024 07:59:02 +0200
Subject: [PATCH 153/192] RichMedia: Respect configuration on status previews

---
 changelog.d/rich_media_config.skip | 0
 lib/pleroma/web/rich_media/card.ex | 3 ++-
 2 files changed, 2 insertions(+), 1 deletion(-)
 create mode 100644 changelog.d/rich_media_config.skip

diff --git a/changelog.d/rich_media_config.skip b/changelog.d/rich_media_config.skip
new file mode 100644
index 000000000..e69de29bb
diff --git a/lib/pleroma/web/rich_media/card.ex b/lib/pleroma/web/rich_media/card.ex
index 36a1ae44a..f49bb2fee 100644
--- a/lib/pleroma/web/rich_media/card.ex
+++ b/lib/pleroma/web/rich_media/card.ex
@@ -104,7 +104,8 @@ defmodule Pleroma.Web.RichMedia.Card do
   @spec get_by_activity(Activity.t()) :: t() | nil | :error
   # Fake/Draft activity
   def get_by_activity(%Activity{id: "pleroma:fakeid"} = activity) do
-    with %Object{} = object <- Object.normalize(activity, fetch: false),
+    with {_, true} <- {:config, @config_impl.get([:rich_media, :enabled])},
+         %Object{} = object <- Object.normalize(activity, fetch: false),
          url when not is_nil(url) <- HTML.extract_first_external_url_from_object(object) do
       case get_by_url(url) do
         # Cache hit

From 65c8763907e28a12bc0a4c1269b5a765006d65b2 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Wed, 29 May 2024 08:00:19 +0200
Subject: [PATCH 154/192] RichMedia: Add extra checks on configuration

---
 lib/pleroma/web/rich_media/card.ex   | 22 +++++++++++++---------
 lib/pleroma/web/rich_media/parser.ex |  6 +++++-
 2 files changed, 18 insertions(+), 10 deletions(-)

diff --git a/lib/pleroma/web/rich_media/card.ex b/lib/pleroma/web/rich_media/card.ex
index f49bb2fee..040066f36 100644
--- a/lib/pleroma/web/rich_media/card.ex
+++ b/lib/pleroma/web/rich_media/card.ex
@@ -77,19 +77,23 @@ defmodule Pleroma.Web.RichMedia.Card do
 
   @spec get_or_backfill_by_url(String.t(), map()) :: t() | nil
   def get_or_backfill_by_url(url, backfill_opts \\ %{}) do
-    case get_by_url(url) do
-      %__MODULE__{} = card ->
-        card
+    if @config_impl.get([:rich_media, :enabled]) do
+      case get_by_url(url) do
+        %__MODULE__{} = card ->
+          card
 
-      nil ->
-        backfill_opts = Map.put(backfill_opts, :url, url)
+        nil ->
+          backfill_opts = Map.put(backfill_opts, :url, url)
 
-        Backfill.start(backfill_opts)
+          Backfill.start(backfill_opts)
 
-        nil
+          nil
 
-      :error ->
-        nil
+        :error ->
+          nil
+      end
+    else
+      nil
     end
   end
 
diff --git a/lib/pleroma/web/rich_media/parser.ex b/lib/pleroma/web/rich_media/parser.ex
index 37cf29029..7f6b5d388 100644
--- a/lib/pleroma/web/rich_media/parser.ex
+++ b/lib/pleroma/web/rich_media/parser.ex
@@ -15,10 +15,14 @@ defmodule Pleroma.Web.RichMedia.Parser do
 
   @spec parse(String.t()) :: {:ok, map()} | {:error, any()}
   def parse(url) do
-    with :ok <- validate_page_url(url),
+    with {_, true} <- {:config, @config_impl.get([:rich_media, :enabled])},
+         :ok <- validate_page_url(url),
          {:ok, data} <- parse_url(url) do
       data = Map.put(data, "url", url)
       {:ok, data}
+    else
+      {:config, _} -> {:error, :rich_media_disabled}
+      e -> e
     end
   end
 

From 14b4bd69a83846b3c117624851b96b9fc30528bf Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Wed, 29 May 2024 10:44:34 -0400
Subject: [PATCH 155/192] Add additional flags to the Pleroma.Search.Indexer
 Mix task

---
 changelog.d/mix-indexer.add             | 1 +
 lib/mix/tasks/pleroma/search/indexer.ex | 9 ++++++---
 2 files changed, 7 insertions(+), 3 deletions(-)
 create mode 100644 changelog.d/mix-indexer.add

diff --git a/changelog.d/mix-indexer.add b/changelog.d/mix-indexer.add
new file mode 100644
index 000000000..6effb959b
--- /dev/null
+++ b/changelog.d/mix-indexer.add
@@ -0,0 +1 @@
+Permit passing --chunk and --step values to the Pleroma.Search.Indexer Mix task
diff --git a/lib/mix/tasks/pleroma/search/indexer.ex b/lib/mix/tasks/pleroma/search/indexer.ex
index 81a9fced6..2a52472f9 100644
--- a/lib/mix/tasks/pleroma/search/indexer.ex
+++ b/lib/mix/tasks/pleroma/search/indexer.ex
@@ -33,15 +33,18 @@ defmodule Mix.Tasks.Pleroma.Search.Indexer do
       OptionParser.parse(
         options,
         strict: [
-          limit: :integer
+          chunk: :integer,
+          limit: :integer,
+          step: :integer
         ]
       )
 
     start_pleroma()
 
+    chunk_size = Keyword.get(options, :chunk, 100)
     limit = Keyword.get(options, :limit, 100_000)
+    per_step = Keyword.get(options, :step, 1000)
 
-    per_step = 1000
     chunks = max(div(limit, per_step), 1)
 
     1..chunks
@@ -65,7 +68,7 @@ defmodule Mix.Tasks.Pleroma.Search.Indexer do
       IO.puts("Got #{length(ids)} activities, adding to indexer")
 
       ids
-      |> Enum.chunk_every(100)
+      |> Enum.chunk_every(chunk_size)
       |> Enum.each(fn chunk ->
         IO.puts("Adding #{length(chunk)} activities to indexing queue")
 

From 36b440d9bebb1ddba817cd9bd8f158ceadbe8aa2 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Wed, 29 May 2024 21:58:39 -0400
Subject: [PATCH 156/192] Update Bandit to 1.5.2

Lots of fixes, also requires Websock Adapter update due to internal module changes in Bandit 1.4.0.
---
 changelog.d/bandit_update_1.5.2.change |  1 +
 mix.exs                                |  3 ++-
 mix.lock                               | 16 ++++++++--------
 3 files changed, 11 insertions(+), 9 deletions(-)
 create mode 100644 changelog.d/bandit_update_1.5.2.change

diff --git a/changelog.d/bandit_update_1.5.2.change b/changelog.d/bandit_update_1.5.2.change
new file mode 100644
index 000000000..c4aae1636
--- /dev/null
+++ b/changelog.d/bandit_update_1.5.2.change
@@ -0,0 +1 @@
+Update Bandit to 1.5.2
diff --git a/mix.exs b/mix.exs
index fe50139ef..a44daab8b 100644
--- a/mix.exs
+++ b/mix.exs
@@ -188,7 +188,8 @@ defmodule Pleroma.Mixfile do
       {:exile,
        git: "https://github.com/akash-akya/exile.git",
        ref: "be87c33b02a7c3c5d22d2ece01fbd462355b28ef"},
-      {:bandit, "~> 1.2"},
+      {:bandit, "~> 1.5.2"},
+      {:websock_adapter, "~> 0.5.6"},
 
       ## dev & test
       {:ex_doc, "~> 0.22", only: :dev, runtime: false},
diff --git a/mix.lock b/mix.lock
index 86545adcf..a55ad0126 100644
--- a/mix.lock
+++ b/mix.lock
@@ -1,6 +1,6 @@
 %{
   "accept": {:hex, :accept, "0.3.5", "b33b127abca7cc948bbe6caa4c263369abf1347cfa9d8e699c6d214660f10cd1", [:rebar3], [], "hexpm", "11b18c220bcc2eab63b5470c038ef10eb6783bcb1fcdb11aa4137defa5ac1bb8"},
-  "bandit": {:hex, :bandit, "1.2.1", "aa485b4ac175065b8e0fb5864ddd5dd7b50d52336b36f61c82f484c3718b3d15", [:mix], [{:hpax, "~> 0.1.1", [hex: :hpax, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:thousand_island, "~> 1.0", [hex: :thousand_island, repo: "hexpm", optional: false]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "27393e590a407f1b7d51c5fee4737f139fe224a30449ce25061eac70f763896b"},
+  "bandit": {:hex, :bandit, "1.5.2", "ed0a41c43a9e529c670d0fd48371db4027e7b80d43b1942893e17deb8bed0540", [:mix], [{:hpax, "~> 0.1.1", [hex: :hpax, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:thousand_island, "~> 1.0", [hex: :thousand_island, repo: "hexpm", optional: false]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "35ddbdce7e8a2a3c6b5093f7299d70832a43ed2f4a1852885a61d334cab1b4ad"},
   "base62": {:hex, :base62, "1.2.2", "85c6627eb609317b70f555294045895ffaaeb1758666ab9ef9ca38865b11e629", [:mix], [{:custom_base, "~> 0.2.1", [hex: :custom_base, repo: "hexpm", optional: false]}], "hexpm", "d41336bda8eaa5be197f1e4592400513ee60518e5b9f4dcf38f4b4dae6f377bb"},
   "bbcode_pleroma": {:hex, :bbcode_pleroma, "0.2.0", "d36f5bca6e2f62261c45be30fa9b92725c0655ad45c99025cb1c3e28e25803ef", [:mix], [{:nimble_parsec, "~> 0.5", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "19851074419a5fedb4ef49e1f01b30df504bb5dbb6d6adfc135238063bebd1c3"},
   "bcrypt_elixir": {:hex, :bcrypt_elixir, "2.3.1", "5114d780459a04f2b4aeef52307de23de961b69e13a5cd98a911e39fda13f420", [:make, :mix], [{:comeonin, "~> 5.3", [hex: :comeonin, repo: "hexpm", optional: false]}, {:elixir_make, "~> 0.6", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm", "42182d5f46764def15bf9af83739e3bf4ad22661b1c34fc3e88558efced07279"},
@@ -19,9 +19,9 @@
   "connection": {:hex, :connection, "1.1.0", "ff2a49c4b75b6fb3e674bfc5536451607270aac754ffd1bdfe175abe4a6d7a68", [:mix], [], "hexpm", "722c1eb0a418fbe91ba7bd59a47e28008a189d47e37e0e7bb85585a016b2869c"},
   "cors_plug": {:hex, :cors_plug, "2.0.3", "316f806d10316e6d10f09473f19052d20ba0a0ce2a1d910ddf57d663dac402ae", [:mix], [{:plug, "~> 1.8", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "ee4ae1418e6ce117fc42c2ba3e6cbdca4e95ecd2fe59a05ec6884ca16d469aea"},
   "covertool": {:hex, :covertool, "2.0.6", "4a291b4e3449025b0595d8f44c8d7635d4f48f033be2ce88d22a329f36f94a91", [:rebar3], [], "hexpm", "5db3fcd82180d8ea4ad857d4d1ab21a8d31b5aee0d60d2f6c0f9e25a411d1e21"},
-  "cowboy": {:hex, :cowboy, "2.10.0", "ff9ffeff91dae4ae270dd975642997afe2a1179d94b1887863e43f681a203e26", [:make, :rebar3], [{:cowlib, "2.12.1", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "1.8.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "3afdccb7183cc6f143cb14d3cf51fa00e53db9ec80cdcd525482f5e99bc41d6b"},
+  "cowboy": {:hex, :cowboy, "2.12.0", "f276d521a1ff88b2b9b4c54d0e753da6c66dd7be6c9fca3d9418b561828a3731", [:make, :rebar3], [{:cowlib, "2.13.0", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "1.8.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "8a7abe6d183372ceb21caa2709bec928ab2b72e18a3911aa1771639bef82651e"},
   "cowboy_telemetry": {:hex, :cowboy_telemetry, "0.4.0", "f239f68b588efa7707abce16a84d0d2acf3a0f50571f8bb7f56a15865aae820c", [:rebar3], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "7d98bac1ee4565d31b62d59f8823dfd8356a169e7fcbb83831b8a5397404c9de"},
-  "cowlib": {:hex, :cowlib, "2.12.1", "a9fa9a625f1d2025fe6b462cb865881329b5caff8f1854d1cbc9f9533f00e1e1", [:make, :rebar3], [], "hexpm", "163b73f6367a7341b33c794c4e88e7dbfe6498ac42dcd69ef44c5bc5507c8db0"},
+  "cowlib": {:hex, :cowlib, "2.13.0", "db8f7505d8332d98ef50a3ef34b34c1afddec7506e4ee4dd4a3a266285d282ca", [:make, :rebar3], [], "hexpm", "e1e1284dc3fc030a64b1ad0d8382ae7e99da46c3246b815318a4b848873800a4"},
   "credo": {:hex, :credo, "1.7.3", "05bb11eaf2f2b8db370ecaa6a6bda2ec49b2acd5e0418bc106b73b07128c0436", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "35ea675a094c934c22fb1dca3696f3c31f2728ae6ef5a53b5d648c11180a4535"},
   "crontab": {:hex, :crontab, "1.1.8", "2ce0e74777dfcadb28a1debbea707e58b879e6aa0ffbf9c9bb540887bce43617", [:mix], [{:ecto, "~> 1.0 or ~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm"},
   "custom_base": {:hex, :custom_base, "0.2.1", "4a832a42ea0552299d81652aa0b1f775d462175293e99dfbe4d7dbaab785a706", [:mix], [], "hexpm", "8df019facc5ec9603e94f7270f1ac73ddf339f56ade76a721eaa57c1493ba463"},
@@ -102,9 +102,9 @@
   "phoenix_swoosh": {:hex, :phoenix_swoosh, "1.2.1", "b74ccaa8046fbc388a62134360ee7d9742d5a8ae74063f34eb050279de7a99e1", [:mix], [{:finch, "~> 0.8", [hex: :finch, repo: "hexpm", optional: true]}, {:hackney, "~> 1.10", [hex: :hackney, repo: "hexpm", optional: true]}, {:phoenix, "~> 1.6", [hex: :phoenix, repo: "hexpm", optional: true]}, {:phoenix_html, "~> 3.0 or ~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:phoenix_view, "~> 1.0 or ~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: false]}, {:swoosh, "~> 1.5", [hex: :swoosh, repo: "hexpm", optional: false]}], "hexpm", "4000eeba3f9d7d1a6bf56d2bd56733d5cadf41a7f0d8ffe5bb67e7d667e204a2"},
   "phoenix_template": {:hex, :phoenix_template, "1.0.4", "e2092c132f3b5e5b2d49c96695342eb36d0ed514c5b252a77048d5969330d639", [:mix], [{:phoenix_html, "~> 2.14.2 or ~> 3.0 or ~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}], "hexpm", "2c0c81f0e5c6753faf5cca2f229c9709919aba34fab866d3bc05060c9c444206"},
   "phoenix_view": {:hex, :phoenix_view, "2.0.3", "4d32c4817fce933693741deeb99ef1392619f942633dde834a5163124813aad3", [:mix], [{:phoenix_html, "~> 2.14.2 or ~> 3.0 or ~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}], "hexpm", "cd34049af41be2c627df99cd4eaa71fc52a328c0c3d8e7d4aa28f880c30e7f64"},
-  "plug": {:hex, :plug, "1.15.3", "712976f504418f6dff0a3e554c40d705a9bcf89a7ccef92fc6a5ef8f16a30a97", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "cc4365a3c010a56af402e0809208873d113e9c38c401cabd88027ef4f5c01fd2"},
-  "plug_cowboy": {:hex, :plug_cowboy, "2.6.2", "753611b23b29231fb916b0cdd96028084b12aff57bfd7b71781bd04b1dbeb5c9", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowboy_telemetry, "~> 0.3", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "951ed2433df22f4c97b85fdb145d4cee561f36b74854d64c06d896d7cd2921a7"},
-  "plug_crypto": {:hex, :plug_crypto, "2.0.0", "77515cc10af06645abbfb5e6ad7a3e9714f805ae118fa1a70205f80d2d70fe73", [:mix], [], "hexpm", "53695bae57cc4e54566d993eb01074e4d894b65a3766f1c43e2c61a1b0f45ea9"},
+  "plug": {:hex, :plug, "1.16.0", "1d07d50cb9bb05097fdf187b31cf087c7297aafc3fed8299aac79c128a707e47", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "cbf53aa1f5c4d758a7559c0bd6d59e286c2be0c6a1fac8cc3eee2f638243b93e"},
+  "plug_cowboy": {:hex, :plug_cowboy, "2.7.1", "87677ffe3b765bc96a89be7960f81703223fe2e21efa42c125fcd0127dd9d6b2", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowboy_telemetry, "~> 0.3", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "02dbd5f9ab571b864ae39418db7811618506256f6d13b4a45037e5fe78dc5de3"},
+  "plug_crypto": {:hex, :plug_crypto, "2.1.0", "f44309c2b06d249c27c8d3f65cfe08158ade08418cf540fd4f72d4d6863abb7b", [:mix], [], "hexpm", "131216a4b030b8f8ce0f26038bc4421ae60e4bb95c5cf5395e1421437824c4fa"},
   "plug_static_index_html": {:hex, :plug_static_index_html, "1.0.0", "840123d4d3975585133485ea86af73cb2600afd7f2a976f9f5fd8b3808e636a0", [:mix], [{:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "79fd4fcf34d110605c26560cbae8f23c603ec4158c08298bd4360fdea90bb5cf"},
   "poison": {:hex, :poison, "3.1.0", "d9eb636610e096f86f25d9a46f35a9facac35609a7591b3be3326e99a0484665", [:mix], [], "hexpm", "fec8660eb7733ee4117b85f55799fd3833eb769a6df71ccf8903e8dc5447cfce"},
   "poolboy": {:hex, :poolboy, "1.5.2", "392b007a1693a64540cead79830443abf5762f5d30cf50bc95cb2c1aaafa006b", [:rebar3], [], "hexpm", "dad79704ce5440f3d5a3681c8590b9dc25d1a561e8f5a9c995281012860901e3"},
@@ -134,7 +134,7 @@
   "telemetry_metrics_prometheus_core": {:hex, :telemetry_metrics_prometheus_core, "1.2.0", "b583c3f18508f5c5561b674d16cf5d9afd2ea3c04505b7d92baaeac93c1b8260", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.6", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "9cba950e1c4733468efbe3f821841f34ac05d28e7af7798622f88ecdbbe63ea3"},
   "telemetry_poller": {:hex, :telemetry_poller, "1.0.0", "db91bb424e07f2bb6e73926fcafbfcbcb295f0193e0a00e825e589a0a47e8453", [:rebar3], [{:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "b3a24eafd66c3f42da30fc3ca7dda1e9d546c12250a2d60d7b81d264fbec4f6e"},
   "tesla": {:hex, :tesla, "1.8.0", "d511a4f5c5e42538d97eef7c40ec4f3e44effdc5068206f42ed859e09e51d1fd", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:exjsx, ">= 3.0.0", [hex: :exjsx, repo: "hexpm", optional: true]}, {:finch, "~> 0.13", [hex: :finch, repo: "hexpm", optional: true]}, {:fuse, "~> 2.4", [hex: :fuse, repo: "hexpm", optional: true]}, {:gun, ">= 1.0.0", [hex: :gun, repo: "hexpm", optional: true]}, {:hackney, "~> 1.6", [hex: :hackney, repo: "hexpm", optional: true]}, {:ibrowse, "4.4.2", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.0", [hex: :mint, repo: "hexpm", optional: true]}, {:msgpax, "~> 2.3", [hex: :msgpax, repo: "hexpm", optional: true]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm", "10501f360cd926a309501287470372af1a6e1cbed0f43949203a4c13300bc79f"},
-  "thousand_island": {:hex, :thousand_island, "1.3.2", "bc27f9afba6e1a676dd36507d42e429935a142cf5ee69b8e3f90bff1383943cd", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "0e085b93012cd1057b378fce40cbfbf381ff6d957a382bfdd5eca1a98eec2535"},
+  "thousand_island": {:hex, :thousand_island, "1.3.5", "6022b6338f1635b3d32406ff98d68b843ba73b3aa95cfc27154223244f3a6ca5", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "2be6954916fdfe4756af3239fb6b6d75d0b8063b5df03ba76fd8a4c87849e180"},
   "timex": {:hex, :timex, "3.7.7", "3ed093cae596a410759104d878ad7b38e78b7c2151c6190340835515d4a46b8a", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 1.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm", "0ec4b09f25fe311321f9fc04144a7e3affe48eb29481d7a5583849b6c4dfa0a7"},
   "toml": {:hex, :toml, "0.7.0", "fbcd773caa937d0c7a02c301a1feea25612720ac3fa1ccb8bfd9d30d822911de", [:mix], [], "hexpm", "0690246a2478c1defd100b0c9b89b4ea280a22be9a7b313a8a058a2408a2fa70"},
   "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "bd4fde4c15f3e993a999e019d64347489b91b7a9096af68b2bdadd192afa693f"},
@@ -145,6 +145,6 @@
   "vix": {:hex, :vix, "0.26.0", "027f10b6969b759318be84bd0bd8c88af877445e4e41cf96a0460392cea5399c", [:make, :mix], [{:castore, "~> 1.0 or ~> 0.1", [hex: :castore, repo: "hexpm", optional: false]}, {:cc_precompiler, "~> 0.2 or ~> 0.1.4", [hex: :cc_precompiler, repo: "hexpm", optional: false]}, {:elixir_make, "~> 0.8 or ~> 0.7.3", [hex: :elixir_make, repo: "hexpm", optional: false]}, {:kino, "~> 0.7", [hex: :kino, repo: "hexpm", optional: true]}], "hexpm", "71b0a79ae7f199cacfc8e679b0e4ba25ee47dc02e182c5b9097efb29fbe14efd"},
   "web_push_encryption": {:hex, :web_push_encryption, "0.3.1", "76d0e7375142dfee67391e7690e89f92578889cbcf2879377900b5620ee4708d", [:mix], [{:httpoison, "~> 1.0", [hex: :httpoison, repo: "hexpm", optional: false]}, {:jose, "~> 1.11.1", [hex: :jose, repo: "hexpm", optional: false]}], "hexpm", "4f82b2e57622fb9337559058e8797cb0df7e7c9790793bdc4e40bc895f70e2a2"},
   "websock": {:hex, :websock, "0.5.3", "2f69a6ebe810328555b6fe5c831a851f485e303a7c8ce6c5f675abeb20ebdadc", [:mix], [], "hexpm", "6105453d7fac22c712ad66fab1d45abdf049868f253cf719b625151460b8b453"},
-  "websock_adapter": {:hex, :websock_adapter, "0.5.5", "9dfeee8269b27e958a65b3e235b7e447769f66b5b5925385f5a569269164a210", [:mix], [{:bandit, ">= 0.6.0", [hex: :bandit, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.6", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "4b977ba4a01918acbf77045ff88de7f6972c2a009213c515a445c48f224ffce9"},
+  "websock_adapter": {:hex, :websock_adapter, "0.5.6", "0437fe56e093fd4ac422de33bf8fc89f7bc1416a3f2d732d8b2c8fd54792fe60", [:mix], [{:bandit, ">= 0.6.0", [hex: :bandit, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.6", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "e04378d26b0af627817ae84c92083b7e97aca3121196679b73c73b99d0d133ea"},
   "websockex": {:hex, :websockex, "0.4.3", "92b7905769c79c6480c02daacaca2ddd49de936d912976a4d3c923723b647bf0", [:mix], [], "hexpm", "95f2e7072b85a3a4cc385602d42115b73ce0b74a9121d0d6dbbf557645ac53e4"},
 }

From b5fcb82bffd3f31cf1318c1504fcb97e56b892cd Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Thu, 30 May 2024 10:47:51 -0400
Subject: [PATCH 157/192] Test for missing FK indexes

---
 test/fixtures/unindexed_fk.sql | 27 +++++++++++++++++++++++++++
 test/pleroma/schema_test.exs   | 17 +++++++++++++++++
 2 files changed, 44 insertions(+)
 create mode 100644 test/fixtures/unindexed_fk.sql
 create mode 100644 test/pleroma/schema_test.exs

diff --git a/test/fixtures/unindexed_fk.sql b/test/fixtures/unindexed_fk.sql
new file mode 100644
index 000000000..3b71679cf
--- /dev/null
+++ b/test/fixtures/unindexed_fk.sql
@@ -0,0 +1,27 @@
+-- Unindexed FK -- Missing indexes - For CI
+
+WITH y AS (
+SELECT
+pg_catalog.format('%I', c1.relname)  AS referencing_tbl,
+pg_catalog.quote_ident(a1.attname) AS referencing_column,
+(SELECT pg_get_expr(indpred, indrelid) FROM pg_catalog.pg_index WHERE indrelid = t.conrelid AND indkey[0] = t.conkey[1] AND indpred IS NOT NULL LIMIT 1) partial_statement
+FROM pg_catalog.pg_constraint t
+JOIN pg_catalog.pg_attribute  a1 ON a1.attrelid = t.conrelid AND a1.attnum = t.conkey[1]
+JOIN pg_catalog.pg_class  c1 ON c1.oid = t.conrelid
+JOIN pg_catalog.pg_namespace  n1 ON n1.oid = c1.relnamespace
+JOIN pg_catalog.pg_class  c2 ON c2.oid = t.confrelid
+JOIN pg_catalog.pg_namespace  n2 ON n2.oid = c2.relnamespace
+JOIN pg_catalog.pg_attribute  a2 ON a2.attrelid = t.confrelid AND a2.attnum = t.confkey[1]
+WHERE t.contype = 'f'
+AND NOT EXISTS (
+SELECT 1
+FROM pg_catalog.pg_index i
+WHERE i.indrelid = t.conrelid
+AND i.indkey[0] = t.conkey[1]
+AND indpred IS NULL
+)
+)
+SELECT  referencing_tbl || '.' || referencing_column as "column"
+FROM y
+WHERE (partial_statement IS NULL OR partial_statement <> ('(' || referencing_column || ' IS NOT NULL)'))
+ORDER BY 1;
\ No newline at end of file
diff --git a/test/pleroma/schema_test.exs b/test/pleroma/schema_test.exs
new file mode 100644
index 000000000..9bddd2031
--- /dev/null
+++ b/test/pleroma/schema_test.exs
@@ -0,0 +1,17 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.SchemaTest do
+  use Pleroma.DataCase, async: true
+
+  alias Pleroma.Repo
+
+  test "No unindexed foreign keys" do
+    query = File.read!("test/fixtures/unindexed_fk.sql")
+
+    {:ok, result} = Repo.query(query)
+
+    assert Enum.empty?(result.rows)
+  end
+end

From c20ac6d1adc224232422640d8bc11a80f5eff350 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Wed, 29 May 2024 21:27:35 -0400
Subject: [PATCH 158/192] Add missing foreign key indexes

---
 ...0240530011739_add_missing_foreign_keys.exs | 20 +++++++++++++++++++
 1 file changed, 20 insertions(+)
 create mode 100644 priv/repo/migrations/20240530011739_add_missing_foreign_keys.exs

diff --git a/priv/repo/migrations/20240530011739_add_missing_foreign_keys.exs b/priv/repo/migrations/20240530011739_add_missing_foreign_keys.exs
new file mode 100644
index 000000000..158f9701b
--- /dev/null
+++ b/priv/repo/migrations/20240530011739_add_missing_foreign_keys.exs
@@ -0,0 +1,20 @@
+defmodule Pleroma.Repo.Migrations.AddMissingForeignKeys do
+  use Ecto.Migration
+
+  def change do
+    create_if_not_exists(index(:announcement_read_relationships, :announcement_id))
+    create_if_not_exists(index(:bookmarks, :activity_id))
+    create_if_not_exists(index(:bookmarks, :folder_id))
+    create_if_not_exists(index(:chats, :recipient))
+    create_if_not_exists(index(:mfa_tokens, :authorization_id))
+    create_if_not_exists(index(:mfa_tokens, :user_id))
+    create_if_not_exists(index(:notifications, :activity_id))
+    create_if_not_exists(index(:oauth_authorizations, :app_id))
+    create_if_not_exists(index(:oauth_authorizations, :user_id))
+    create_if_not_exists(index(:password_reset_tokens, :user_id))
+    create_if_not_exists(index(:push_subscriptions, :token_id))
+    create_if_not_exists(index(:report_notes, :activity_id))
+    create_if_not_exists(index(:report_notes, :user_id))
+    create_if_not_exists(index(:user_notes, :target_id))
+  end
+end

From 5f6e477ecaa941a79b22599aca169164b7241bcf Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Thu, 30 May 2024 10:51:50 -0400
Subject: [PATCH 159/192] Missing FKs changelog

---
 changelog.d/missing-fks.add | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/missing-fks.add

diff --git a/changelog.d/missing-fks.add b/changelog.d/missing-fks.add
new file mode 100644
index 000000000..cf74de03b
--- /dev/null
+++ b/changelog.d/missing-fks.add
@@ -0,0 +1 @@
+Add missing indexes on foreign key relationships

From f5065eaf99a76695df26966055f90368df7043f3 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Thu, 30 May 2024 11:09:42 -0400
Subject: [PATCH 160/192] Fix Logger.warn deprecation error on OTP25

---
 changelog.d/mrf-nsfw-otp25.skip                     | 1 +
 lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)
 create mode 100644 changelog.d/mrf-nsfw-otp25.skip

diff --git a/changelog.d/mrf-nsfw-otp25.skip b/changelog.d/mrf-nsfw-otp25.skip
new file mode 100644
index 000000000..e804f19a0
--- /dev/null
+++ b/changelog.d/mrf-nsfw-otp25.skip
@@ -0,0 +1 @@
+noop
diff --git a/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex b/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex
index f7863039b..3d1c273b9 100644
--- a/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex
@@ -64,7 +64,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.NsfwApiPolicy do
       Jason.decode(body)
     else
       error ->
-        Logger.warn("""
+        Logger.warning("""
         [NsfwApiPolicy]: The API server failed. Skipping.
         #{inspect(error)}
         """)

From cfc8d7aade526b8f119683984977064cd3cd3d87 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Tue, 28 May 2024 13:14:34 -0400
Subject: [PATCH 161/192] IPFS uploader: dialyzer fixes

lib/pleroma/uploaders/ipfs.ex:43:no_return
Function put_file/1 has no local return.
________________________________________________________________________________
lib/pleroma/uploaders/ipfs.ex:49:call
The function call will not succeed.

Pleroma.HTTP.post(
  binary(),
  _mp :: %Tesla.Multipart{
    :boundary => binary(),
    :content_type_params => [binary()],
    :parts => [
      %Tesla.Multipart.Part{
        :body => binary(),
        :dispositions => [any()],
        :headers => [any()]
      },
      ...
    ]
  },
  [],
  [{:params, [{:"cid-version", <<49>>}]}]
)

will never return since the success typing is:
(binary(), binary(), [{binary(), binary()}], Keyword.t()) ::
  {:error, _}
  | {:ok,
     %Tesla.Env{
       :__client__ => %Tesla.Client{
         :adapter => nil | {_, _} | {_, _, _},
         :fun => _,
         :post => [any()],
         :pre => [any()]
       },
       :__module__ => atom(),
       :body => _,
       :headers => [{_, _}],
       :method => :delete | :get | :head | :options | :patch | :post | :put | :trace,
       :opts => [{_, _}],
       :query => [{_, _}],
       :status => nil | integer(),
       :url => binary()
     }}

and the contract is
(Pleroma.HTTP.Request.url(), String.t(), Pleroma.HTTP.Request.headers(), :elixir.keyword()) ::
  {:ok, Tesla.Env.t()} | {:error, any()}
---
 changelog.d/ipfs-dialyzer.skip       |  1 +
 config/config.exs                    |  4 +-
 lib/pleroma/http.ex                  |  4 +-
 lib/pleroma/uploaders/ipfs.ex        | 57 +++++++++++++---------------
 test/pleroma/uploaders/ipfs_test.exs | 51 ++++++++++++-------------
 5 files changed, 55 insertions(+), 62 deletions(-)
 create mode 100644 changelog.d/ipfs-dialyzer.skip

diff --git a/changelog.d/ipfs-dialyzer.skip b/changelog.d/ipfs-dialyzer.skip
new file mode 100644
index 000000000..b3e74cd19
--- /dev/null
+++ b/changelog.d/ipfs-dialyzer.skip
@@ -0,0 +1 @@
+no comment
diff --git a/config/config.exs b/config/config.exs
index c3b8ae0b7..a40ed28af 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -83,8 +83,8 @@ config :ex_aws, :s3,
   scheme: "https://"
 
 config :pleroma, Pleroma.Uploaders.IPFS,
-  post_gateway_url: nil,
-  get_gateway_url: nil
+  post_gateway_url: "http://localhost:5001",
+  get_gateway_url: "http://localhost:8080"
 
 config :pleroma, :emoji,
   shortcode_globs: ["/emoji/custom/**/*.png"],
diff --git a/lib/pleroma/http.ex b/lib/pleroma/http.ex
index eec61cf14..ec837e509 100644
--- a/lib/pleroma/http.ex
+++ b/lib/pleroma/http.ex
@@ -37,7 +37,7 @@ defmodule Pleroma.HTTP do
 
   See `Pleroma.HTTP.request/5`
   """
-  @spec post(Request.url(), String.t(), Request.headers(), keyword()) ::
+  @spec post(Request.url(), Tesla.Env.body(), Request.headers(), keyword()) ::
           {:ok, Env.t()} | {:error, any()}
   def post(url, body, headers \\ [], options \\ []),
     do: request(:post, url, body, headers, options)
@@ -56,7 +56,7 @@ defmodule Pleroma.HTTP do
   `{:ok, %Tesla.Env{}}` or `{:error, error}`
 
   """
-  @spec request(method(), Request.url(), String.t(), Request.headers(), keyword()) ::
+  @spec request(method(), Request.url(), Tesla.Env.body(), Request.headers(), keyword()) ::
           {:ok, Env.t()} | {:error, any()}
   def request(method, url, body, headers, options) when is_binary(url) do
     uri = URI.parse(url)
diff --git a/lib/pleroma/uploaders/ipfs.ex b/lib/pleroma/uploaders/ipfs.ex
index d171e4652..5930a129e 100644
--- a/lib/pleroma/uploaders/ipfs.ex
+++ b/lib/pleroma/uploaders/ipfs.ex
@@ -8,23 +8,10 @@ defmodule Pleroma.Uploaders.IPFS do
 
   alias Tesla.Multipart
 
+  @api_add "/api/v0/add"
+  @api_delete "/api/v0/files/rm"
   @config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config)
 
-  defp get_final_url(method) do
-    config = @config_impl.get([__MODULE__])
-    post_base_url = Keyword.get(config, :post_gateway_url)
-
-    Path.join([post_base_url, method])
-  end
-
-  def put_file_endpoint do
-    get_final_url("/api/v0/add")
-  end
-
-  def delete_file_endpoint do
-    get_final_url("/api/v0/files/rm")
-  end
-
   @placeholder "{CID}"
   def placeholder, do: @placeholder
 
@@ -40,26 +27,26 @@ defmodule Pleroma.Uploaders.IPFS do
   end
 
   @impl true
-  def put_file(%Pleroma.Upload{} = upload) do
+  def put_file(%Pleroma.Upload{tempfile: tempfile}) do
     mp =
       Multipart.new()
       |> Multipart.add_content_type_param("charset=utf-8")
-      |> Multipart.add_file(upload.tempfile)
+      |> Multipart.add_file(tempfile)
 
-    case Pleroma.HTTP.post(put_file_endpoint(), mp, [], params: ["cid-version": "1"]) do
-      {:ok, ret} ->
-        case Jason.decode(ret.body) do
-          {:ok, ret} ->
-            if Map.has_key?(ret, "Hash") do
-              {:ok, {:file, ret["Hash"]}}
-            else
-              {:error, "JSON doesn't contain Hash key"}
-            end
+    endpoint = ipfs_endpoint(@api_add)
 
-          error ->
-            Logger.error("#{__MODULE__}: #{inspect(error)}")
-            {:error, "JSON decode failed"}
-        end
+    with {:ok, %{body: body}} when is_binary(body) <-
+           Pleroma.HTTP.post(endpoint, mp, [], params: ["cid-version": "1"], pool: :upload),
+         {_, {:ok, decoded}} <- {:json, Jason.decode(body)},
+         {_, true} <- {:hash, Map.has_key?(decoded, "Hash")} do
+      {:ok, {:file, decoded["Hash"]}}
+    else
+      {:hash, false} ->
+        {:error, "JSON doesn't contain Hash key"}
+
+      {:json, error} ->
+        Logger.error("#{__MODULE__}: #{inspect(error)}")
+        {:error, "JSON decode failed"}
 
       error ->
         Logger.error("#{__MODULE__}: #{inspect(error)}")
@@ -69,9 +56,17 @@ defmodule Pleroma.Uploaders.IPFS do
 
   @impl true
   def delete_file(file) do
-    case Pleroma.HTTP.post(delete_file_endpoint(), "", [], params: [arg: file]) do
+    endpoint = ipfs_endpoint(@api_delete)
+
+    case Pleroma.HTTP.post(endpoint, "", [], params: [arg: file]) do
       {:ok, %{status: 204}} -> :ok
       error -> {:error, inspect(error)}
     end
   end
+
+  defp ipfs_endpoint(path) do
+    URI.parse(@config_impl.get([__MODULE__, :post_gateway_url]))
+    |> Map.put(:path, path)
+    |> URI.to_string()
+  end
 end
diff --git a/test/pleroma/uploaders/ipfs_test.exs b/test/pleroma/uploaders/ipfs_test.exs
index cf325b54f..bdf2933ac 100644
--- a/test/pleroma/uploaders/ipfs_test.exs
+++ b/test/pleroma/uploaders/ipfs_test.exs
@@ -14,25 +14,6 @@ defmodule Pleroma.Uploaders.IPFSTest do
 
   alias Pleroma.UnstubbedConfigMock, as: Config
 
-  describe "get_final_url" do
-    setup do
-      Config
-      |> expect(:get, fn [Pleroma.Uploaders.IPFS] ->
-        [post_gateway_url: "http://localhost:5001"]
-      end)
-
-      :ok
-    end
-
-    test "it returns the final url for put_file" do
-      assert IPFS.put_file_endpoint() == "http://localhost:5001/api/v0/add"
-    end
-
-    test "it returns the final url for delete_file" do
-      assert IPFS.delete_file_endpoint() == "http://localhost:5001/api/v0/files/rm"
-    end
-  end
-
   describe "get_file/1" do
     setup do
       Config
@@ -71,8 +52,8 @@ defmodule Pleroma.Uploaders.IPFSTest do
   describe "put_file/1" do
     setup do
       Config
-      |> expect(:get, fn [Pleroma.Uploaders.IPFS] ->
-        [post_gateway_url: "http://localhost:5001"]
+      |> expect(:get, fn [Pleroma.Uploaders.IPFS, :post_gateway_url] ->
+        "http://localhost:5001"
       end)
 
       file_upload = %Pleroma.Upload{
@@ -92,7 +73,11 @@ defmodule Pleroma.Uploaders.IPFSTest do
 
     test "save file", %{file_upload: file_upload} do
       with_mock Pleroma.HTTP,
-        post: fn "http://localhost:5001/api/v0/add", _mp, [], params: ["cid-version": "1"] ->
+        post: fn "http://localhost:5001/api/v0/add",
+                 _mp,
+                 [],
+                 params: ["cid-version": "1"],
+                 pool: :upload ->
           {:ok,
            %Tesla.Env{
              status: 200,
@@ -107,7 +92,11 @@ defmodule Pleroma.Uploaders.IPFSTest do
 
     test "returns error", %{file_upload: file_upload} do
       with_mock Pleroma.HTTP,
-        post: fn "http://localhost:5001/api/v0/add", _mp, [], params: ["cid-version": "1"] ->
+        post: fn "http://localhost:5001/api/v0/add",
+                 _mp,
+                 [],
+                 params: ["cid-version": "1"],
+                 pool: :upload ->
           {:error, "IPFS Gateway upload failed"}
         end do
         assert capture_log(fn ->
@@ -118,7 +107,11 @@ defmodule Pleroma.Uploaders.IPFSTest do
 
     test "returns error if JSON decode fails", %{file_upload: file_upload} do
       with_mock Pleroma.HTTP, [],
-        post: fn "http://localhost:5001/api/v0/add", _mp, [], params: ["cid-version": "1"] ->
+        post: fn "http://localhost:5001/api/v0/add",
+                 _mp,
+                 [],
+                 params: ["cid-version": "1"],
+                 pool: :upload ->
           {:ok, %Tesla.Env{status: 200, body: "invalid"}}
         end do
         assert capture_log(fn ->
@@ -130,7 +123,11 @@ defmodule Pleroma.Uploaders.IPFSTest do
 
     test "returns error if JSON body doesn't contain Hash key", %{file_upload: file_upload} do
       with_mock Pleroma.HTTP, [],
-        post: fn "http://localhost:5001/api/v0/add", _mp, [], params: ["cid-version": "1"] ->
+        post: fn "http://localhost:5001/api/v0/add",
+                 _mp,
+                 [],
+                 params: ["cid-version": "1"],
+                 pool: :upload ->
           {:ok, %Tesla.Env{status: 200, body: "{\"key\": \"value\"}"}}
         end do
         assert IPFS.put_file(file_upload) == {:error, "JSON doesn't contain Hash key"}
@@ -141,8 +138,8 @@ defmodule Pleroma.Uploaders.IPFSTest do
   describe "delete_file/1" do
     setup do
       Config
-      |> expect(:get, fn [Pleroma.Uploaders.IPFS] ->
-        [post_gateway_url: "http://localhost:5001"]
+      |> expect(:get, fn [Pleroma.Uploaders.IPFS, :post_gateway_url] ->
+        "http://localhost:5001"
       end)
 
       :ok

From 0302431888d457d254f152a502946e6ffe7935e4 Mon Sep 17 00:00:00 2001
From: Floatingghost <hannah@coffee-and-dreams.uk>
Date: Fri, 31 May 2024 09:04:00 -0400
Subject: [PATCH 162/192] Use proper workers for fetching pins instead of an
 ad-hoc task

BUG: https://git.pleroma.social/pleroma/pleroma/-/issues/3276
---
 changelog.d/pinned-collection-fetch.security  |  1 +
 lib/pleroma/web/activity_pub/activity_pub.ex  | 25 ++++++++--------
 .../web/activity_pub/activity_pub_test.exs    | 30 +++++++++++++++----
 3 files changed, 38 insertions(+), 18 deletions(-)
 create mode 100644 changelog.d/pinned-collection-fetch.security

diff --git a/changelog.d/pinned-collection-fetch.security b/changelog.d/pinned-collection-fetch.security
new file mode 100644
index 000000000..4e8746924
--- /dev/null
+++ b/changelog.d/pinned-collection-fetch.security
@@ -0,0 +1 @@
+Use proper workers for fetching pins instead of an ad-hoc task, fixing a potential fetch loop
diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index 5bb0fba6e..1247ae7ce 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -1794,24 +1794,25 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     end
   end
 
-  def pinned_fetch_task(nil), do: nil
-
-  def pinned_fetch_task(%{pinned_objects: pins}) do
-    if Enum.all?(pins, fn {ap_id, _} ->
-         Object.get_cached_by_ap_id(ap_id) ||
-           match?({:ok, _object}, Fetcher.fetch_object_from_id(ap_id))
-       end) do
-      :ok
-    else
-      :error
-    end
+  def enqueue_pin_fetches(%{pinned_objects: pins}) do
+    # enqueue a task to fetch all pinned objects
+    Enum.each(pins, fn {ap_id, _} ->
+      if is_nil(Object.get_cached_by_ap_id(ap_id)) do
+        Pleroma.Workers.RemoteFetcherWorker.enqueue("fetch_remote", %{
+          "id" => ap_id,
+          "depth" => 1
+        })
+      end
+    end)
   end
 
+  def enqueue_pin_fetches(_), do: nil
+
   def make_user_from_ap_id(ap_id, additional \\ []) do
     user = User.get_cached_by_ap_id(ap_id)
 
     with {:ok, data} <- fetch_and_prepare_user_from_ap_id(ap_id, additional) do
-      {:ok, _pid} = Task.start(fn -> pinned_fetch_task(data) end)
+      enqueue_pin_fetches(data)
 
       if user do
         user
diff --git a/test/pleroma/web/activity_pub/activity_pub_test.exs b/test/pleroma/web/activity_pub/activity_pub_test.exs
index 524294385..d278125ee 100644
--- a/test/pleroma/web/activity_pub/activity_pub_test.exs
+++ b/test/pleroma/web/activity_pub/activity_pub_test.exs
@@ -291,9 +291,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do
             body: featured_data,
             headers: [{"content-type", "application/activity+json"}]
           }
-      end)
 
-      Tesla.Mock.mock_global(fn
         %{
           method: :get,
           url: ^object_url
@@ -306,7 +304,18 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do
       end)
 
       {:ok, user} = ActivityPub.make_user_from_ap_id(ap_id)
-      Process.sleep(50)
+
+      assert_enqueued(
+        worker: Pleroma.Workers.RemoteFetcherWorker,
+        args: %{
+          "op" => "fetch_remote",
+          "id" => object_url,
+          "depth" => 1
+        }
+      )
+
+      # wait for oban
+      Pleroma.Tests.ObanHelpers.perform_all()
 
       assert user.featured_address == featured_url
       assert Map.has_key?(user.pinned_objects, object_url)
@@ -368,9 +377,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do
             body: featured_data,
             headers: [{"content-type", "application/activity+json"}]
           }
-      end)
 
-      Tesla.Mock.mock_global(fn
         %{
           method: :get,
           url: ^object_url
@@ -383,7 +390,18 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do
       end)
 
       {:ok, user} = ActivityPub.make_user_from_ap_id(ap_id)
-      Process.sleep(50)
+
+      assert_enqueued(
+        worker: Pleroma.Workers.RemoteFetcherWorker,
+        args: %{
+          "op" => "fetch_remote",
+          "id" => object_url,
+          "depth" => 1
+        }
+      )
+
+      # wait for oban
+      Pleroma.Tests.ObanHelpers.perform_all()
 
       assert user.featured_address == featured_url
       assert Map.has_key?(user.pinned_objects, object_url)

From 858d528cc1b551e23850a4b65eb43e44a2a18cf4 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Tue, 4 Jun 2024 22:44:56 -0400
Subject: [PATCH 163/192] Allow Cowboy to stream the response instead of chunk
 it

---
 changelog.d/cowboy-stream-chunked.fix |  1 +
 lib/pleroma/reverse_proxy.ex          | 16 +++++++++++++++-
 2 files changed, 16 insertions(+), 1 deletion(-)
 create mode 100644 changelog.d/cowboy-stream-chunked.fix

diff --git a/changelog.d/cowboy-stream-chunked.fix b/changelog.d/cowboy-stream-chunked.fix
new file mode 100644
index 000000000..07211bf18
--- /dev/null
+++ b/changelog.d/cowboy-stream-chunked.fix
@@ -0,0 +1 @@
+Restore Cowboy's ability to stream MediaProxy responses without Chunked encoding.
diff --git a/lib/pleroma/reverse_proxy.ex b/lib/pleroma/reverse_proxy.ex
index 4d13e51fc..8aec4ae58 100644
--- a/lib/pleroma/reverse_proxy.ex
+++ b/lib/pleroma/reverse_proxy.ex
@@ -8,7 +8,7 @@ defmodule Pleroma.ReverseProxy do
                       ~w(if-unmodified-since if-none-match) ++ @range_headers
   @resp_cache_headers ~w(etag date last-modified)
   @keep_resp_headers @resp_cache_headers ++
-                       ~w(content-type content-disposition content-encoding) ++
+                       ~w(content-length content-type content-disposition content-encoding) ++
                        ~w(content-range accept-ranges vary)
   @default_cache_control_header "public, max-age=1209600"
   @valid_resp_codes [200, 206, 304]
@@ -180,6 +180,7 @@ defmodule Pleroma.ReverseProxy do
     result =
       conn
       |> put_resp_headers(build_resp_headers(headers, opts))
+      |> streaming_compat
       |> send_chunked(status)
       |> chunk_reply(client, opts)
 
@@ -417,4 +418,17 @@ defmodule Pleroma.ReverseProxy do
 
     @cachex.put(:failed_proxy_url_cache, url, true, ttl: ttl)
   end
+
+  # When Cowboy handles a chunked response with a content-length header it streams
+  # over HTTP 1.1 instead of chunking. Bandit cannot stream over HTTP 1.1 so the header
+  # must be stripped or it breaks RFC compliance for Transfer Encoding: Chunked. RFC9112§6.2
+  #
+  # HTTP2 is always streamed for all adapters.
+  defp streaming_compat(conn) do
+    with Phoenix.Endpoint.Cowboy2Adapter <- Pleroma.Web.Endpoint.config(:adapter) do
+      conn
+    else
+      _ -> delete_resp_header(conn, "content-length")
+    end
+  end
 end

From 49156f0183524f13a1363d17b2220718405ea5b6 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Fri, 7 Jun 2024 13:31:32 -0400
Subject: [PATCH 164/192] Fixes for default disabled rich media in test
 environment

Also add a test to validate the correct error when disabled
---
 test/pleroma/web/rich_media/parser_test.exs | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/test/pleroma/web/rich_media/parser_test.exs b/test/pleroma/web/rich_media/parser_test.exs
index 3fcb5c808..a5f2563a2 100644
--- a/test/pleroma/web/rich_media/parser_test.exs
+++ b/test/pleroma/web/rich_media/parser_test.exs
@@ -13,6 +13,8 @@ defmodule Pleroma.Web.RichMedia.ParserTest do
     mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end)
   end
 
+  setup_all do: clear_config([:rich_media, :enabled], true)
+
   test "returns error when no metadata present" do
     assert {:error, _} = Parser.parse("https://example.com/empty")
   end
@@ -127,4 +129,10 @@ defmodule Pleroma.Web.RichMedia.ParserTest do
       assert :error == Parser.parse(url)
     end)
   end
+
+  test "returns error when disabled" do
+    clear_config([:rich_media, :enabled], false)
+
+    assert match?({:error, :rich_media_disabled}, Parser.parse("https://example.com/ogp"))
+  end
 end

From 5915062874f0697beb51bd80eabafb6cfa34d362 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Fri, 7 Jun 2024 15:50:11 -0400
Subject: [PATCH 165/192] Add missing notification types to the api spec

---
 changelog.d/notification-spex.skip                        | 0
 .../web/api_spec/operations/notification_operation.ex     | 8 +++++++-
 2 files changed, 7 insertions(+), 1 deletion(-)
 create mode 100644 changelog.d/notification-spex.skip

diff --git a/changelog.d/notification-spex.skip b/changelog.d/notification-spex.skip
new file mode 100644
index 000000000..e69de29bb
diff --git a/lib/pleroma/web/api_spec/operations/notification_operation.ex b/lib/pleroma/web/api_spec/operations/notification_operation.ex
index a79eb8f74..2dc0f66df 100644
--- a/lib/pleroma/web/api_spec/operations/notification_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/notification_operation.ex
@@ -203,7 +203,10 @@ defmodule Pleroma.Web.ApiSpec.NotificationOperation do
         "move",
         "follow_request",
         "poll",
-        "status"
+        "status",
+        "update",
+        "admin.sign_up",
+        "admin.report"
       ],
       description: """
       The type of event that resulted in the notification.
@@ -218,6 +221,9 @@ defmodule Pleroma.Web.ApiSpec.NotificationOperation do
       - `pleroma:chat_mention` - Someone mentioned you in a chat message
       - `pleroma:report` - Someone was reported
       - `status` - Someone you are subscribed to created a status
+      - `update` - A status you boosted has been edited
+      - `admin.sign_up` - Someone signed up (optionally sent to admins)
+      - `admin.report` - A new report has been filed
       """
     }
   end

From b52d772a6e66b13bac875b7289de3bd63c087db3 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Fri, 7 Jun 2024 16:03:10 -0400
Subject: [PATCH 166/192] Add some useful logging for ApiSpec errors

---
 lib/pleroma/web/api_spec/cast_and_validate.ex | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/lib/pleroma/web/api_spec/cast_and_validate.ex b/lib/pleroma/web/api_spec/cast_and_validate.ex
index f3e8e093e..eb487fce7 100644
--- a/lib/pleroma/web/api_spec/cast_and_validate.ex
+++ b/lib/pleroma/web/api_spec/cast_and_validate.ex
@@ -18,6 +18,8 @@ defmodule Pleroma.Web.ApiSpec.CastAndValidate do
   alias OpenApiSpex.Plug.PutApiSpec
   alias Plug.Conn
 
+  require Logger
+
   @impl Plug
   def init(opts) do
     opts
@@ -51,6 +53,10 @@ defmodule Pleroma.Web.ApiSpec.CastAndValidate do
         conn
 
       {:error, reason} ->
+        Logger.error(
+          "Strict ApiSpec: request denied to #{conn.path_info} with params #{inspect(conn.params)}"
+        )
+
         opts = render_error.init(reason)
 
         conn

From 6ded017dee983e22890d639c1fbd12b44e04e964 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Tue, 4 Jun 2024 13:31:51 -0400
Subject: [PATCH 167/192] Do not start unused ConcurrentLimiter processes

---
 lib/pleroma/application.ex | 2 --
 1 file changed, 2 deletions(-)

diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex
index 0d9757b44..d98293e8e 100644
--- a/lib/pleroma/application.ex
+++ b/lib/pleroma/application.ex
@@ -292,8 +292,6 @@ defmodule Pleroma.Application do
     config = Config.get(ConcurrentLimiter, [])
 
     [
-      Pleroma.Web.RichMedia.Helpers,
-      Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy,
       Pleroma.Search
     ]
     |> Enum.each(fn module ->

From 5ed07aba7ad33dec57a16f6a6ecd17fa0538a3d2 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Tue, 4 Jun 2024 13:28:00 -0400
Subject: [PATCH 168/192] Add missing pool to the type

---
 lib/pleroma/http/adapter_helper/gun.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pleroma/http/adapter_helper/gun.ex b/lib/pleroma/http/adapter_helper/gun.ex
index 74ab9851e..1fe8dd4b2 100644
--- a/lib/pleroma/http/adapter_helper/gun.ex
+++ b/lib/pleroma/http/adapter_helper/gun.ex
@@ -15,7 +15,7 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do
     retry_timeout: 1_000
   ]
 
-  @type pool() :: :federation | :upload | :media | :default
+  @type pool() :: :federation | :upload | :media | :rich_media | :default
 
   @spec options(keyword(), URI.t()) :: keyword()
   def options(incoming_opts \\ [], %URI{} = uri) do

From d98b1c2757409bf4b96e126e96ec8fd114a723d4 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Fri, 7 Jun 2024 16:35:26 -0400
Subject: [PATCH 169/192] Small cleanup / fixes

---
 changelog.d/cleanup.skip | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 changelog.d/cleanup.skip

diff --git a/changelog.d/cleanup.skip b/changelog.d/cleanup.skip
new file mode 100644
index 000000000..e69de29bb

From dbf29cbae482d2ab7da65cd8ade0b0a6bb915c02 Mon Sep 17 00:00:00 2001
From: Pleroma User <keine@partyheld.de>
Date: Sat, 8 Jun 2024 01:39:08 +0000
Subject: [PATCH 170/192] Bump deps

---
 changelog.d/deps-bump-2024-06-07.skip         |  2 ++
 mix.exs                                       | 15 +++++---------
 mix.lock                                      | 20 +++++++++----------
 ...0240608003957_upgrade_oban_jobs_to_v12.exs |  7 +++++++
 4 files changed, 24 insertions(+), 20 deletions(-)
 create mode 100644 changelog.d/deps-bump-2024-06-07.skip
 create mode 100644 priv/repo/migrations/20240608003957_upgrade_oban_jobs_to_v12.exs

diff --git a/changelog.d/deps-bump-2024-06-07.skip b/changelog.d/deps-bump-2024-06-07.skip
new file mode 100644
index 000000000..4f377a4d7
--- /dev/null
+++ b/changelog.d/deps-bump-2024-06-07.skip
@@ -0,0 +1,2 @@
+Update dependencies held back due to old Elixir version
+
diff --git a/mix.exs b/mix.exs
index a44daab8b..4de32f86e 100644
--- a/mix.exs
+++ b/mix.exs
@@ -120,15 +120,13 @@ defmodule Pleroma.Mixfile do
       {:ecto_enum, "~> 1.4"},
       {:postgrex, ">= 0.0.0"},
       {:phoenix_html, "~> 3.3"},
-      {:phoenix_live_reload, "~> 1.3.3", only: :dev},
       {:phoenix_live_view, "~> 0.19.0"},
       {:phoenix_live_dashboard, "~> 0.8.0"},
       {:telemetry_metrics, "~> 0.6"},
       {:telemetry_poller, "~> 1.0"},
       {:tzdata, "~> 1.0.3"},
       {:plug_cowboy, "~> 2.5"},
-      # oban 2.14 requires Elixir 1.12+
-      {:oban, "~> 2.13.4"},
+      {:oban, "~> 2.17.9"},
       {:gettext, "~> 0.20"},
       {:bcrypt_elixir, "~> 2.2"},
       {:trailing_format_plug, "~> 0.0.7"},
@@ -147,13 +145,11 @@ defmodule Pleroma.Mixfile do
       {:ex_aws, "~> 2.1.6"},
       {:ex_aws_s3, "~> 2.0"},
       {:sweet_xml, "~> 0.7.2"},
-      # earmark 1.4.23 requires Elixir 1.12+
-      {:earmark, "1.4.22"},
+      {:earmark, "1.4.46"},
       {:bbcode_pleroma, "~> 0.2.0"},
       {:cors_plug, "~> 2.0"},
       {:web_push_encryption, "~> 0.3.1"},
-      # swoosh 1.11.2+ requires Elixir 1.12+
-      {:swoosh, "~> 1.10.0"},
+      {:swoosh, "~> 1.16.9"},
       {:phoenix_swoosh, "~> 1.1"},
       {:gen_smtp, "~> 0.13"},
       {:ex_syslogger, "~> 1.4"},
@@ -185,13 +181,12 @@ defmodule Pleroma.Mixfile do
       {:vix, "~> 0.26.0"},
       {:elixir_make, "~> 0.7.7", override: true},
       {:blurhash, "~> 0.1.0", hex: :rinpatch_blurhash},
-      {:exile,
-       git: "https://github.com/akash-akya/exile.git",
-       ref: "be87c33b02a7c3c5d22d2ece01fbd462355b28ef"},
+      {:exile, "~> 0.10.0"},
       {:bandit, "~> 1.5.2"},
       {:websock_adapter, "~> 0.5.6"},
 
       ## dev & test
+      {:phoenix_live_reload, "~> 1.3.3", only: :dev},
       {:ex_doc, "~> 0.22", only: :dev, runtime: false},
       {:ex_machina, "~> 2.4", only: :test},
       {:credo, "~> 1.6", only: [:dev, :test], runtime: false},
diff --git a/mix.lock b/mix.lock
index a55ad0126..164bc83c3 100644
--- a/mix.lock
+++ b/mix.lock
@@ -29,13 +29,13 @@
   "decimal": {:hex, :decimal, "2.1.1", "5611dca5d4b2c3dd497dec8f68751f1f1a54755e8ed2a966c2633cf885973ad6", [:mix], [], "hexpm", "53cfe5f497ed0e7771ae1a475575603d77425099ba5faef9394932b35020ffcc"},
   "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm", "ce708e5f094b9cd4e8f2be4f00d2f4250c4095be93f8cd6d018c753894885430"},
   "dialyxir": {:hex, :dialyxir, "1.4.3", "edd0124f358f0b9e95bfe53a9fcf806d615d8f838e2202a9f430d59566b6b53b", [:mix], [{:erlex, ">= 0.2.6", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "bf2cfb75cd5c5006bec30141b131663299c661a864ec7fbbc72dfa557487a986"},
-  "earmark": {:hex, :earmark, "1.4.22", "ea3e45c6359446dc308be0a64ce82a03260d973de7d0625a762e6d352ff57958", [:mix], [{:earmark_parser, "~> 1.4.23", [hex: :earmark_parser, repo: "hexpm", optional: false]}], "hexpm", "1caf5145665a42fd76d5317286b0c171861fb1c04f86ab103dde76868814fdfb"},
+  "earmark": {:hex, :earmark, "1.4.46", "8c7287bd3137e99d26ae4643e5b7ef2129a260e3dcf41f251750cb4563c8fb81", [:mix], [], "hexpm", "798d86db3d79964e759ddc0c077d5eb254968ed426399fbf5a62de2b5ff8910a"},
   "earmark_parser": {:hex, :earmark_parser, "1.4.39", "424642f8335b05bb9eb611aa1564c148a8ee35c9c8a8bba6e129d51a3e3c6769", [:mix], [], "hexpm", "06553a88d1f1846da9ef066b87b57c6f605552cfbe40d20bd8d59cc6bde41944"},
   "eblurhash": {:git, "https://github.com/zotonic/eblurhash.git", "bc37ceb426ef021ee9927fb249bb93f7059194ab", [ref: "bc37ceb426ef021ee9927fb249bb93f7059194ab"]},
-  "ecto": {:hex, :ecto, "3.11.1", "4b4972b717e7ca83d30121b12998f5fcdc62ba0ed4f20fd390f16f3270d85c3e", [:mix], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "ebd3d3772cd0dfcd8d772659e41ed527c28b2a8bde4b00fe03e0463da0f1983b"},
+  "ecto": {:hex, :ecto, "3.11.2", "e1d26be989db350a633667c5cda9c3d115ae779b66da567c68c80cfb26a8c9ee", [:mix], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "3c38bca2c6f8d8023f2145326cc8a80100c3ffe4dcbd9842ff867f7fc6156c65"},
   "ecto_enum": {:hex, :ecto_enum, "1.4.0", "d14b00e04b974afc69c251632d1e49594d899067ee2b376277efd8233027aec8", [:mix], [{:ecto, ">= 3.0.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:ecto_sql, "> 3.0.0", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:mariaex, ">= 0.0.0", [hex: :mariaex, repo: "hexpm", optional: true]}, {:postgrex, ">= 0.0.0", [hex: :postgrex, repo: "hexpm", optional: true]}], "hexpm", "8fb55c087181c2b15eee406519dc22578fa60dd82c088be376d0010172764ee4"},
   "ecto_psql_extras": {:hex, :ecto_psql_extras, "0.7.15", "0fc29dbae0e444a29bd6abeee4cf3c4c037e692a272478a234a1cc765077dbb1", [:mix], [{:ecto_sql, "~> 3.7", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.16.0 or ~> 0.17.0", [hex: :postgrex, repo: "hexpm", optional: false]}, {:table_rex, "~> 3.1.1 or ~> 4.0.0", [hex: :table_rex, repo: "hexpm", optional: false]}], "hexpm", "b6127f3a5c6fc3d84895e4768cc7c199f22b48b67d6c99b13fbf4a374e73f039"},
-  "ecto_sql": {:hex, :ecto_sql, "3.11.1", "e9abf28ae27ef3916b43545f9578b4750956ccea444853606472089e7d169470", [:mix], [{:db_connection, "~> 2.5 or ~> 2.4.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.11.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.6.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.16.0 or ~> 0.17.0 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1 or ~> 2.2", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "ce14063ab3514424276e7e360108ad6c2308f6d88164a076aac8a387e1fea634"},
+  "ecto_sql": {:hex, :ecto_sql, "3.11.2", "c7cc7f812af571e50b80294dc2e535821b3b795ce8008d07aa5f336591a185a8", [:mix], [{:db_connection, "~> 2.4.1 or ~> 2.5", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.11.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.6.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.16 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1 or ~> 2.2", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "73c07f995ac17dbf89d3cfaaf688fcefabcd18b7b004ac63b0dc4ef39499ed6b"},
   "eimp": {:hex, :eimp, "1.0.14", "fc297f0c7e2700457a95a60c7010a5f1dcb768a083b6d53f49cd94ab95a28f22", [:rebar3], [{:p1_utils, "1.0.18", [hex: :p1_utils, repo: "hexpm", optional: false]}], "hexpm", "501133f3112079b92d9e22da8b88bf4f0e13d4d67ae9c15c42c30bd25ceb83b6"},
   "elixir_make": {:hex, :elixir_make, "0.7.8", "505026f266552ee5aabca0b9f9c229cbb496c689537c9f922f3eb5431157efc7", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:certifi, "~> 2.0", [hex: :certifi, repo: "hexpm", optional: true]}], "hexpm", "7a71945b913d37ea89b06966e1342c85cfe549b15e6d6d081e8081c493062c07"},
   "erlex": {:hex, :erlex, "0.2.6", "c7987d15e899c7a2f34f5420d2a2ea0d659682c06ac607572df55a43753aa12e", [:mix], [], "hexpm", "2ed2e25711feb44d52b17d2780eabf998452f6efda104877a3881c2f8c0c0c75"},
@@ -47,7 +47,7 @@
   "ex_doc": {:hex, :ex_doc, "0.31.1", "8a2355ac42b1cc7b2379da9e40243f2670143721dd50748bf6c3b1184dae2089", [:mix], [{:earmark_parser, "~> 1.4.39", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.1", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1", [hex: :makeup_erlang, repo: "hexpm", optional: false]}], "hexpm", "3178c3a407c557d8343479e1ff117a96fd31bafe52a039079593fb0524ef61b0"},
   "ex_machina": {:hex, :ex_machina, "2.7.0", "b792cc3127fd0680fecdb6299235b4727a4944a09ff0fa904cc639272cd92dc7", [:mix], [{:ecto, "~> 2.2 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:ecto_sql, "~> 3.0", [hex: :ecto_sql, repo: "hexpm", optional: true]}], "hexpm", "419aa7a39bde11894c87a615c4ecaa52d8f107bbdd81d810465186f783245bf8"},
   "ex_syslogger": {:hex, :ex_syslogger, "1.5.2", "72b6aa2d47a236e999171f2e1ec18698740f40af0bd02c8c650bf5f1fd1bac79", [:mix], [{:poison, ">= 1.5.0", [hex: :poison, repo: "hexpm", optional: true]}, {:syslog, "~> 1.1.0", [hex: :syslog, repo: "hexpm", optional: false]}], "hexpm", "ab9fab4136dbc62651ec6f16fa4842f10cf02ab4433fa3d0976c01be99398399"},
-  "exile": {:git, "https://github.com/akash-akya/exile.git", "be87c33b02a7c3c5d22d2ece01fbd462355b28ef", [ref: "be87c33b02a7c3c5d22d2ece01fbd462355b28ef"]},
+  "exile": {:hex, :exile, "0.10.0", "b69e2d27a9af670b0f0a0898addca0eda78f6f5ba95ccfbc9bc6ccdd04925436", [:make, :mix], [{:elixir_make, "~> 0.6", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm", "c62ee8fee565b5ac4a898d0dcd58d2b04fb5eec1655af1ddcc9eb582c6732c33"},
   "expo": {:hex, :expo, "0.5.1", "249e826a897cac48f591deba863b26c16682b43711dd15ee86b92f25eafd96d9", [:mix], [], "hexpm", "68a4233b0658a3d12ee00d27d37d856b1ba48607e7ce20fd376958d0ba6ce92b"},
   "fast_html": {:hex, :fast_html, "2.2.0", "6c5ef1be087a4ed613b0379c13f815c4d11742b36b67bb52cee7859847c84520", [:make, :mix], [{:elixir_make, "~> 0.4", [hex: :elixir_make, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 0.2.0", [hex: :nimble_pool, repo: "hexpm", optional: false]}], "hexpm", "064c4f23b4a6168f9187dac8984b056f2c531bb0787f559fd6a8b34b38aefbae"},
   "fast_sanitize": {:hex, :fast_sanitize, "0.2.3", "67b93dfb34e302bef49fec3aaab74951e0f0602fd9fa99085987af05bd91c7a5", [:mix], [{:fast_html, "~> 2.0", [hex: :fast_html, repo: "hexpm", optional: false]}, {:plug, "~> 1.8", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "e8ad286d10d0386e15d67d0ee125245ebcfbc7d7290b08712ba9013c8c5e56e2"},
@@ -77,17 +77,17 @@
   "meck": {:hex, :meck, "0.9.2", "85ccbab053f1db86c7ca240e9fc718170ee5bda03810a6292b5306bf31bae5f5", [:rebar3], [], "hexpm", "81344f561357dc40a8344afa53767c32669153355b626ea9fcbc8da6b3045826"},
   "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"},
   "mime": {:hex, :mime, "1.6.0", "dabde576a497cef4bbdd60aceee8160e02a6c89250d6c0b29e56c0dfb00db3d2", [:mix], [], "hexpm", "31a1a8613f8321143dde1dafc36006a17d28d02bdfecb9e95a880fa7aabd19a7"},
-  "mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm", "f278585650aa581986264638ebf698f8bb19df297f66ad91b18910dfc6e19323"},
-  "mint": {:hex, :mint, "1.5.2", "4805e059f96028948870d23d7783613b7e6b0e2fb4e98d720383852a760067fd", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "d77d9e9ce4eb35941907f1d3df38d8f750c357865353e21d335bdcdf6d892a02"},
+  "mimerl": {:hex, :mimerl, "1.3.0", "d0cd9fc04b9061f82490f6581e0128379830e78535e017f7780f37fea7545726", [:rebar3], [], "hexpm", "a1e15a50d1887217de95f0b9b0793e32853f7c258a5cd227650889b38839fe9d"},
+  "mint": {:hex, :mint, "1.6.0", "88a4f91cd690508a04ff1c3e28952f322528934be541844d54e0ceb765f01d5e", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1 or ~> 0.2.0", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "3c5ae85d90a5aca0a49c0d8b67360bbe407f3b54f1030a111047ff988e8fefaa"},
   "mochiweb": {:hex, :mochiweb, "2.18.0", "eb55f1db3e6e960fac4e6db4e2db9ec3602cc9f30b86cd1481d56545c3145d2e", [:rebar3], [], "hexpm"},
   "mock": {:hex, :mock, "0.3.8", "7046a306b71db2488ef54395eeb74df0a7f335a7caca4a3d3875d1fc81c884dd", [:mix], [{:meck, "~> 0.9.2", [hex: :meck, repo: "hexpm", optional: false]}], "hexpm", "7fa82364c97617d79bb7d15571193fc0c4fe5afd0c932cef09426b3ee6fe2022"},
   "mogrify": {:hex, :mogrify, "0.8.0", "3506f3ca3f7b95a155f3b4ef803b5db176f5a0633723e3fe85e0d6399e3b11c8", [:mix], [], "hexpm", "2278d245f07056ea3b586e98801e933695147066fa4cf563f552c1b4f0ff8ad9"},
   "mox": {:hex, :mox, "1.1.0", "0f5e399649ce9ab7602f72e718305c0f9cdc351190f72844599545e4996af73c", [:mix], [], "hexpm", "d44474c50be02d5b72131070281a5d3895c0e7a95c780e90bc0cfe712f633a13"},
-  "nimble_options": {:hex, :nimble_options, "1.1.0", "3b31a57ede9cb1502071fade751ab0c7b8dbe75a9a4c2b5bbb0943a690b63172", [:mix], [], "hexpm", "8bbbb3941af3ca9acc7835f5655ea062111c9c27bcac53e004460dfd19008a99"},
+  "nimble_options": {:hex, :nimble_options, "1.1.1", "e3a492d54d85fc3fd7c5baf411d9d2852922f66e69476317787a7b2bb000a61b", [:mix], [], "hexpm", "821b2470ca9442c4b6984882fe9bb0389371b8ddec4d45a9504f00a66f650b44"},
   "nimble_parsec": {:hex, :nimble_parsec, "0.6.0", "32111b3bf39137144abd7ba1cce0914533b2d16ef35e8abc5ec8be6122944263", [:mix], [], "hexpm", "27eac315a94909d4dc68bc07a4a83e06c8379237c5ea528a9acff4ca1c873c52"},
   "nimble_pool": {:hex, :nimble_pool, "0.2.6", "91f2f4c357da4c4a0a548286c84a3a28004f68f05609b4534526871a22053cde", [:mix], [], "hexpm", "1c715055095d3f2705c4e236c18b618420a35490da94149ff8b580a2144f653f"},
   "nodex": {:git, "https://git.pleroma.social/pleroma/nodex", "cb6730f943cfc6aad674c92161be23a8411f15d1", [ref: "cb6730f943cfc6aad674c92161be23a8411f15d1"]},
-  "oban": {:hex, :oban, "2.13.6", "a0cb1bce3bd393770512231fb5a3695fa19fd3af10d7575bf73f837aee7abf43", [:mix], [{:ecto_sql, "~> 3.6", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.16", [hex: :postgrex, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "3c1c5eb16f377b3cbbf2ea14be24d20e3d91285af9d1ac86260b7c2af5464887"},
+  "oban": {:hex, :oban, "2.17.10", "c3e5bd739b5c3fdc38eba1d43ab270a8c6ca4463bb779b7705c69400b0d87678", [:mix], [{:ecto_sql, "~> 3.10", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:ecto_sqlite3, "~> 0.9", [hex: :ecto_sqlite3, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.16", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "4afd027b8e2bc3c399b54318b4f46ee8c40251fb55a285cb4e38b5363f0ee7c4"},
   "octo_fetch": {:hex, :octo_fetch, "0.4.0", "074b5ecbc08be10b05b27e9db08bc20a3060142769436242702931c418695b19", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~> 1.1", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm", "cf8be6f40cd519d7000bb4e84adcf661c32e59369ca2827c4e20042eda7a7fc6"},
   "open_api_spex": {:hex, :open_api_spex, "3.18.2", "8c855e83bfe8bf81603d919d6e892541eafece3720f34d1700b58024dadde247", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:poison, "~> 3.0 or ~> 4.0 or ~> 5.0", [hex: :poison, repo: "hexpm", optional: true]}, {:ymlr, "~> 2.0 or ~> 3.0 or ~> 4.0", [hex: :ymlr, repo: "hexpm", optional: true]}], "hexpm", "aa3e6dcfc0ad6a02596b2172662da21c9dd848dac145ea9e603f54e3d81b8d2b"},
   "parse_trans": {:hex, :parse_trans, "3.4.1", "6e6aa8167cb44cc8f39441d05193be6e6f4e7c2946cb2759f015f8c56b76e5ff", [:rebar3], [], "hexpm", "620a406ce75dada827b82e453c19cf06776be266f5a67cff34e1ef2cbb60e49a"},
@@ -108,7 +108,7 @@
   "plug_static_index_html": {:hex, :plug_static_index_html, "1.0.0", "840123d4d3975585133485ea86af73cb2600afd7f2a976f9f5fd8b3808e636a0", [:mix], [{:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "79fd4fcf34d110605c26560cbae8f23c603ec4158c08298bd4360fdea90bb5cf"},
   "poison": {:hex, :poison, "3.1.0", "d9eb636610e096f86f25d9a46f35a9facac35609a7591b3be3326e99a0484665", [:mix], [], "hexpm", "fec8660eb7733ee4117b85f55799fd3833eb769a6df71ccf8903e8dc5447cfce"},
   "poolboy": {:hex, :poolboy, "1.5.2", "392b007a1693a64540cead79830443abf5762f5d30cf50bc95cb2c1aaafa006b", [:rebar3], [], "hexpm", "dad79704ce5440f3d5a3681c8590b9dc25d1a561e8f5a9c995281012860901e3"},
-  "postgrex": {:hex, :postgrex, "0.17.4", "5777781f80f53b7c431a001c8dad83ee167bcebcf3a793e3906efff680ab62b3", [:mix], [{:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "6458f7d5b70652bc81c3ea759f91736c16a31be000f306d3c64bcdfe9a18b3cc"},
+  "postgrex": {:hex, :postgrex, "0.17.5", "0483d054938a8dc069b21bdd636bf56c487404c241ce6c319c1f43588246b281", [:mix], [{:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "50b8b11afbb2c4095a3ba675b4f055c416d0f3d7de6633a595fc131a828a67eb"},
   "pot": {:hex, :pot, "1.0.2", "13abb849139fdc04ab8154986abbcb63bdee5de6ed2ba7e1713527e33df923dd", [:rebar3], [], "hexpm", "78fe127f5a4f5f919d6ea5a2a671827bd53eb9d37e5b4128c0ad3df99856c2e0"},
   "prom_ex": {:hex, :prom_ex, "1.9.0", "63e6dda6c05cdeec1f26c48443dcc38ffd2118b3665ae8d2bd0e5b79f2aea03e", [:mix], [{:absinthe, ">= 1.6.0", [hex: :absinthe, repo: "hexpm", optional: true]}, {:broadway, ">= 1.0.2", [hex: :broadway, repo: "hexpm", optional: true]}, {:ecto, ">= 3.5.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:finch, "~> 0.15", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.2", [hex: :jason, repo: "hexpm", optional: false]}, {:oban, ">= 2.4.0", [hex: :oban, repo: "hexpm", optional: true]}, {:octo_fetch, "~> 0.3", [hex: :octo_fetch, repo: "hexpm", optional: false]}, {:phoenix, ">= 1.5.0", [hex: :phoenix, repo: "hexpm", optional: true]}, {:phoenix_live_view, ">= 0.14.0", [hex: :phoenix_live_view, repo: "hexpm", optional: true]}, {:plug, ">= 1.12.1", [hex: :plug, repo: "hexpm", optional: true]}, {:plug_cowboy, "~> 2.5 or ~> 2.6", [hex: :plug_cowboy, repo: "hexpm", optional: false]}, {:telemetry, ">= 1.0.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.6", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}, {:telemetry_metrics_prometheus_core, "~> 1.0", [hex: :telemetry_metrics_prometheus_core, repo: "hexpm", optional: false]}, {:telemetry_poller, "~> 1.0", [hex: :telemetry_poller, repo: "hexpm", optional: false]}], "hexpm", "01f3d4f69ec93068219e686cc65e58a29c42bea5429a8ff4e2121f19db178ee6"},
   "prometheus": {:hex, :prometheus, "4.10.0", "792adbf0130ff61b5fa8826f013772af24b6e57b984445c8d602c8a0355704a1", [:mix, :rebar3], [{:quantile_estimator, "~> 0.2.1", [hex: :quantile_estimator, repo: "hexpm", optional: false]}], "hexpm", "2a99bb6dce85e238c7236fde6b0064f9834dc420ddbd962aac4ea2a3c3d59384"},
@@ -126,7 +126,7 @@
   "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.7", "354c321cf377240c7b8716899e182ce4890c5938111a1296add3ec74cf1715df", [:make, :mix, :rebar3], [], "hexpm", "fe4c190e8f37401d30167c8c405eda19469f34577987c76dde613e838bbc67f8"},
   "statistex": {:hex, :statistex, "1.0.0", "f3dc93f3c0c6c92e5f291704cf62b99b553253d7969e9a5fa713e5481cd858a5", [:mix], [], "hexpm", "ff9d8bee7035028ab4742ff52fc80a2aa35cece833cf5319009b52f1b5a86c27"},
   "sweet_xml": {:hex, :sweet_xml, "0.7.4", "a8b7e1ce7ecd775c7e8a65d501bc2cd933bff3a9c41ab763f5105688ef485d08", [:mix], [], "hexpm", "e7c4b0bdbf460c928234951def54fe87edf1a170f6896675443279e2dbeba167"},
-  "swoosh": {:hex, :swoosh, "1.10.3", "32f1531ee3fe4e82da8175c597bf3692938f8152eb981e0cbf57107b6c5924c1", [:mix], [{:cowboy, "~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:ex_aws, "~> 2.1", [hex: :ex_aws, repo: "hexpm", optional: true]}, {:finch, "~> 0.6", [hex: :finch, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13 or ~> 1.0", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.2 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "8b7167d93047bac6e1a1c367bf7d899cf2e4fea0592ee04a70673548ef6091b9"},
+  "swoosh": {:hex, :swoosh, "1.16.9", "20c6a32ea49136a4c19f538e27739bb5070558c0fa76b8a95f4d5d5ca7d319a1", [:mix], [{:bandit, ">= 1.0.0", [hex: :bandit, repo: "hexpm", optional: true]}, {:cowboy, "~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:ex_aws, "~> 2.1", [hex: :ex_aws, repo: "hexpm", optional: true]}, {:finch, "~> 0.6", [hex: :finch, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13 or ~> 1.0", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mua, "~> 0.2.0", [hex: :mua, repo: "hexpm", optional: true]}, {:multipart, "~> 0.4", [hex: :multipart, repo: "hexpm", optional: true]}, {:plug, "~> 1.9", [hex: :plug, repo: "hexpm", optional: true]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:req, "~> 0.5 or ~> 1.0", [hex: :req, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.2 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "878b1a7a6c10ebbf725a3349363f48f79c5e3d792eb621643b0d276a38acc0a6"},
   "syslog": {:hex, :syslog, "1.1.0", "6419a232bea84f07b56dc575225007ffe34d9fdc91abe6f1b2f254fd71d8efc2", [:rebar3], [], "hexpm", "4c6a41373c7e20587be33ef841d3de6f3beba08519809329ecc4d27b15b659e1"},
   "table_rex": {:hex, :table_rex, "4.0.0", "3c613a68ebdc6d4d1e731bc973c233500974ec3993c99fcdabb210407b90959b", [:mix], [], "hexpm", "c35c4d5612ca49ebb0344ea10387da4d2afe278387d4019e4d8111e815df8f55"},
   "telemetry": {:hex, :telemetry, "1.0.0", "0f453a102cdf13d506b7c0ab158324c337c41f1cc7548f0bc0e130bbf0ae9452", [:rebar3], [], "hexpm", "73bc09fa59b4a0284efb4624335583c528e07ec9ae76aca96ea0673850aec57a"},
diff --git a/priv/repo/migrations/20240608003957_upgrade_oban_jobs_to_v12.exs b/priv/repo/migrations/20240608003957_upgrade_oban_jobs_to_v12.exs
new file mode 100644
index 000000000..ed5bf4ebb
--- /dev/null
+++ b/priv/repo/migrations/20240608003957_upgrade_oban_jobs_to_v12.exs
@@ -0,0 +1,7 @@
+defmodule Pleroma.Repo.Migrations.UpgradeObanJobsToV12 do
+  use Ecto.Migration
+
+  def up, do: Oban.Migrations.up(version: 12)
+
+  def down, do: Oban.Migrations.down(version: 12)
+end

From a5424c3681a537e08fa1333ee9749ab22fd3ece0 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Sat, 8 Jun 2024 13:08:06 -0400
Subject: [PATCH 171/192] Test that end of poll notifications are streamed over
 websockets

---
 test/pleroma/notification_test.exs | 27 +++++++++++++++++++++++++--
 1 file changed, 25 insertions(+), 2 deletions(-)

diff --git a/test/pleroma/notification_test.exs b/test/pleroma/notification_test.exs
index 2c582c708..02ae06c63 100644
--- a/test/pleroma/notification_test.exs
+++ b/test/pleroma/notification_test.exs
@@ -5,6 +5,7 @@
 defmodule Pleroma.NotificationTest do
   use Pleroma.DataCase, async: false
 
+  import Mock
   import Pleroma.Factory
 
   alias Pleroma.FollowingRelationship
@@ -183,9 +184,31 @@ defmodule Pleroma.NotificationTest do
     {:ok, _, _} = CommonAPI.vote(user2, question, [0])
     {:ok, _, _} = CommonAPI.vote(user3, question, [1])
 
-    {:ok, notifications} = Notification.create_poll_notifications(activity)
+    with_mocks([
+      {
+        Pleroma.Web.Streamer,
+        [],
+        [
+          stream: fn _, _ -> nil end
+        ]
+      },
+      {
+        Pleroma.Web.Push,
+        [],
+        [
+          send: fn _ -> nil end
+        ]
+      }
+    ]) do
+      {:ok, notifications} = Notification.create_poll_notifications(activity)
 
-    assert [user2.id, user3.id, user1.id] == Enum.map(notifications, & &1.user_id)
+      Enum.each(notifications, fn notification ->
+        assert called(Pleroma.Web.Streamer.stream(["user", "user:notification"], notification))
+        assert called(Pleroma.Web.Push.send(notification))
+      end)
+
+      assert [user2.id, user3.id, user1.id] == Enum.map(notifications, & &1.user_id)
+    end
   end
 
   describe "create_notification" do

From b1c52c30628212f881825a702ba75e6170f82407 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Sat, 8 Jun 2024 12:40:32 -0400
Subject: [PATCH 172/192] Rename Notification.send/1 to Notification.stream/1

Also update other places where we use the term "send" instead of "stream". This should make it clearer that we are streaming these over websockets / web push and not sending an activity.
---
 lib/pleroma/notification.ex                  | 5 +++--
 lib/pleroma/web/activity_pub/activity_pub.ex | 2 +-
 lib/pleroma/web/activity_pub/side_effects.ex | 6 +++---
 3 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex
index f521a2998..de2508b93 100644
--- a/lib/pleroma/notification.ex
+++ b/lib/pleroma/notification.ex
@@ -757,8 +757,9 @@ defmodule Pleroma.Notification do
     |> Repo.update_all(set: [seen: true])
   end
 
-  @spec send(list(Notification.t())) :: :ok
-  def send(notifications) do
+  @doc "Streams a list of notifications over websockets and web push"
+  @spec stream(list(Notification.t())) :: :ok
+  def stream(notifications) do
     Enum.each(notifications, fn notification ->
       Streamer.stream(["user", "user:notification"], notification)
       Push.send(notification)
diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index 1247ae7ce..6c1d6ded9 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -201,7 +201,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   def notify_and_stream(activity) do
     {:ok, notifications} = Notification.create_notifications(activity)
-    Notification.send(notifications)
+    Notification.stream(notifications)
 
     original_activity =
       case activity do
diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex
index 60b4d5f1b..2a141b0f5 100644
--- a/lib/pleroma/web/activity_pub/side_effects.ex
+++ b/lib/pleroma/web/activity_pub/side_effects.ex
@@ -592,9 +592,9 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
     with {:ok, _} <- Repo.delete(object), do: :ok
   end
 
-  defp send_notifications(meta) do
+  defp stream_notifications(meta) do
     Keyword.get(meta, :notifications, [])
-    |> Notification.send()
+    |> Notification.stream()
 
     meta
   end
@@ -625,7 +625,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
   @impl true
   def handle_after_transaction(meta) do
     meta
-    |> send_notifications()
+    |> stream_notifications()
     |> send_streamables()
   end
 end

From 471412ad34e87106641101c9e9e44fb1df2ca905 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Tue, 4 Jun 2024 14:39:03 -0400
Subject: [PATCH 173/192] Stream end of poll notification over websockets and
 web push

---
 changelog.d/stream-end-poll.fix | 1 +
 lib/pleroma/notification.ex     | 2 ++
 2 files changed, 3 insertions(+)
 create mode 100644 changelog.d/stream-end-poll.fix

diff --git a/changelog.d/stream-end-poll.fix b/changelog.d/stream-end-poll.fix
new file mode 100644
index 000000000..db513efdc
--- /dev/null
+++ b/changelog.d/stream-end-poll.fix
@@ -0,0 +1 @@
+End of poll notifications were not streamed over websockets or web push
diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex
index de2508b93..b9694a353 100644
--- a/lib/pleroma/notification.ex
+++ b/lib/pleroma/notification.ex
@@ -479,6 +479,8 @@ defmodule Pleroma.Notification do
           end
         end)
 
+      stream(notifications)
+
       {:ok, notifications}
     end
   end

From 84319dbca4946cf51d51797252a96ab2a2385efb Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Thu, 6 Jun 2024 20:37:26 +0000
Subject: [PATCH 174/192] OTP updates

Update to otp25 for Elixir 1.13; otp26 for Elixir 1.15
---
 .gitlab-ci.yml                                       | 12 ++++++------
 changelog.d/ci-otp-update.skip                       |  0
 ci/{elixir-1.13 => elixir-1.13.4-otp-25}/Dockerfile  |  2 +-
 .../build_and_push.sh                                |  2 +-
 ci/elixir-1.15-otp25/build_and_push.sh               |  1 -
 .../Dockerfile                                       |  2 +-
 ci/elixir-1.15.8-otp-26/build_and_push.sh            |  1 +
 7 files changed, 10 insertions(+), 10 deletions(-)
 create mode 100644 changelog.d/ci-otp-update.skip
 rename ci/{elixir-1.13 => elixir-1.13.4-otp-25}/Dockerfile (91%)
 rename ci/{elixir-1.13 => elixir-1.13.4-otp-25}/build_and_push.sh (52%)
 delete mode 100755 ci/elixir-1.15-otp25/build_and_push.sh
 rename ci/{elixir-1.15-otp25 => elixir-1.15.8-otp-26}/Dockerfile (91%)
 create mode 100755 ci/elixir-1.15.8-otp-26/build_and_push.sh

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 21d7b2242..09e3de1d9 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,4 +1,4 @@
-image: git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.13.4-otp-24
+image: git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.13.4-otp-25
 
 variables: &global_variables
   # Only used for the release
@@ -72,7 +72,7 @@ check-changelog:
   tags:
     - amd64
 
-build-1.13.4:
+build-1.13.4-otp-25:
   extends:
   - .build_changes_policy
   - .using-ci-base
@@ -80,12 +80,12 @@ build-1.13.4:
   script:
   - mix compile --force
 
-build-1.15.7-otp-25:
+build-1.15.8-otp-26:
   extends:
   - .build_changes_policy
   - .using-ci-base
   stage: build
-  image: git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.15-otp25
+  image: git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.15.8-otp-26
   allow_failure: true
   script:
   - mix compile --force
@@ -144,12 +144,12 @@ unit-testing-1.12.3:
         coverage_format: cobertura
         path: coverage.xml
 
-unit-testing-1.15.7-otp-25:
+unit-testing-1.15.8-otp-26:
   extends:
   - .build_changes_policy
   - .using-ci-base
   stage: test
-  image: git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.15-otp25
+  image: git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.15.8-otp-26
   allow_failure: true
   cache: *testing_cache_policy
   services: *testing_services
diff --git a/changelog.d/ci-otp-update.skip b/changelog.d/ci-otp-update.skip
new file mode 100644
index 000000000..e69de29bb
diff --git a/ci/elixir-1.13/Dockerfile b/ci/elixir-1.13.4-otp-25/Dockerfile
similarity index 91%
rename from ci/elixir-1.13/Dockerfile
rename to ci/elixir-1.13.4-otp-25/Dockerfile
index b8bceb3d9..25a1639e8 100644
--- a/ci/elixir-1.13/Dockerfile
+++ b/ci/elixir-1.13.4-otp-25/Dockerfile
@@ -1,4 +1,4 @@
-FROM elixir:1.13.4-otp-24
+FROM elixir:1.13.4-otp-25
 
 # Single RUN statement, otherwise intermediate images are created
 # https://docs.docker.com/develop/develop-images/dockerfile_best-practices/#run
diff --git a/ci/elixir-1.13/build_and_push.sh b/ci/elixir-1.13.4-otp-25/build_and_push.sh
similarity index 52%
rename from ci/elixir-1.13/build_and_push.sh
rename to ci/elixir-1.13.4-otp-25/build_and_push.sh
index 64e1856db..b8ca1d24d 100755
--- a/ci/elixir-1.13/build_and_push.sh
+++ b/ci/elixir-1.13.4-otp-25/build_and_push.sh
@@ -1 +1 @@
-docker buildx build --platform linux/amd64,linux/arm64 -t git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.13.4-otp-24 --push .
+docker buildx build --platform linux/amd64,linux/arm64 -t git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.13.4-otp-25 --push .
diff --git a/ci/elixir-1.15-otp25/build_and_push.sh b/ci/elixir-1.15-otp25/build_and_push.sh
deleted file mode 100755
index a28e0d33c..000000000
--- a/ci/elixir-1.15-otp25/build_and_push.sh
+++ /dev/null
@@ -1 +0,0 @@
-docker buildx build --platform linux/amd64,linux/arm64 -t git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.15-otp25 --push .
diff --git a/ci/elixir-1.15-otp25/Dockerfile b/ci/elixir-1.15.8-otp-26/Dockerfile
similarity index 91%
rename from ci/elixir-1.15-otp25/Dockerfile
rename to ci/elixir-1.15.8-otp-26/Dockerfile
index 3335c6e36..7142ace04 100644
--- a/ci/elixir-1.15-otp25/Dockerfile
+++ b/ci/elixir-1.15.8-otp-26/Dockerfile
@@ -1,4 +1,4 @@
-FROM elixir:1.15.7-otp-25
+FROM elixir:1.15.8-otp-26
 
 # Single RUN statement, otherwise intermediate images are created
 # https://docs.docker.com/develop/develop-images/dockerfile_best-practices/#run
diff --git a/ci/elixir-1.15.8-otp-26/build_and_push.sh b/ci/elixir-1.15.8-otp-26/build_and_push.sh
new file mode 100755
index 000000000..59bc63454
--- /dev/null
+++ b/ci/elixir-1.15.8-otp-26/build_and_push.sh
@@ -0,0 +1 @@
+docker buildx build --platform linux/amd64,linux/arm64 -t git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.15.8-otp-26 --push .

From 0641a1058a4fa5a7d12c6b8ae041036a3fbeccf5 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Sat, 8 Jun 2024 15:18:46 -0400
Subject: [PATCH 175/192] Update job names

---
 .gitlab-ci.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 09e3de1d9..45dcd6091 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -121,7 +121,7 @@ benchmark:
     - mix ecto.migrate
     - mix pleroma.load_testing
 
-unit-testing-1.12.3:
+unit-testing-1.13.4-otp-25:
   extends:
   - .build_changes_policy
   - .using-ci-base
@@ -155,7 +155,7 @@ unit-testing-1.15.8-otp-26:
   services: *testing_services
   script: *testing_script
 
-unit-testing-1.12-erratic:
+unit-testing-1.13.4-otp-25-erratic:
   extends:
   - .build_changes_policy
   - .using-ci-base

From 07cb89823f2a33bc540fe53bd784e4b8e9197506 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Sat, 8 Jun 2024 18:59:51 -0400
Subject: [PATCH 176/192] More robust validation the vapid config is set

---
 lib/pleroma/web/push.ex | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)

diff --git a/lib/pleroma/web/push.ex b/lib/pleroma/web/push.ex
index 0d43f402e..af895467b 100644
--- a/lib/pleroma/web/push.ex
+++ b/lib/pleroma/web/push.ex
@@ -20,16 +20,10 @@ defmodule Pleroma.Web.Push do
   end
 
   def vapid_config do
-    Application.get_env(:web_push_encryption, :vapid_details, [])
+    Application.get_env(:web_push_encryption, :vapid_details, nil)
   end
 
-  def enabled do
-    case vapid_config() do
-      [] -> false
-      list when is_list(list) -> true
-      _ -> false
-    end
-  end
+  def enabled, do: match?([subject: _, public_key: _, private_key: _], vapid_config())
 
   def send(notification) do
     WebPusherWorker.enqueue("web_push", %{"notification_id" => notification.id})

From db88bf30d50a546114d3ee22a3c67b53975257d4 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Sat, 8 Jun 2024 19:20:38 -0400
Subject: [PATCH 177/192] Add spec for send/1

---
 lib/pleroma/web/push.ex | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/lib/pleroma/web/push.ex b/lib/pleroma/web/push.ex
index af895467b..d4693f63e 100644
--- a/lib/pleroma/web/push.ex
+++ b/lib/pleroma/web/push.ex
@@ -25,6 +25,8 @@ defmodule Pleroma.Web.Push do
 
   def enabled, do: match?([subject: _, public_key: _, private_key: _], vapid_config())
 
+  @spec send(Pleroma.Notification.t()) ::
+          {:ok, Oban.Job.t()} | {:error, Oban.Job.changeset() | term()}
   def send(notification) do
     WebPusherWorker.enqueue("web_push", %{"notification_id" => notification.id})
   end

From 86fa0889bc9906aced78b60ada932b569743340a Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Sat, 8 Jun 2024 19:30:27 -0400
Subject: [PATCH 178/192] Remove unnecessary mastodon_type hack

---
 lib/pleroma/web/push/impl.ex | 53 ++++++++++++------------------------
 1 file changed, 18 insertions(+), 35 deletions(-)

diff --git a/lib/pleroma/web/push/impl.ex b/lib/pleroma/web/push/impl.ex
index 53334e72c..1853fdc2d 100644
--- a/lib/pleroma/web/push/impl.ex
+++ b/lib/pleroma/web/push/impl.ex
@@ -29,7 +29,6 @@ defmodule Pleroma.Web.Push.Impl do
       when activity_type in @types do
     actor = User.get_cached_by_ap_id(notification.activity.data["actor"])
 
-    mastodon_type = notification.type
     gcm_api_key = Application.get_env(:web_push_encryption, :gcm_api_key)
     avatar_url = User.avatar_url(actor)
     object = Object.normalize(activity, fetch: false)
@@ -37,11 +36,11 @@ defmodule Pleroma.Web.Push.Impl do
     direct_conversation_id = Activity.direct_conversation_id(activity, user)
 
     for subscription <- fetch_subscriptions(user_id),
-        Subscription.enabled?(subscription, mastodon_type) do
+        Subscription.enabled?(subscription, notification.type) do
       %{
         access_token: subscription.token.token,
         notification_id: notification.id,
-        notification_type: mastodon_type,
+        notification_type: notification.type,
         icon: avatar_url,
         preferred_locale: "en",
         pleroma: %{
@@ -49,7 +48,7 @@ defmodule Pleroma.Web.Push.Impl do
           direct_conversation_id: direct_conversation_id
         }
       }
-      |> Map.merge(build_content(notification, actor, object, mastodon_type))
+      |> Map.merge(build_content(notification, actor, object))
       |> Jason.encode!()
       |> push_message(build_sub(subscription), gcm_api_key, subscription)
     end
@@ -106,31 +105,24 @@ defmodule Pleroma.Web.Push.Impl do
     }
   end
 
-  def build_content(notification, actor, object, mastodon_type \\ nil)
-
   def build_content(
         %{
           user: %{notification_settings: %{hide_notification_contents: true}}
         } = notification,
         _actor,
-        _object,
-        mastodon_type
+        _object
       ) do
-    %{body: format_title(notification, mastodon_type)}
+    %{body: format_title(notification)}
   end
 
-  def build_content(notification, actor, object, mastodon_type) do
-    mastodon_type = mastodon_type || notification.type
-
+  def build_content(notification, actor, object) do
     %{
-      title: format_title(notification, mastodon_type),
-      body: format_body(notification, actor, object, mastodon_type)
+      title: format_title(notification),
+      body: format_body(notification, actor, object)
     }
   end
 
-  def format_body(activity, actor, object, mastodon_type \\ nil)
-
-  def format_body(_activity, actor, %{data: %{"type" => "ChatMessage"} = data}, _) do
+  def format_body(_activity, actor, %{data: %{"type" => "ChatMessage"} = data}) do
     case data["content"] do
       nil -> "@#{actor.nickname}: (Attachment)"
       content -> "@#{actor.nickname}: #{Utils.scrub_html_and_truncate(content, 80)}"
@@ -140,8 +132,7 @@ defmodule Pleroma.Web.Push.Impl do
   def format_body(
         %{activity: %{data: %{"type" => "Create"}}},
         actor,
-        %{data: %{"content" => content}},
-        _mastodon_type
+        %{data: %{"content" => content}}
       ) do
     "@#{actor.nickname}: #{Utils.scrub_html_and_truncate(content, 80)}"
   end
@@ -149,8 +140,7 @@ defmodule Pleroma.Web.Push.Impl do
   def format_body(
         %{activity: %{data: %{"type" => "Announce"}}},
         actor,
-        %{data: %{"content" => content}},
-        _mastodon_type
+        %{data: %{"content" => content}}
       ) do
     "@#{actor.nickname} repeated: #{Utils.scrub_html_and_truncate(content, 80)}"
   end
@@ -158,8 +148,7 @@ defmodule Pleroma.Web.Push.Impl do
   def format_body(
         %{activity: %{data: %{"type" => "EmojiReact", "content" => content}}},
         actor,
-        _object,
-        _mastodon_type
+        _object
       ) do
     "@#{actor.nickname} reacted with #{content}"
   end
@@ -167,13 +156,10 @@ defmodule Pleroma.Web.Push.Impl do
   def format_body(
         %{activity: %{data: %{"type" => type}}} = notification,
         actor,
-        _object,
-        mastodon_type
+        _object
       )
       when type in ["Follow", "Like"] do
-    mastodon_type = mastodon_type || notification.type
-
-    case mastodon_type do
+    case notification.type do
       "follow" -> "@#{actor.nickname} has followed you"
       "follow_request" -> "@#{actor.nickname} has requested to follow you"
       "favourite" -> "@#{actor.nickname} has favorited your post"
@@ -183,20 +169,17 @@ defmodule Pleroma.Web.Push.Impl do
   def format_body(
         %{activity: %{data: %{"type" => "Update"}}},
         actor,
-        _object,
-        _mastodon_type
+        _object
       ) do
     "@#{actor.nickname} edited a status"
   end
 
-  def format_title(activity, mastodon_type \\ nil)
-
-  def format_title(%{activity: %{data: %{"directMessage" => true}}}, _mastodon_type) do
+  def format_title(%{activity: %{data: %{"directMessage" => true}}}) do
     "New Direct Message"
   end
 
-  def format_title(%{type: type}, mastodon_type) do
-    case mastodon_type || type do
+  def format_title(%{type: type}) do
+    case type do
       "mention" -> "New Mention"
       "status" -> "New Status"
       "follow" -> "New Follower"

From b1ef6e5e9ad5afc9d556a4d88ff34c9b62f5fea2 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Sat, 8 Jun 2024 19:48:41 -0400
Subject: [PATCH 179/192] Cleanup to make the code easier to follow

---
 lib/pleroma/web/push/impl.ex | 46 +++++++++++++++++++-----------------
 1 file changed, 24 insertions(+), 22 deletions(-)

diff --git a/lib/pleroma/web/push/impl.ex b/lib/pleroma/web/push/impl.ex
index 1853fdc2d..df7e0c53f 100644
--- a/lib/pleroma/web/push/impl.ex
+++ b/lib/pleroma/web/push/impl.ex
@@ -27,10 +27,10 @@ defmodule Pleroma.Web.Push.Impl do
         } = notification
       )
       when activity_type in @types do
-    actor = User.get_cached_by_ap_id(notification.activity.data["actor"])
+    user = User.get_cached_by_ap_id(notification.activity.data["actor"])
 
     gcm_api_key = Application.get_env(:web_push_encryption, :gcm_api_key)
-    avatar_url = User.avatar_url(actor)
+    avatar_url = User.avatar_url(user)
     object = Object.normalize(activity, fetch: false)
     user = User.get_cached_by_id(user_id)
     direct_conversation_id = Activity.direct_conversation_id(activity, user)
@@ -48,7 +48,7 @@ defmodule Pleroma.Web.Push.Impl do
           direct_conversation_id: direct_conversation_id
         }
       }
-      |> Map.merge(build_content(notification, actor, object))
+      |> Map.merge(build_content(notification, user, object))
       |> Jason.encode!()
       |> push_message(build_sub(subscription), gcm_api_key, subscription)
     end
@@ -109,71 +109,73 @@ defmodule Pleroma.Web.Push.Impl do
         %{
           user: %{notification_settings: %{hide_notification_contents: true}}
         } = notification,
-        _actor,
+        _user,
         _object
       ) do
     %{body: format_title(notification)}
   end
 
-  def build_content(notification, actor, object) do
+  def build_content(notification, user, object) do
     %{
       title: format_title(notification),
-      body: format_body(notification, actor, object)
+      body: format_body(notification, user, object)
     }
   end
 
-  def format_body(_activity, actor, %{data: %{"type" => "ChatMessage"} = data}) do
-    case data["content"] do
-      nil -> "@#{actor.nickname}: (Attachment)"
-      content -> "@#{actor.nickname}: #{Utils.scrub_html_and_truncate(content, 80)}"
+  @spec format_body(Notification.t(), User.t(), Object.t()) :: String.t()
+  def format_body(_notification, user, %{data: %{"type" => "ChatMessage"} = object}) do
+    case object["content"] do
+      nil -> "@#{user.nickname}: (Attachment)"
+      content -> "@#{user.nickname}: #{Utils.scrub_html_and_truncate(content, 80)}"
     end
   end
 
   def format_body(
         %{activity: %{data: %{"type" => "Create"}}},
-        actor,
+        user,
         %{data: %{"content" => content}}
       ) do
-    "@#{actor.nickname}: #{Utils.scrub_html_and_truncate(content, 80)}"
+    "@#{user.nickname}: #{Utils.scrub_html_and_truncate(content, 80)}"
   end
 
   def format_body(
         %{activity: %{data: %{"type" => "Announce"}}},
-        actor,
+        user,
         %{data: %{"content" => content}}
       ) do
-    "@#{actor.nickname} repeated: #{Utils.scrub_html_and_truncate(content, 80)}"
+    "@#{user.nickname} repeated: #{Utils.scrub_html_and_truncate(content, 80)}"
   end
 
   def format_body(
         %{activity: %{data: %{"type" => "EmojiReact", "content" => content}}},
-        actor,
+        user,
         _object
       ) do
-    "@#{actor.nickname} reacted with #{content}"
+    "@#{user.nickname} reacted with #{content}"
   end
 
   def format_body(
         %{activity: %{data: %{"type" => type}}} = notification,
-        actor,
+        user,
         _object
       )
       when type in ["Follow", "Like"] do
     case notification.type do
-      "follow" -> "@#{actor.nickname} has followed you"
-      "follow_request" -> "@#{actor.nickname} has requested to follow you"
-      "favourite" -> "@#{actor.nickname} has favorited your post"
+      "follow" -> "@#{user.nickname} has followed you"
+      "follow_request" -> "@#{user.nickname} has requested to follow you"
+      "favourite" -> "@#{user.nickname} has favorited your post"
     end
   end
 
   def format_body(
         %{activity: %{data: %{"type" => "Update"}}},
-        actor,
+        user,
         _object
       ) do
-    "@#{actor.nickname} edited a status"
+    "@#{user.nickname} edited a status"
   end
 
+  @spec format_title(Notification.t()) :: String.t()
   def format_title(%{activity: %{data: %{"directMessage" => true}}}) do
     "New Direct Message"
   end

From 3211557f742e07d5144426a71c39267480656a38 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Sat, 8 Jun 2024 20:30:43 -0400
Subject: [PATCH 180/192] Render nice web push notifications for polls

---
 changelog.d/webpush-polls.change    |  1 +
 lib/pleroma/web/push/impl.ex        | 19 +++++++++++++++++++
 test/pleroma/web/push/impl_test.exs | 21 +++++++++++++++++++++
 test/support/factory.ex             |  1 +
 4 files changed, 42 insertions(+)
 create mode 100644 changelog.d/webpush-polls.change

diff --git a/changelog.d/webpush-polls.change b/changelog.d/webpush-polls.change
new file mode 100644
index 000000000..5607d6bfc
--- /dev/null
+++ b/changelog.d/webpush-polls.change
@@ -0,0 +1 @@
+Render nice web push notifications for polls
diff --git a/lib/pleroma/web/push/impl.ex b/lib/pleroma/web/push/impl.ex
index df7e0c53f..98e7659ab 100644
--- a/lib/pleroma/web/push/impl.ex
+++ b/lib/pleroma/web/push/impl.ex
@@ -130,6 +130,24 @@ defmodule Pleroma.Web.Push.Impl do
     end
   end
 
+  def format_body(
+        %{type: "poll"} = _notification,
+        _user,
+        %{data: %{"content" => content} = data} = _object
+      ) do
+    options = Map.get(data, "anyOf") || Map.get(data, "oneOf")
+
+    content_text = content <> "\n"
+
+    options_text =
+      Enum.map(options, fn x -> "○ #{x["name"]}" end)
+      |> Enum.join("\n")
+
+    [content_text, options_text]
+    |> Enum.join("\n")
+    |> Utils.scrub_html_and_truncate(80)
+  end
+
   def format_body(
         %{activity: %{data: %{"type" => "Create"}}},
         user,
@@ -191,6 +209,7 @@ defmodule Pleroma.Web.Push.Impl do
       "update" -> "New Update"
       "pleroma:chat_mention" -> "New Chat Message"
       "pleroma:emoji_reaction" -> "New Reaction"
+      "poll" -> "Poll Results"
       type -> "New #{String.capitalize(type || "event")}"
     end
   end
diff --git a/test/pleroma/web/push/impl_test.exs b/test/pleroma/web/push/impl_test.exs
index 3ceea3d71..e1bbf8d17 100644
--- a/test/pleroma/web/push/impl_test.exs
+++ b/test/pleroma/web/push/impl_test.exs
@@ -232,6 +232,27 @@ defmodule Pleroma.Web.Push.ImplTest do
              "New Direct Message"
   end
 
+  test "renders poll notification" do
+    user = insert(:user)
+    question = insert(:question, user: user)
+    activity = insert(:question_activity, question: question)
+
+    {:ok, [notification]} = Notification.create_poll_notifications(activity)
+
+    assert Impl.format_title(notification) == "Poll Results"
+
+    expected_body =
+      """
+      Which flavor of ice cream do you prefer?
+
+      ○ chocolate
+      ○ vanilla
+      """
+      |> String.trim_trailing("\n")
+
+    assert Impl.format_body(notification, user, question) == expected_body
+  end
+
   describe "build_content/3" do
     test "builds content for chat messages" do
       user = insert(:user)
diff --git a/test/support/factory.ex b/test/support/factory.ex
index 20bc5162e..b248508fa 100644
--- a/test/support/factory.ex
+++ b/test/support/factory.ex
@@ -249,6 +249,7 @@ defmodule Pleroma.Factory do
       "cc" => [user.follower_address],
       "context" => Pleroma.Web.ActivityPub.Utils.generate_context_id(),
       "closed" => DateTime.utc_now() |> DateTime.add(86_400) |> DateTime.to_iso8601(),
+      "content" => "Which flavor of ice cream do you prefer?",
       "oneOf" => [
         %{
           "type" => "Note",

From dcc50da4009f19303dec36e0969f3fca0f690e4f Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Sat, 8 Jun 2024 22:40:08 -0400
Subject: [PATCH 181/192] Stream the notifications as part of the job

---
 lib/pleroma/notification.ex        | 2 --
 lib/pleroma/workers/poll_worker.ex | 5 +++--
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex
index b9694a353..de2508b93 100644
--- a/lib/pleroma/notification.ex
+++ b/lib/pleroma/notification.ex
@@ -479,8 +479,6 @@ defmodule Pleroma.Notification do
           end
         end)
 
-      stream(notifications)
-
       {:ok, notifications}
     end
   end
diff --git a/lib/pleroma/workers/poll_worker.ex b/lib/pleroma/workers/poll_worker.ex
index 70df54193..3fcac9bc3 100644
--- a/lib/pleroma/workers/poll_worker.ex
+++ b/lib/pleroma/workers/poll_worker.ex
@@ -14,8 +14,9 @@ defmodule Pleroma.Workers.PollWorker do
 
   @impl Oban.Worker
   def perform(%Job{args: %{"op" => "poll_end", "activity_id" => activity_id}}) do
-    with %Activity{} = activity <- find_poll_activity(activity_id) do
-      Notification.create_poll_notifications(activity)
+    with %Activity{} = activity <- find_poll_activity(activity_id),
+         {:ok, notifications} <- Notification.create_poll_notifications(activity) do
+      Notification.stream(notifications)
     end
   end
 

From c1b84edefcd8db998a7eabff110f2dabecfc8dde Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Sat, 8 Jun 2024 22:48:38 -0400
Subject: [PATCH 182/192] Increase web push character limit for the body

---
 lib/pleroma/web/push/impl.ex | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/lib/pleroma/web/push/impl.ex b/lib/pleroma/web/push/impl.ex
index 98e7659ab..13c054e05 100644
--- a/lib/pleroma/web/push/impl.ex
+++ b/lib/pleroma/web/push/impl.ex
@@ -16,6 +16,7 @@ defmodule Pleroma.Web.Push.Impl do
   require Logger
   import Ecto.Query
 
+  @body_chars 140
   @types ["Create", "Follow", "Announce", "Like", "Move", "EmojiReact", "Update"]
 
   @doc "Performs sending notifications for user subscriptions"
@@ -126,7 +127,7 @@ defmodule Pleroma.Web.Push.Impl do
   def format_body(_notification, user, %{data: %{"type" => "ChatMessage"} = object}) do
     case object["content"] do
       nil -> "@#{user.nickname}: (Attachment)"
-      content -> "@#{user.nickname}: #{Utils.scrub_html_and_truncate(content, 80)}"
+      content -> "@#{user.nickname}: #{Utils.scrub_html_and_truncate(content, @body_chars)}"
     end
   end
 
@@ -145,7 +146,7 @@ defmodule Pleroma.Web.Push.Impl do
 
     [content_text, options_text]
     |> Enum.join("\n")
-    |> Utils.scrub_html_and_truncate(80)
+    |> Utils.scrub_html_and_truncate(@body_chars)
   end
 
   def format_body(
@@ -153,7 +154,7 @@ defmodule Pleroma.Web.Push.Impl do
         user,
         %{data: %{"content" => content}}
       ) do
-    "@#{user.nickname}: #{Utils.scrub_html_and_truncate(content, 80)}"
+    "@#{user.nickname}: #{Utils.scrub_html_and_truncate(content, @body_chars)}"
   end
 
   def format_body(
@@ -161,7 +162,7 @@ defmodule Pleroma.Web.Push.Impl do
         user,
         %{data: %{"content" => content}}
       ) do
-    "@#{user.nickname} repeated: #{Utils.scrub_html_and_truncate(content, 80)}"
+    "@#{user.nickname} repeated: #{Utils.scrub_html_and_truncate(content, @body_chars)}"
   end
 
   def format_body(

From 8468d78882fb004eb5f64e6e0a0dca1ef3284c93 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Sat, 8 Jun 2024 22:48:38 -0400
Subject: [PATCH 183/192] Increase web push character limit for the body

---
 test/pleroma/web/push/impl_test.exs | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/test/pleroma/web/push/impl_test.exs b/test/pleroma/web/push/impl_test.exs
index e1bbf8d17..969c7a605 100644
--- a/test/pleroma/web/push/impl_test.exs
+++ b/test/pleroma/web/push/impl_test.exs
@@ -41,7 +41,7 @@ defmodule Pleroma.Web.Push.ImplTest do
     }
   }
   @api_key "BASgACIHpN1GYgzSRp"
-  @message "@Bob: Lorem ipsum dolor sit amet, consectetur  adipiscing elit. Fusce sagittis fini..."
+  @message "@Bob: Lorem ipsum dolor sit amet, consectetur  adipiscing elit. Fusce sagittis finibus turpis."
 
   test "performs sending notifications" do
     user = insert(:user)
@@ -129,7 +129,7 @@ defmodule Pleroma.Web.Push.ImplTest do
              user,
              object
            ) ==
-             "@Bob: Lorem ipsum dolor sit amet, consectetur  adipiscing elit. Fusce sagittis fini..."
+             "@Bob: Lorem ipsum dolor sit amet, consectetur  adipiscing elit. Fusce sagittis finibus turpis."
 
     assert Impl.format_title(%{activity: activity, type: "mention"}) ==
              "New Mention"
@@ -161,7 +161,7 @@ defmodule Pleroma.Web.Push.ImplTest do
     object = Object.normalize(activity, fetch: false)
 
     assert Impl.format_body(%{activity: announce_activity}, user, object) ==
-             "@#{user.nickname} repeated: Lorem ipsum dolor sit amet, consectetur  adipiscing elit. Fusce sagittis fini..."
+             "@#{user.nickname} repeated: Lorem ipsum dolor sit amet, consectetur  adipiscing elit. Fusce sagittis finibus turpis."
 
     assert Impl.format_title(%{activity: announce_activity, type: "reblog"}) ==
              "New Repeat"
@@ -365,7 +365,7 @@ defmodule Pleroma.Web.Push.ImplTest do
 
       assert Impl.build_content(notif, actor, object) == %{
                body:
-                 "@Bob: Lorem ipsum dolor sit amet, consectetur  adipiscing elit. Fusce sagittis fini...",
+                 "@Bob: Lorem ipsum dolor sit amet, consectetur  adipiscing elit. Fusce sagittis finibus turpis.",
                title: "New Direct Message"
              }
 
@@ -383,7 +383,7 @@ defmodule Pleroma.Web.Push.ImplTest do
 
       assert Impl.build_content(notif, actor, object) == %{
                body:
-                 "@Bob: Lorem ipsum dolor sit amet, consectetur  adipiscing elit. Fusce sagittis fini...",
+                 "@Bob: Lorem ipsum dolor sit amet, consectetur  adipiscing elit. Fusce sagittis finibus turpis.",
                title: "New Mention"
              }
 

From f000dab372993e1cbd494535b82c248d44595ab5 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Sat, 8 Jun 2024 22:54:47 -0400
Subject: [PATCH 184/192] Switch test case to Impl.build_content/3

---
 test/pleroma/web/push/impl_test.exs | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/test/pleroma/web/push/impl_test.exs b/test/pleroma/web/push/impl_test.exs
index 969c7a605..c263a1280 100644
--- a/test/pleroma/web/push/impl_test.exs
+++ b/test/pleroma/web/push/impl_test.exs
@@ -239,7 +239,7 @@ defmodule Pleroma.Web.Push.ImplTest do
 
     {:ok, [notification]} = Notification.create_poll_notifications(activity)
 
-    assert Impl.format_title(notification) == "Poll Results"
+    expected_title = "Poll Results"
 
     expected_body =
       """
@@ -250,7 +250,9 @@ defmodule Pleroma.Web.Push.ImplTest do
       """
       |> String.trim_trailing("\n")
 
-    assert Impl.format_body(notification, user, question) == expected_body
+    content = Impl.build_content(notification, user, question)
+
+    assert match?(%{title: ^expected_title, body: ^expected_body}, content)
   end
 
   describe "build_content/3" do

From 2fd155fb9bb82401131b40cc62dd17fc44086361 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Tue, 11 Jun 2024 15:59:48 -0400
Subject: [PATCH 185/192] Add PollWorker test; move the streaming notification
 test to it

---
 test/pleroma/notification_test.exs        | 27 +------------
 test/pleroma/workers/poll_worker_test.exs | 49 +++++++++++++++++++++++
 2 files changed, 51 insertions(+), 25 deletions(-)
 create mode 100644 test/pleroma/workers/poll_worker_test.exs

diff --git a/test/pleroma/notification_test.exs b/test/pleroma/notification_test.exs
index 02ae06c63..2c582c708 100644
--- a/test/pleroma/notification_test.exs
+++ b/test/pleroma/notification_test.exs
@@ -5,7 +5,6 @@
 defmodule Pleroma.NotificationTest do
   use Pleroma.DataCase, async: false
 
-  import Mock
   import Pleroma.Factory
 
   alias Pleroma.FollowingRelationship
@@ -184,31 +183,9 @@ defmodule Pleroma.NotificationTest do
     {:ok, _, _} = CommonAPI.vote(user2, question, [0])
     {:ok, _, _} = CommonAPI.vote(user3, question, [1])
 
-    with_mocks([
-      {
-        Pleroma.Web.Streamer,
-        [],
-        [
-          stream: fn _, _ -> nil end
-        ]
-      },
-      {
-        Pleroma.Web.Push,
-        [],
-        [
-          send: fn _ -> nil end
-        ]
-      }
-    ]) do
-      {:ok, notifications} = Notification.create_poll_notifications(activity)
+    {:ok, notifications} = Notification.create_poll_notifications(activity)
 
-      Enum.each(notifications, fn notification ->
-        assert called(Pleroma.Web.Streamer.stream(["user", "user:notification"], notification))
-        assert called(Pleroma.Web.Push.send(notification))
-      end)
-
-      assert [user2.id, user3.id, user1.id] == Enum.map(notifications, & &1.user_id)
-    end
+    assert [user2.id, user3.id, user1.id] == Enum.map(notifications, & &1.user_id)
   end
 
   describe "create_notification" do
diff --git a/test/pleroma/workers/poll_worker_test.exs b/test/pleroma/workers/poll_worker_test.exs
new file mode 100644
index 000000000..749df8aff
--- /dev/null
+++ b/test/pleroma/workers/poll_worker_test.exs
@@ -0,0 +1,49 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Workers.PollWorkerTest do
+  use Pleroma.DataCase
+  use Oban.Testing, repo: Pleroma.Repo
+
+  import Mock
+  import Pleroma.Factory
+
+  alias Pleroma.Workers.PollWorker
+
+  test "poll notification job" do
+    user = insert(:user)
+    question = insert(:question, user: user)
+    activity = insert(:question_activity, question: question)
+
+    PollWorker.schedule_poll_end(activity)
+
+    expected_job_args = %{"activity_id" => activity.id, "op" => "poll_end"}
+
+    assert_enqueued(args: expected_job_args)
+
+    with_mocks([
+      {
+        Pleroma.Web.Streamer,
+        [],
+        [
+          stream: fn _, _ -> nil end
+        ]
+      },
+      {
+        Pleroma.Web.Push,
+        [],
+        [
+          send: fn _ -> nil end
+        ]
+      }
+    ]) do
+      [job] = all_enqueued(worker: PollWorker)
+      PollWorker.perform(job)
+
+      # Ensure notifications were streamed out when job executes
+      assert called(Pleroma.Web.Streamer.stream(["user", "user:notification"], :_))
+      assert called(Pleroma.Web.Push.send(:_))
+    end
+  end
+end

From 5313255b1afcff394fbfe145081ac0ceb2cbe283 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Tue, 11 Jun 2024 16:06:18 -0400
Subject: [PATCH 186/192] Use conn.request_path for more legible error log

---
 changelog.d/spex-error-log.skip               | 0
 lib/pleroma/web/api_spec/cast_and_validate.ex | 2 +-
 2 files changed, 1 insertion(+), 1 deletion(-)
 create mode 100644 changelog.d/spex-error-log.skip

diff --git a/changelog.d/spex-error-log.skip b/changelog.d/spex-error-log.skip
new file mode 100644
index 000000000..e69de29bb
diff --git a/lib/pleroma/web/api_spec/cast_and_validate.ex b/lib/pleroma/web/api_spec/cast_and_validate.ex
index eb487fce7..672d1c4a1 100644
--- a/lib/pleroma/web/api_spec/cast_and_validate.ex
+++ b/lib/pleroma/web/api_spec/cast_and_validate.ex
@@ -54,7 +54,7 @@ defmodule Pleroma.Web.ApiSpec.CastAndValidate do
 
       {:error, reason} ->
         Logger.error(
-          "Strict ApiSpec: request denied to #{conn.path_info} with params #{inspect(conn.params)}"
+          "Strict ApiSpec: request denied to #{conn.request_path} with params #{inspect(conn.params)}"
         )
 
         opts = render_error.init(reason)

From 568819c08afee68636a4871e78838db1ac1f590c Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Tue, 11 Jun 2024 17:58:02 -0400
Subject: [PATCH 187/192] WebPush refactoring: separate build and deliver steps

---
 lib/pleroma/web/push/impl.ex             | 95 ++++++++++++------------
 lib/pleroma/workers/web_pusher_worker.ex |  4 +-
 test/pleroma/web/push/impl_test.exs      | 77 +++++++++++--------
 3 files changed, 97 insertions(+), 79 deletions(-)

diff --git a/lib/pleroma/web/push/impl.ex b/lib/pleroma/web/push/impl.ex
index 13c054e05..c5ba7ca65 100644
--- a/lib/pleroma/web/push/impl.ex
+++ b/lib/pleroma/web/push/impl.ex
@@ -19,69 +19,72 @@ defmodule Pleroma.Web.Push.Impl do
   @body_chars 140
   @types ["Create", "Follow", "Announce", "Like", "Move", "EmojiReact", "Update"]
 
-  @doc "Performs sending notifications for user subscriptions"
-  @spec perform(Notification.t()) :: list(any) | :error | {:error, :unknown_type}
-  def perform(
+  @doc "Builds webpush notification payloads for the subscriptions enabled by the receiving user"
+  @spec build(Notification.t()) ::
+          list(%{content: map(), subscription: Subscription.t()})
+          | :error
+          | {:error, :unknown_type}
+  def build(
         %{
           activity: %{data: %{"type" => activity_type}} = activity,
-          user: %User{id: user_id}
+          user: user
         } = notification
       )
       when activity_type in @types do
-    user = User.get_cached_by_ap_id(notification.activity.data["actor"])
+    notification_actor = User.get_cached_by_ap_id(notification.activity.data["actor"])
+    avatar_url = User.avatar_url(notification_actor)
 
-    gcm_api_key = Application.get_env(:web_push_encryption, :gcm_api_key)
-    avatar_url = User.avatar_url(user)
     object = Object.normalize(activity, fetch: false)
-    user = User.get_cached_by_id(user_id)
     direct_conversation_id = Activity.direct_conversation_id(activity, user)
 
-    for subscription <- fetch_subscriptions(user_id),
-        Subscription.enabled?(subscription, notification.type) do
-      %{
-        access_token: subscription.token.token,
-        notification_id: notification.id,
-        notification_type: notification.type,
-        icon: avatar_url,
-        preferred_locale: "en",
-        pleroma: %{
-          activity_id: notification.activity.id,
-          direct_conversation_id: direct_conversation_id
+    subscriptions = fetch_subscriptions(user.id)
+
+    subscriptions
+    |> Enum.filter(&Subscription.enabled?(&1, notification.type))
+    |> Enum.map(fn subscription ->
+      payload =
+        %{
+          access_token: subscription.token.token,
+          notification_id: notification.id,
+          notification_type: notification.type,
+          icon: avatar_url,
+          preferred_locale: "en",
+          pleroma: %{
+            activity_id: notification.activity.id,
+            direct_conversation_id: direct_conversation_id
+          }
         }
-      }
-      |> Map.merge(build_content(notification, user, object))
-      |> Jason.encode!()
-      |> push_message(build_sub(subscription), gcm_api_key, subscription)
-    end
-    |> (&{:ok, &1}).()
+        |> Map.merge(build_content(notification, notification_actor, object))
+        |> Jason.encode!()
+
+      %{payload: payload, subscription: subscription}
+    end)
   end
 
-  def perform(_) do
+  def build(_) do
     Logger.warning("Unknown notification type")
     {:error, :unknown_type}
   end
 
-  @doc "Push message to web"
-  def push_message(body, sub, api_key, subscription) do
-    try do
-      case WebPushEncryption.send_web_push(body, sub, api_key) do
-        {:ok, %{status: code}} when code in 400..499 ->
-          Logger.debug("Removing subscription record")
-          Repo.delete!(subscription)
-          :ok
+  @doc "Deliver push notification to the provided webpush subscription"
+  @spec deliver(%{payload: String.t(), subscription: Subscription.t()}) :: :ok | :error
+  def deliver(%{payload: payload, subscription: subscription}) do
+    gcm_api_key = Application.get_env(:web_push_encryption, :gcm_api_key)
+    formatted_subscription = build_sub(subscription)
 
-        {:ok, %{status: code}} when code in 200..299 ->
-          :ok
+    case WebPushEncryption.send_web_push(payload, formatted_subscription, gcm_api_key) do
+      {:ok, %{status: code}} when code in 200..299 ->
+        :ok
 
-        {:ok, %{status: code}} ->
-          Logger.error("Web Push Notification failed with code: #{code}")
-          :error
+      {:ok, %{status: code}} when code in 400..499 ->
+        Logger.debug("Removing subscription record")
+        Repo.delete!(subscription)
+        :ok
+
+      {:ok, %{status: code}} ->
+        Logger.error("Web Push Notification failed with code: #{code}")
+        :error
 
-        error ->
-          Logger.error("Web Push Notification failed with #{inspect(error)}")
-          :error
-      end
-    rescue
       error ->
         Logger.error("Web Push Notification failed with #{inspect(error)}")
         :error
@@ -140,9 +143,7 @@ defmodule Pleroma.Web.Push.Impl do
 
     content_text = content <> "\n"
 
-    options_text =
-      Enum.map(options, fn x -> "○ #{x["name"]}" end)
-      |> Enum.join("\n")
+    options_text = Enum.map_join(options, "\n", fn x -> "○ #{x["name"]}" end)
 
     [content_text, options_text]
     |> Enum.join("\n")
diff --git a/lib/pleroma/workers/web_pusher_worker.ex b/lib/pleroma/workers/web_pusher_worker.ex
index 67e84b0c9..c549d3cd6 100644
--- a/lib/pleroma/workers/web_pusher_worker.ex
+++ b/lib/pleroma/workers/web_pusher_worker.ex
@@ -5,6 +5,7 @@
 defmodule Pleroma.Workers.WebPusherWorker do
   alias Pleroma.Notification
   alias Pleroma.Repo
+  alias Pleroma.Web.Push.Impl
 
   use Pleroma.Workers.WorkerHelper, queue: "web_push"
 
@@ -15,7 +16,8 @@ defmodule Pleroma.Workers.WebPusherWorker do
       |> Repo.get(notification_id)
       |> Repo.preload([:activity, :user])
 
-    Pleroma.Web.Push.Impl.perform(notification)
+    Impl.build(notification)
+    |> Enum.each(&Impl.deliver(&1))
   end
 
   @impl Oban.Worker
diff --git a/test/pleroma/web/push/impl_test.exs b/test/pleroma/web/push/impl_test.exs
index c263a1280..7f8dc2e6e 100644
--- a/test/pleroma/web/push/impl_test.exs
+++ b/test/pleroma/web/push/impl_test.exs
@@ -32,17 +32,6 @@ defmodule Pleroma.Web.Push.ImplTest do
     :ok
   end
 
-  @sub %{
-    endpoint: "https://example.com/example/1234",
-    keys: %{
-      auth: "8eDyX_uCN0XRhSbY5hs7Hg==",
-      p256dh:
-        "BCIWgsnyXDv1VkhqL2P7YRBvdeuDnlwAPT2guNhdIoW3IP7GmHh1SMKPLxRf7x8vJy6ZFK3ol2ohgn_-0yP7QQA="
-    }
-  }
-  @api_key "BASgACIHpN1GYgzSRp"
-  @message "@Bob: Lorem ipsum dolor sit amet, consectetur  adipiscing elit. Fusce sagittis finibus turpis."
-
   test "performs sending notifications" do
     user = insert(:user)
     user2 = insert(:user)
@@ -68,39 +57,65 @@ defmodule Pleroma.Web.Push.ImplTest do
         type: "mention"
       )
 
-    assert Impl.perform(notif) == {:ok, [:ok, :ok]}
+    Impl.build(notif)
+    |> Enum.each(fn push -> assert match?(:ok, Impl.deliver(push)) end)
   end
 
   @tag capture_log: true
   test "returns error if notif does not match " do
-    assert Impl.perform(%{}) == {:error, :unknown_type}
-  end
-
-  test "successful message sending" do
-    assert Impl.push_message(@message, @sub, @api_key, %Subscription{}) == :ok
+    assert Impl.build(%{}) == {:error, :unknown_type}
   end
 
   @tag capture_log: true
   test "fail message sending" do
-    assert Impl.push_message(
-             @message,
-             Map.merge(@sub, %{endpoint: "https://example.com/example/bad"}),
-             @api_key,
-             %Subscription{}
-           ) == :error
+    user = insert(:user)
+
+    insert(:push_subscription,
+      user: user,
+      endpoint: "https://example.com/example/bad",
+      data: %{alerts: %{"follow" => true}}
+    )
+
+    other_user = insert(:user)
+    {:ok, _, _, activity} = CommonAPI.follow(user, other_user)
+
+    notif =
+      insert(:notification,
+        user: user,
+        activity: activity,
+        type: "follow"
+      )
+
+    [push] = Impl.build(notif)
+
+    assert Impl.deliver(push) == :error
   end
 
   test "delete subscription if result send message between 400..500" do
-    subscription = insert(:push_subscription)
+    user = insert(:user)
 
-    assert Impl.push_message(
-             @message,
-             Map.merge(@sub, %{endpoint: "https://example.com/example/not_found"}),
-             @api_key,
-             subscription
-           ) == :ok
+    bad_subscription =
+      insert(:push_subscription,
+        user: user,
+        endpoint: "https://example.com/example/not_found",
+        data: %{alerts: %{"follow" => true}}
+      )
 
-    refute Pleroma.Repo.get(Subscription, subscription.id)
+    other_user = insert(:user)
+    {:ok, _, _, activity} = CommonAPI.follow(user, other_user)
+
+    notif =
+      insert(:notification,
+        user: user,
+        activity: activity,
+        type: "follow"
+      )
+
+    [push] = Impl.build(notif)
+
+    assert Impl.deliver(push) == :ok
+
+    refute Pleroma.Repo.get(Subscription, bad_subscription.id)
   end
 
   test "deletes subscription when token has been deleted" do

From 603a57576638c9732fe873db3c2b8d56d0a413a5 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Tue, 11 Jun 2024 18:14:07 -0400
Subject: [PATCH 188/192] The user is not always preloaded into the
 notification

---
 lib/pleroma/web/push/impl.ex | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/lib/pleroma/web/push/impl.ex b/lib/pleroma/web/push/impl.ex
index c5ba7ca65..65801922d 100644
--- a/lib/pleroma/web/push/impl.ex
+++ b/lib/pleroma/web/push/impl.ex
@@ -27,7 +27,7 @@ defmodule Pleroma.Web.Push.Impl do
   def build(
         %{
           activity: %{data: %{"type" => activity_type}} = activity,
-          user: user
+          user_id: user_id
         } = notification
       )
       when activity_type in @types do
@@ -35,9 +35,10 @@ defmodule Pleroma.Web.Push.Impl do
     avatar_url = User.avatar_url(notification_actor)
 
     object = Object.normalize(activity, fetch: false)
+    user = User.get_cached_by_id(user_id)
     direct_conversation_id = Activity.direct_conversation_id(activity, user)
 
-    subscriptions = fetch_subscriptions(user.id)
+    subscriptions = fetch_subscriptions(user_id)
 
     subscriptions
     |> Enum.filter(&Subscription.enabled?(&1, notification.type))

From a291a6b8c0c6e27ba6f9a03d9a1183184fc2ef1d Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Tue, 11 Jun 2024 18:14:48 -0400
Subject: [PATCH 189/192] Ensure the webpush notification for e.g., mentions
 start with the nickname of the actor it originates from

---
 changelog.d/web_push_actor_regression.skip |  0
 test/pleroma/web/push/impl_test.exs        | 19 +++++++++++++++++++
 2 files changed, 19 insertions(+)
 create mode 100644 changelog.d/web_push_actor_regression.skip

diff --git a/changelog.d/web_push_actor_regression.skip b/changelog.d/web_push_actor_regression.skip
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/pleroma/web/push/impl_test.exs b/test/pleroma/web/push/impl_test.exs
index 7f8dc2e6e..0eead956f 100644
--- a/test/pleroma/web/push/impl_test.exs
+++ b/test/pleroma/web/push/impl_test.exs
@@ -417,4 +417,23 @@ defmodule Pleroma.Web.Push.ImplTest do
              }
     end
   end
+
+  test "build/1 notification payload body starts with nickname of actor the notification originated from" do
+    user = insert(:user, nickname: "Bob")
+    user2 = insert(:user, nickname: "Tom")
+    insert(:push_subscription, user: user2, data: %{alerts: %{"mention" => true}})
+
+    {:ok, activity} =
+      CommonAPI.post(user, %{
+        status: "@Tom Hey are you okay?"
+      })
+
+    {:ok, [notification]} = Notification.create_notifications(activity)
+
+    [push] = Impl.build(notification)
+
+    {:ok, payload} = Jason.decode(push.payload)
+
+    assert String.starts_with?(payload["body"], "@Bob:")
+  end
 end

From 6a9d9da26feefe9b19e8072fd378f01a977b9d1a Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Tue, 11 Jun 2024 18:38:41 -0400
Subject: [PATCH 190/192] Cyclical complexity

---
 lib/pleroma/web/push/impl.ex | 26 +++++++++++---------------
 1 file changed, 11 insertions(+), 15 deletions(-)

diff --git a/lib/pleroma/web/push/impl.ex b/lib/pleroma/web/push/impl.ex
index 65801922d..2bcfa97f5 100644
--- a/lib/pleroma/web/push/impl.ex
+++ b/lib/pleroma/web/push/impl.ex
@@ -201,19 +201,15 @@ defmodule Pleroma.Web.Push.Impl do
     "New Direct Message"
   end
 
-  def format_title(%{type: type}) do
-    case type do
-      "mention" -> "New Mention"
-      "status" -> "New Status"
-      "follow" -> "New Follower"
-      "follow_request" -> "New Follow Request"
-      "reblog" -> "New Repeat"
-      "favourite" -> "New Favorite"
-      "update" -> "New Update"
-      "pleroma:chat_mention" -> "New Chat Message"
-      "pleroma:emoji_reaction" -> "New Reaction"
-      "poll" -> "Poll Results"
-      type -> "New #{String.capitalize(type || "event")}"
-    end
-  end
+  def format_title(%{type: "mention"}), do: "New Mention"
+  def format_title(%{type: "status"}), do: "New Status"
+  def format_title(%{type: "follow"}), do: "New Follower"
+  def format_title(%{type: "follow_request"}), do: "New Follow Request"
+  def format_title(%{type: "reblog"}), do: "New Repeat"
+  def format_title(%{type: "favourite"}), do: "New Favorite"
+  def format_title(%{type: "update"}), do: "New Update"
+  def format_title(%{type: "pleroma:chat_mention"}), do: "New Chat Message"
+  def format_title(%{type: "pleroma:emoji_reaction"}), do: "New Reaction"
+  def format_title(%{type: "poll"}), do: "Poll Results"
+  def format_title(%{type: type}), do: "New #{String.capitalize(type || "event")}"
 end

From 5c8afbe646c874eea32d7063aa499c97191f3a6e Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Tue, 11 Jun 2024 18:54:22 -0400
Subject: [PATCH 191/192] Fix tests

---
 lib/pleroma/web/push/impl.ex        | 10 ++++------
 test/pleroma/web/push/impl_test.exs |  7 +++++--
 2 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/lib/pleroma/web/push/impl.ex b/lib/pleroma/web/push/impl.ex
index 2bcfa97f5..d71e134cb 100644
--- a/lib/pleroma/web/push/impl.ex
+++ b/lib/pleroma/web/push/impl.ex
@@ -21,9 +21,7 @@ defmodule Pleroma.Web.Push.Impl do
 
   @doc "Builds webpush notification payloads for the subscriptions enabled by the receiving user"
   @spec build(Notification.t()) ::
-          list(%{content: map(), subscription: Subscription.t()})
-          | :error
-          | {:error, :unknown_type}
+          list(%{content: map(), subscription: Subscription.t()}) | []
   def build(
         %{
           activity: %{data: %{"type" => activity_type}} = activity,
@@ -62,9 +60,9 @@ defmodule Pleroma.Web.Push.Impl do
     end)
   end
 
-  def build(_) do
-    Logger.warning("Unknown notification type")
-    {:error, :unknown_type}
+  def build(notif) do
+    Logger.warning("WebPush: unknown activity type: #{inspect(notif)}")
+    []
   end
 
   @doc "Deliver push notification to the provided webpush subscription"
diff --git a/test/pleroma/web/push/impl_test.exs b/test/pleroma/web/push/impl_test.exs
index 0eead956f..169c380c7 100644
--- a/test/pleroma/web/push/impl_test.exs
+++ b/test/pleroma/web/push/impl_test.exs
@@ -5,6 +5,7 @@
 defmodule Pleroma.Web.Push.ImplTest do
   use Pleroma.DataCase, async: true
 
+  import ExUnit.CaptureLog
   import Mox
   import Pleroma.Factory
 
@@ -62,8 +63,10 @@ defmodule Pleroma.Web.Push.ImplTest do
   end
 
   @tag capture_log: true
-  test "returns error if notif does not match " do
-    assert Impl.build(%{}) == {:error, :unknown_type}
+  test "returns error if notification activity type does not match" do
+    assert capture_log(fn ->
+             assert Impl.build(%{}) == []
+           end) =~ "WebPush: unknown activity type"
   end
 
   @tag capture_log: true

From 1ae5c2b020810eda7243e7e6b52cf89e6bb7f8d0 Mon Sep 17 00:00:00 2001
From: Lain Soykaf <lain@lain.com>
Date: Wed, 12 Jun 2024 12:40:01 +0400
Subject: [PATCH 192/192] Transmogrifier: Encode Emoji id to be valid.

---
 changelog.d/3280-fix-emoji-ids.fix                 |  1 +
 lib/pleroma/web/activity_pub/transmogrifier.ex     |  6 ++++--
 .../transmogrifier/emoji_tag_building_test.exs     | 14 ++++++++++++++
 3 files changed, 19 insertions(+), 2 deletions(-)
 create mode 100644 changelog.d/3280-fix-emoji-ids.fix
 create mode 100644 test/pleroma/web/activity_pub/transmogrifier/emoji_tag_building_test.exs

diff --git a/changelog.d/3280-fix-emoji-ids.fix b/changelog.d/3280-fix-emoji-ids.fix
new file mode 100644
index 000000000..1bce5b653
--- /dev/null
+++ b/changelog.d/3280-fix-emoji-ids.fix
@@ -0,0 +1 @@
+Fix Emoji object IDs not always being valid
diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex
index edfe73a25..4d851559f 100644
--- a/lib/pleroma/web/activity_pub/transmogrifier.ex
+++ b/lib/pleroma/web/activity_pub/transmogrifier.ex
@@ -913,9 +913,11 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
 
   def add_emoji_tags(object), do: object
 
-  defp build_emoji_tag({name, url}) do
+  def build_emoji_tag({name, url}) do
+    url = URI.encode(url)
+
     %{
-      "icon" => %{"url" => "#{URI.encode(url)}", "type" => "Image"},
+      "icon" => %{"url" => "#{url}", "type" => "Image"},
       "name" => ":" <> name <> ":",
       "type" => "Emoji",
       "updated" => "1970-01-01T00:00:00Z",
diff --git a/test/pleroma/web/activity_pub/transmogrifier/emoji_tag_building_test.exs b/test/pleroma/web/activity_pub/transmogrifier/emoji_tag_building_test.exs
new file mode 100644
index 000000000..c632c199c
--- /dev/null
+++ b/test/pleroma/web/activity_pub/transmogrifier/emoji_tag_building_test.exs
@@ -0,0 +1,14 @@
+defmodule Pleroma.Web.ActivityPub.Transmogrifier.EmojiTagBuildingTest do
+  use Pleroma.DataCase, async: true
+
+  alias Pleroma.Web.ActivityPub.Transmogrifier
+
+  test "it encodes the id to be a valid url" do
+    name = "hanapog"
+    url = "https://misskey.local.live/emojis/hana pog.png"
+
+    tag = Transmogrifier.build_emoji_tag({name, url})
+
+    assert tag["id"] == "https://misskey.local.live/emojis/hana%20pog.png"
+  end
+end