From c44cf7d74f513397a9b25e7f7e0fe6092d679840 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Wed, 29 May 2024 06:24:24 +0200 Subject: [PATCH 001/183] StatusOperation: Remove obsolete card_operation --- .../api_spec/operations/status_operation.ex | 28 ------------------- 1 file changed, 28 deletions(-) diff --git a/lib/pleroma/web/api_spec/operations/status_operation.ex b/lib/pleroma/web/api_spec/operations/status_operation.ex index 1717c68c8..f7914406a 100644 --- a/lib/pleroma/web/api_spec/operations/status_operation.ex +++ b/lib/pleroma/web/api_spec/operations/status_operation.ex @@ -339,34 +339,6 @@ defmodule Pleroma.Web.ApiSpec.StatusOperation do } end - def card_operation do - %Operation{ - tags: ["Retrieve status information"], - deprecated: true, - summary: "Preview card", - description: "Deprecated in favor of card property inlined on Status entity", - operationId: "StatusController.card", - parameters: [id_param()], - security: [%{"oAuth" => ["read:statuses"]}], - responses: %{ - 200 => - Operation.response("Card", "application/json", %Schema{ - type: :object, - nullable: true, - properties: %{ - type: %Schema{type: :string, enum: ["link", "photo", "video", "rich"]}, - provider_name: %Schema{type: :string, nullable: true}, - provider_url: %Schema{type: :string, format: :uri}, - url: %Schema{type: :string, format: :uri}, - image: %Schema{type: :string, nullable: true, format: :uri}, - title: %Schema{type: :string}, - description: %Schema{type: :string} - } - }) - } - } - end - def favourited_by_operation do %Operation{ tags: ["Retrieve status information"], From d860d5059703bfbcd3550a8ff8361d83ec06d00d Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Wed, 29 May 2024 06:27:11 +0200 Subject: [PATCH 002/183] StatusController: Restore preview cards tests This is a partial revert of 19002fd6c11760898daf0b5ed648d6ba58d84b97 Subsequent commit updating it to cards embedded in statuses required to pass tests. --- .../controllers/status_controller_test.exs | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs index f34911e5b..0386a5b7a 100644 --- a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs @@ -1653,6 +1653,63 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do end end + describe "cards" do + setup do + Pleroma.StaticStubbedConfigMock + |> stub(:get, fn + [:rich_media, :enabled] -> true + path -> Pleroma.Test.StaticConfig.get(path) + end) + + oauth_access(["read:statuses"]) + end + + test "returns rich-media card", %{conn: conn, user: user} do + Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) + + {:ok, activity} = CommonAPI.post(user, %{status: "https://example.com/ogp"}) + + card_data = %{ + "image" => "http://ia.media-imdb.com/images/rock.jpg", + "provider_name" => "example.com", + "provider_url" => "https://example.com", + "title" => "The Rock", + "type" => "link", + "url" => "https://example.com/ogp", + "description" => + "Directed by Michael Bay. With Sean Connery, Nicolas Cage, Ed Harris, John Spencer.", + "pleroma" => %{ + "opengraph" => %{ + "image" => "http://ia.media-imdb.com/images/rock.jpg", + "title" => "The Rock", + "type" => "video.movie", + "url" => "https://example.com/ogp", + "description" => + "Directed by Michael Bay. With Sean Connery, Nicolas Cage, Ed Harris, John Spencer." + } + } + } + + response = + conn + |> get("/api/v1/statuses/#{activity.id}/card") + |> json_response_and_validate_schema(200) + + assert response == card_data + + # works with private posts + {:ok, activity} = + CommonAPI.post(user, %{status: "https://example.com/ogp", visibility: "direct"}) + + response_two = + conn + |> get("/api/v1/statuses/#{activity.id}/card") + |> json_response_and_validate_schema(200) + + assert response_two == card_data + end + end + test "bookmarks" do bookmarks_uri = "/api/v1/bookmarks" From c7119d17d874ad15291decc29ce1147bbdf78488 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Wed, 29 May 2024 06:24:50 +0200 Subject: [PATCH 003/183] StatusControllerTest: Update preview cards tests to current API --- .../mastodon_api/controllers/status_controller_test.exs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs index 0386a5b7a..08589f6ae 100644 --- a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs @@ -1692,10 +1692,10 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do response = conn - |> get("/api/v1/statuses/#{activity.id}/card") + |> get("/api/v1/statuses/#{activity.id}") |> json_response_and_validate_schema(200) - assert response == card_data + assert response["card"] == card_data # works with private posts {:ok, activity} = @@ -1703,10 +1703,10 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do response_two = conn - |> get("/api/v1/statuses/#{activity.id}/card") + |> get("/api/v1/statuses/#{activity.id}") |> json_response_and_validate_schema(200) - assert response_two == card_data + assert response_two["card"] == card_data end end From 656dce96dc3d4ceb2ac2fed1badc0f2baa290c7f Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Thu, 15 Feb 2024 00:57:34 +0100 Subject: [PATCH 004/183] Status: Fill all MastoAPI required values in preview cards --- changelog.d/previewcard-void-fill.fix | 1 + lib/pleroma/web/api_spec/schemas/status.ex | 49 +++++++++++++++++-- .../web/mastodon_api/views/status_view.ex | 7 +++ .../controllers/status_controller_test.exs | 44 ++++++++++++++++- 4 files changed, 95 insertions(+), 6 deletions(-) create mode 100644 changelog.d/previewcard-void-fill.fix diff --git a/changelog.d/previewcard-void-fill.fix b/changelog.d/previewcard-void-fill.fix new file mode 100644 index 000000000..04e571347 --- /dev/null +++ b/changelog.d/previewcard-void-fill.fix @@ -0,0 +1 @@ +Status: Fill all MastoAPI required values in preview cards diff --git a/lib/pleroma/web/api_spec/schemas/status.ex b/lib/pleroma/web/api_spec/schemas/status.ex index 6e537b5da..1432be264 100644 --- a/lib/pleroma/web/api_spec/schemas/status.ex +++ b/lib/pleroma/web/api_spec/schemas/status.ex @@ -34,16 +34,40 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Status do type: :object, nullable: true, description: "Preview card for links included within status content", - required: [:url, :title, :description, :type], + # Yeah, it's effectivelly all required in MastoAPI so far + required: [ + :url, + :title, + :description, + :type, + :author_name, + :author_url, + :provider_name, + :provider_url, + :html, + :width, + :height, + :image, + :embed_url, + :blurhash + ], properties: %{ + url: %Schema{type: :string, format: :uri, description: "Location of linked resource"}, + title: %Schema{type: :string, description: "Title of linked resource"}, + description: %Schema{type: :string, description: "Description of preview"}, type: %Schema{ type: :string, enum: ["link", "photo", "video", "rich"], description: "The type of the preview card" }, + author_name: %Schema{type: :string, description: "author of the original resource"}, + author_url: %Schema{ + type: :string, + format: :uri, + description: "link to the author of the original resource" + }, provider_name: %Schema{ type: :string, - nullable: true, description: "The provider of the original resource" }, provider_url: %Schema{ @@ -51,7 +75,13 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Status do format: :uri, description: "A link to the provider of the original resource" }, - url: %Schema{type: :string, format: :uri, description: "Location of linked resource"}, + html: %Schema{ + type: :string, + format: :html, + description: "HTML to be used for generating the preview card" + }, + width: %Schema{type: :integer, description: "Width of preview, in pixels"}, + height: %Schema{type: :integer, description: "Height of preview, in pixels"}, image: %Schema{ type: :string, nullable: true, @@ -62,8 +92,17 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Status do type: :string, description: "Alternate text that describes what is in the thumbnail" }, - title: %Schema{type: :string, description: "Title of linked resource"}, - description: %Schema{type: :string, description: "Description of preview"} + embed_url: %Schema{ + type: :string, + format: :uri, + description: "Used for photo embeds, instead of custom `html`" + }, + blurhash: %Schema{ + type: :string, + nullable: true, + description: + "A hash computed by the (BlurHash algorithm)[https://github.com/woltapp/blurhash], for generating colorful preview thumbnails when media has not been downloaded yet." + } } }, content: %Schema{type: :string, format: :html, description: "HTML-encoded status content"}, diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 0c16749a4..7a654381d 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -586,6 +586,13 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do image_description: rich_media["image:alt"] || "", title: rich_media["title"] || "", description: rich_media["description"] || "", + author_name: "", + author_url: "", + html: "", + width: 0, + height: 0, + embed_url: "", + blurhash: nil, pleroma: %{ opengraph: rich_media diff --git a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs index 08589f6ae..725356edc 100644 --- a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs @@ -1687,7 +1687,14 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do "description" => "Directed by Michael Bay. With Sean Connery, Nicolas Cage, Ed Harris, John Spencer." } - } + }, + "author_name" => "", + "author_url" => "", + "blurhash" => nil, + "embed_url" => "", + "height" => 0, + "html" => "", + "width" => 0 } response = @@ -1708,6 +1715,41 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do assert response_two["card"] == card_data end + + test "replaces missing description with an empty string", %{conn: conn, user: user} do + Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) + + {:ok, activity} = CommonAPI.post(user, %{status: "https://example.com/ogp-missing-data"}) + + response = + conn + |> get("/api/v1/statuses/#{activity.id}") + |> json_response_and_validate_schema(:ok) + + assert response["card"] == %{ + "type" => "link", + "title" => "Pleroma", + "description" => "", + "image" => nil, + "provider_name" => "example.com", + "provider_url" => "https://example.com", + "url" => "https://example.com/ogp-missing-data", + "pleroma" => %{ + "opengraph" => %{ + "title" => "Pleroma", + "type" => "website", + "url" => "https://example.com/ogp-missing-data" + } + }, + "author_name" => "", + "author_url" => "", + "blurhash" => nil, + "embed_url" => "", + "height" => 0, + "html" => "", + "width" => 0 + } + end end test "bookmarks" do From 0feb23a3082b65915e0abbe09c2df83f6014d6c3 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Fri, 16 Jan 2026 23:16:51 +0400 Subject: [PATCH 005/183] AP ingest: accept object type as list --- .../web/activity_pub/transmogrifier.ex | 6 +++ .../fedidev.fun/object_type_list.json | 28 ++++++++++ .../fedidev_fun_object_type_list_test.exs | 53 +++++++++++++++++++ 3 files changed, 87 insertions(+) create mode 100644 test/fixtures/fedidev.fun/object_type_list.json create mode 100644 test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_object_type_list_test.exs diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index ba6f4030d..6b3194709 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -433,9 +433,15 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def handle_incoming(data, options \\ []) do data |> fix_recursive(&strip_internal_fields/1) + |> fix_recursive(&fix_type_as_string/1) |> handle_incoming_normalized(options) end + defp fix_type_as_string(%{"type" => [type | _]} = data) when is_binary(type), + do: Map.put(data, "type", type) + + defp fix_type_as_string(data), do: data + # Flag objects are placed ahead of the ID check because Mastodon 2.8 and earlier send them # with nil ID. defp handle_incoming_normalized( diff --git a/test/fixtures/fedidev.fun/object_type_list.json b/test/fixtures/fedidev.fun/object_type_list.json new file mode 100644 index 000000000..f4ab00578 --- /dev/null +++ b/test/fixtures/fedidev.fun/object_type_list.json @@ -0,0 +1,28 @@ +{ + "@context": [ + "https://www.w3.org/ns/activitystreams" + ], + "type": "Create", + "id": "https://fedidev.fun/activity/object-type-list", + "actor": "https://fedidev.fun/users/alice", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [], + "published": "2025-11-21T14:43:32Z", + "object": { + "type": [ + "Note" + ], + "id": "https://fedidev.fun/objects/note-object-type-list", + "attributedTo": "https://fedidev.fun/users/alice", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [], + "published": "2025-11-21T14:43:32Z", + "name": "name", + "summary": "summary", + "content": "content" + } +} diff --git a/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_object_type_list_test.exs b/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_object_type_list_test.exs new file mode 100644 index 000000000..619e6f477 --- /dev/null +++ b/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_object_type_list_test.exs @@ -0,0 +1,53 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.Transmogrifier.FedidevFunObjectTypeListTest do + use Oban.Testing, repo: Pleroma.Repo + use Pleroma.DataCase + + alias Pleroma.Object + alias Pleroma.User + alias Pleroma.Web.ActivityPub.Transmogrifier + + import Pleroma.Factory + + setup_all do + Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) + :ok + end + + defp ensure_remote_actor!(ap_id) when is_binary(ap_id) do + case User.get_cached_by_ap_id(ap_id) do + %User{} -> + :ok + + _ -> + insert(:user, + local: false, + ap_id: ap_id, + follower_address: ap_id <> "/followers", + following_address: ap_id <> "/following", + featured_address: ap_id <> "/collections/featured", + inbox: ap_id <> "/inbox" + ) + + :ok + end + end + + defp ingest_fixture!(rel_path) do + data = rel_path |> File.read!() |> Jason.decode!() + ensure_remote_actor!(data["actor"]) + + assert {:ok, activity} = Transmogrifier.handle_incoming(data) + object = Object.normalize(activity.data["object"], fetch: false) + + {activity, object} + end + + test "accepts object type as a list and stores the first type" do + {_activity, object} = ingest_fixture!("test/fixtures/fedidev.fun/object_type_list.json") + assert object.data["type"] == "Note" + end +end From 39817fa54727204dc997e9211bae57d5fd625583 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Fri, 16 Jan 2026 23:25:07 +0400 Subject: [PATCH 006/183] AP ingest: normalize object url list --- .../article_note_page_validator.ex | 27 +++++++++- test/fixtures/fedidev.fun/url_list.json | 27 ++++++++++ .../fedidev_fun_url_list_test.exs | 53 +++++++++++++++++++ 3 files changed, 106 insertions(+), 1 deletion(-) create mode 100644 test/fixtures/fedidev.fun/url_list.json create mode 100644 test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_url_list_test.exs diff --git a/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex b/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex index c0626ce4d..b0cdee552 100644 --- a/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex @@ -46,7 +46,32 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidator do end defp fix_url(%{"url" => url} = data) when is_bitstring(url), do: data - defp fix_url(%{"url" => url} = data) when is_map(url), do: Map.put(data, "url", url["href"]) + + defp fix_url(%{"url" => url} = data) when is_map(url) do + if is_binary(url["href"]) do + Map.put(data, "url", url["href"]) + else + Map.delete(data, "url") + end + end + + defp fix_url(%{"url" => url} = data) when is_list(url) do + first_element = Enum.at(url, 0) + + url_string = + cond do + is_bitstring(first_element) -> first_element + is_map(first_element) -> first_element["href"] + true -> nil + end + + if is_binary(url_string) do + Map.put(data, "url", url_string) + else + Map.delete(data, "url") + end + end + defp fix_url(data), do: data defp fix_tag(%{"tag" => tag} = data) when is_list(tag) do diff --git a/test/fixtures/fedidev.fun/url_list.json b/test/fixtures/fedidev.fun/url_list.json new file mode 100644 index 000000000..b4cfd4cd6 --- /dev/null +++ b/test/fixtures/fedidev.fun/url_list.json @@ -0,0 +1,27 @@ +{ + "@context": [ + "https://www.w3.org/ns/activitystreams" + ], + "type": "Create", + "id": "https://fedidev.fun/activity/url-list", + "actor": "https://fedidev.fun/users/alice", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [], + "published": "2025-11-21T14:39:06Z", + "object": { + "type": "Note", + "id": "https://fedidev.fun/objects/note-url-list", + "attributedTo": "https://fedidev.fun/users/alice", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [], + "published": "2025-11-21T14:39:06Z", + "content": "text", + "url": [ + "https://fedidev.fun/objects/123" + ] + } +} diff --git a/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_url_list_test.exs b/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_url_list_test.exs new file mode 100644 index 000000000..bc084a622 --- /dev/null +++ b/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_url_list_test.exs @@ -0,0 +1,53 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.Transmogrifier.FedidevFunUrlListTest do + use Oban.Testing, repo: Pleroma.Repo + use Pleroma.DataCase + + alias Pleroma.Object + alias Pleroma.User + alias Pleroma.Web.ActivityPub.Transmogrifier + + import Pleroma.Factory + + setup_all do + Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) + :ok + end + + defp ensure_remote_actor!(ap_id) when is_binary(ap_id) do + case User.get_cached_by_ap_id(ap_id) do + %User{} -> + :ok + + _ -> + insert(:user, + local: false, + ap_id: ap_id, + follower_address: ap_id <> "/followers", + following_address: ap_id <> "/following", + featured_address: ap_id <> "/collections/featured", + inbox: ap_id <> "/inbox" + ) + + :ok + end + end + + defp ingest_fixture!(rel_path) do + data = rel_path |> File.read!() |> Jason.decode!() + ensure_remote_actor!(data["actor"]) + + assert {:ok, activity} = Transmogrifier.handle_incoming(data) + object = Object.normalize(activity.data["object"], fetch: false) + + {activity, object} + end + + test "normalizes object url list to a string" do + {_activity, object} = ingest_fixture!("test/fixtures/fedidev.fun/url_list.json") + assert object.data["url"] == "https://fedidev.fun/objects/123" + end +end From 6701f6199006b52a70288d2c90cd93aa97ce34ff Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Fri, 16 Jan 2026 23:27:06 +0400 Subject: [PATCH 007/183] AP ingest: normalize quoteUrl fields --- .../object_validators/common_fixes.ex | 27 ++++++-- .../fedidev.fun/misskey_quote_list.json | 28 +++++++++ test/fixtures/fedidev.fun/quote_uri_list.json | 28 +++++++++ test/fixtures/fedidev.fun/quote_url_list.json | 28 +++++++++ .../fedidev_fun_quote_url_list_test.exs | 63 +++++++++++++++++++ 5 files changed, 170 insertions(+), 4 deletions(-) create mode 100644 test/fixtures/fedidev.fun/misskey_quote_list.json create mode 100644 test/fixtures/fedidev.fun/quote_uri_list.json create mode 100644 test/fixtures/fedidev.fun/quote_url_list.json create mode 100644 test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_quote_url_list_test.exs diff --git a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex index 2e0f7a8a0..f34a4327f 100644 --- a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex +++ b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex @@ -96,23 +96,37 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes do Map.put(data, "to", to) end - def fix_quote_url(%{"quoteUrl" => _quote_url} = data), do: data + def fix_quote_url(%{"quoteUrl" => quote_url} = data) do + case normalize_object_id(quote_url) do + quote_url when is_binary(quote_url) -> Map.put(data, "quoteUrl", quote_url) + _ -> Map.delete(data, "quoteUrl") + end + end # Fedibird # https://github.com/fedibird/mastodon/commit/dbd7ae6cf58a92ec67c512296b4daaea0d01e6ac def fix_quote_url(%{"quoteUri" => quote_url} = data) do - Map.put(data, "quoteUrl", quote_url) + case normalize_object_id(quote_url) do + quote_url when is_binary(quote_url) -> Map.put(data, "quoteUrl", quote_url) + _ -> data + end end # Old Fedibird (bug) # https://github.com/fedibird/mastodon/issues/9 def fix_quote_url(%{"quoteURL" => quote_url} = data) do - Map.put(data, "quoteUrl", quote_url) + case normalize_object_id(quote_url) do + quote_url when is_binary(quote_url) -> Map.put(data, "quoteUrl", quote_url) + _ -> data + end end # Misskey fallback def fix_quote_url(%{"_misskey_quote" => quote_url} = data) do - Map.put(data, "quoteUrl", quote_url) + case normalize_object_id(quote_url) do + quote_url when is_binary(quote_url) -> Map.put(data, "quoteUrl", quote_url) + _ -> data + end end def fix_quote_url(%{"tag" => [_ | _] = tags} = data) do @@ -128,6 +142,11 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes do def fix_quote_url(data), do: data + defp normalize_object_id(object_id) when is_binary(object_id), do: object_id + defp normalize_object_id(%{"id" => object_id}) when is_binary(object_id), do: object_id + defp normalize_object_id([object_id | _]) when is_binary(object_id), do: object_id + defp normalize_object_id(_), do: nil + # On Mastodon, `"likes"` attribute includes an inlined `Collection` with `totalItems`, # not a list of users. # https://github.com/mastodon/mastodon/pull/32007 diff --git a/test/fixtures/fedidev.fun/misskey_quote_list.json b/test/fixtures/fedidev.fun/misskey_quote_list.json new file mode 100644 index 000000000..b9ea08691 --- /dev/null +++ b/test/fixtures/fedidev.fun/misskey_quote_list.json @@ -0,0 +1,28 @@ +{ + "@context": [ + "https://www.w3.org/ns/activitystreams" + ], + "type": "Create", + "id": "https://fedidev.fun/activity/misskey-quote-list", + "actor": "https://fedidev.fun/users/alice", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [], + "published": "2025-11-21T14:43:53Z", + "object": { + "type": "Note", + "id": "https://fedidev.fun/objects/note-misskey-quote-list", + "attributedTo": "https://fedidev.fun/users/alice", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [], + "published": "2025-11-21T14:43:53Z", + "content": "_misskey_quote attribute as list", + "_misskey_quote": [ + "https://fedidev.fun/assets/note1.jsonap", + "https://fedidev.fun/assets/note2.jsonap" + ] + } +} diff --git a/test/fixtures/fedidev.fun/quote_uri_list.json b/test/fixtures/fedidev.fun/quote_uri_list.json new file mode 100644 index 000000000..9245077e6 --- /dev/null +++ b/test/fixtures/fedidev.fun/quote_uri_list.json @@ -0,0 +1,28 @@ +{ + "@context": [ + "https://www.w3.org/ns/activitystreams" + ], + "type": "Create", + "id": "https://fedidev.fun/activity/quote-uri-list", + "actor": "https://fedidev.fun/users/alice", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [], + "published": "2025-11-21T14:43:52Z", + "object": { + "type": "Note", + "id": "https://fedidev.fun/objects/note-quote-uri-list", + "attributedTo": "https://fedidev.fun/users/alice", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [], + "published": "2025-11-21T14:43:52Z", + "content": "quoteUri attribute as list", + "quoteUri": [ + "https://fedidev.fun/assets/note1.jsonap", + "https://fedidev.fun/assets/note2.jsonap" + ] + } +} diff --git a/test/fixtures/fedidev.fun/quote_url_list.json b/test/fixtures/fedidev.fun/quote_url_list.json new file mode 100644 index 000000000..eecd0dd35 --- /dev/null +++ b/test/fixtures/fedidev.fun/quote_url_list.json @@ -0,0 +1,28 @@ +{ + "@context": [ + "https://www.w3.org/ns/activitystreams" + ], + "type": "Create", + "id": "https://fedidev.fun/activity/quote-url-list", + "actor": "https://fedidev.fun/users/alice", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [], + "published": "2025-11-21T14:43:52Z", + "object": { + "type": "Note", + "id": "https://fedidev.fun/objects/note-quote-url-list", + "attributedTo": "https://fedidev.fun/users/alice", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [], + "published": "2025-11-21T14:43:52Z", + "content": "quoteUrl attribute as list", + "quoteUrl": [ + "https://fedidev.fun/assets/note1.jsonap", + "https://fedidev.fun/assets/note2.jsonap" + ] + } +} diff --git a/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_quote_url_list_test.exs b/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_quote_url_list_test.exs new file mode 100644 index 000000000..9607fabdd --- /dev/null +++ b/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_quote_url_list_test.exs @@ -0,0 +1,63 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.Transmogrifier.FedidevFunQuoteUrlListTest do + use Oban.Testing, repo: Pleroma.Repo + use Pleroma.DataCase + + alias Pleroma.Object + alias Pleroma.User + alias Pleroma.Web.ActivityPub.Transmogrifier + + import Pleroma.Factory + + setup_all do + Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) + :ok + end + + defp ensure_remote_actor!(ap_id) when is_binary(ap_id) do + case User.get_cached_by_ap_id(ap_id) do + %User{} -> + :ok + + _ -> + insert(:user, + local: false, + ap_id: ap_id, + follower_address: ap_id <> "/followers", + following_address: ap_id <> "/following", + featured_address: ap_id <> "/collections/featured", + inbox: ap_id <> "/inbox" + ) + + :ok + end + end + + defp ingest_fixture!(rel_path) do + data = rel_path |> File.read!() |> Jason.decode!() + ensure_remote_actor!(data["actor"]) + + assert {:ok, activity} = Transmogrifier.handle_incoming(data) + object = Object.normalize(activity.data["object"], fetch: false) + + {activity, object} + end + + test "normalizes quoteUri list to the first object ID" do + {_activity, object} = ingest_fixture!("test/fixtures/fedidev.fun/quote_uri_list.json") + assert object.data["quoteUrl"] == "https://fedidev.fun/assets/note1.jsonap" + end + + test "normalizes quoteUrl list to the first object ID" do + {_activity, object} = ingest_fixture!("test/fixtures/fedidev.fun/quote_url_list.json") + assert object.data["quoteUrl"] == "https://fedidev.fun/assets/note1.jsonap" + end + + test "normalizes _misskey_quote list to the first object ID" do + {_activity, object} = ingest_fixture!("test/fixtures/fedidev.fun/misskey_quote_list.json") + assert object.data["quoteUrl"] == "https://fedidev.fun/assets/note1.jsonap" + end +end From bff67650e109b8f2064c4b6570164da06fe90ac4 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Fri, 16 Jan 2026 23:57:01 +0400 Subject: [PATCH 008/183] AP ingest: normalize inReplyTo shapes --- .../object_validators/common_fixes.ex | 30 +++++--- .../web/activity_pub/transmogrifier.ex | 30 +++++++- .../fedidev.fun/in_reply_to_collection.json | 30 ++++++++ .../in_reply_to_embedded_note.json | 35 +++++++++ .../fedidev.fun/in_reply_to_link.json | 28 ++++++++ .../fedidev.fun/in_reply_to_list.json | 28 ++++++++ .../fedidev_fun_in_reply_to_test.exs | 72 +++++++++++++++++++ .../transmogrifier/note_handling_test.exs | 4 +- 8 files changed, 245 insertions(+), 12 deletions(-) create mode 100644 test/fixtures/fedidev.fun/in_reply_to_collection.json create mode 100644 test/fixtures/fedidev.fun/in_reply_to_embedded_note.json create mode 100644 test/fixtures/fedidev.fun/in_reply_to_link.json create mode 100644 test/fixtures/fedidev.fun/in_reply_to_list.json create mode 100644 test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_in_reply_to_test.exs diff --git a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex index f34a4327f..631f301af 100644 --- a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex +++ b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex @@ -41,22 +41,34 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes do def fix_object_defaults(data) do data = Maps.filter_empty_values(data) + in_reply_to_id = in_reply_to_id(data["inReplyTo"]) + context = Utils.maybe_create_context( - data["context"] || data["conversation"] || data["inReplyTo"] || data["id"] + data["context"] || data["conversation"] || in_reply_to_id || data["id"] ) - %User{follower_address: follower_collection} = User.get_cached_by_ap_id(data["attributedTo"]) + data = Map.put(data, "context", context) - data - |> Map.put("context", context) - |> cast_and_filter_recipients("to", follower_collection) - |> cast_and_filter_recipients("cc", follower_collection) - |> cast_and_filter_recipients("bto", follower_collection) - |> cast_and_filter_recipients("bcc", follower_collection) - |> Transmogrifier.fix_implicit_addressing(follower_collection) + with attributed_to when is_binary(attributed_to) <- data["attributedTo"], + %User{follower_address: follower_collection} <- User.get_cached_by_ap_id(attributed_to) do + data + |> cast_and_filter_recipients("to", follower_collection) + |> cast_and_filter_recipients("cc", follower_collection) + |> cast_and_filter_recipients("bto", follower_collection) + |> cast_and_filter_recipients("bcc", follower_collection) + |> Transmogrifier.fix_implicit_addressing(follower_collection) + else + _ -> data + end end + defp in_reply_to_id(in_reply_to) when is_binary(in_reply_to), do: in_reply_to + defp in_reply_to_id(%{"id" => in_reply_to}) when is_binary(in_reply_to), do: in_reply_to + defp in_reply_to_id(%{"href" => in_reply_to}) when is_binary(in_reply_to), do: in_reply_to + defp in_reply_to_id([in_reply_to | _]), do: in_reply_to_id(in_reply_to) + defp in_reply_to_id(_), do: nil + def fix_activity_addressing(activity) do %User{follower_address: follower_collection} = User.get_cached_by_ap_id(activity["actor"]) diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 6b3194709..43d991a22 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -201,9 +201,29 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def fix_in_reply_to(%{"inReplyTo" => in_reply_to} = object, options) when not is_nil(in_reply_to) do in_reply_to_id = prepare_in_reply_to(in_reply_to) + + object = + cond do + is_list(in_reply_to) -> + if not_empty_string(in_reply_to_id) do + Map.put(object, "inReplyTo", in_reply_to_id) + else + Map.delete(object, "inReplyTo") + end + + is_map(in_reply_to) and is_nil(in_reply_to["id"]) and is_binary(in_reply_to["href"]) -> + Map.put(object, "inReplyTo", in_reply_to["href"]) + + is_map(in_reply_to) and is_nil(in_reply_to["id"]) and is_nil(in_reply_to["href"]) -> + Map.delete(object, "inReplyTo") + + true -> + object + end + depth = (options[:depth] || 0) + 1 - if Federator.allowed_thread_distance?(depth) do + if not_empty_string(in_reply_to_id) and Federator.allowed_thread_distance?(depth) do with {:ok, replied_object} <- get_obj_helper(in_reply_to_id, options), %Activity{} <- Activity.get_create_by_object_ap_id(replied_object.data["id"]) do object @@ -249,9 +269,17 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do is_map(in_reply_to) && is_bitstring(in_reply_to["id"]) -> in_reply_to["id"] + is_map(in_reply_to) && is_bitstring(in_reply_to["href"]) -> + in_reply_to["href"] + is_list(in_reply_to) && is_bitstring(Enum.at(in_reply_to, 0)) -> Enum.at(in_reply_to, 0) + is_list(in_reply_to) && is_map(Enum.at(in_reply_to, 0)) -> + in_reply_to + |> Enum.at(0) + |> prepare_in_reply_to() + true -> "" end diff --git a/test/fixtures/fedidev.fun/in_reply_to_collection.json b/test/fixtures/fedidev.fun/in_reply_to_collection.json new file mode 100644 index 000000000..4deb7ed9a --- /dev/null +++ b/test/fixtures/fedidev.fun/in_reply_to_collection.json @@ -0,0 +1,30 @@ +{ + "@context": [ + "https://www.w3.org/ns/activitystreams" + ], + "type": "Create", + "id": "https://fedidev.fun/activity/in-reply-to-collection", + "actor": "https://fedidev.fun/users/alice", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [], + "published": "2025-11-21T14:40:08Z", + "object": { + "type": "Note", + "id": "https://fedidev.fun/objects/note-in-reply-to-collection", + "attributedTo": "https://fedidev.fun/users/alice", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [], + "published": "2025-11-21T14:40:08Z", + "content": "collection", + "inReplyTo": { + "type": "Collection", + "items": [ + "https://fedidev.fun/users/alice" + ] + } + } +} diff --git a/test/fixtures/fedidev.fun/in_reply_to_embedded_note.json b/test/fixtures/fedidev.fun/in_reply_to_embedded_note.json new file mode 100644 index 000000000..de66f144f --- /dev/null +++ b/test/fixtures/fedidev.fun/in_reply_to_embedded_note.json @@ -0,0 +1,35 @@ +{ + "@context": [ + "https://www.w3.org/ns/activitystreams" + ], + "type": "Create", + "id": "https://fedidev.fun/activity/in-reply-to-embedded-note", + "actor": "https://fedidev.fun/users/alice", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [], + "published": "2025-11-21T14:40:21Z", + "object": { + "type": "Note", + "id": "https://fedidev.fun/objects/note-in-reply-to-embedded-note", + "attributedTo": "https://fedidev.fun/users/alice", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [], + "published": "2025-11-21T14:40:21Z", + "content": "embedded note", + "inReplyTo": { + "@context": "https://www.w3.org/ns/activitystreams", + "type": "Note", + "id": "https://fedidev.fun/assets/note1.jsonap", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "attributedTo": "https://fedidev.fun/users/alice", + "content": "One", + "published": "2024-01-06T13:11:45Z" + } + } +} diff --git a/test/fixtures/fedidev.fun/in_reply_to_link.json b/test/fixtures/fedidev.fun/in_reply_to_link.json new file mode 100644 index 000000000..f000ddcac --- /dev/null +++ b/test/fixtures/fedidev.fun/in_reply_to_link.json @@ -0,0 +1,28 @@ +{ + "@context": [ + "https://www.w3.org/ns/activitystreams" + ], + "type": "Create", + "id": "https://fedidev.fun/activity/in-reply-to-link", + "actor": "https://fedidev.fun/users/alice", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [], + "published": "2025-11-21T14:39:56Z", + "object": { + "type": "Note", + "id": "https://fedidev.fun/objects/note-in-reply-to-link", + "attributedTo": "https://fedidev.fun/users/alice", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [], + "published": "2025-11-21T14:39:56Z", + "content": "Link in inReplyTo", + "inReplyTo": { + "type": "Link", + "href": "https://fedidev.fun/assets/note1.jsonap" + } + } +} diff --git a/test/fixtures/fedidev.fun/in_reply_to_list.json b/test/fixtures/fedidev.fun/in_reply_to_list.json new file mode 100644 index 000000000..7242558b3 --- /dev/null +++ b/test/fixtures/fedidev.fun/in_reply_to_list.json @@ -0,0 +1,28 @@ +{ + "@context": [ + "https://www.w3.org/ns/activitystreams" + ], + "type": "Create", + "id": "https://fedidev.fun/activity/in-reply-to-list", + "actor": "https://fedidev.fun/users/alice", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [], + "published": "2025-11-21T14:40:08Z", + "object": { + "type": "Note", + "id": "https://fedidev.fun/objects/note-in-reply-to-list", + "attributedTo": "https://fedidev.fun/users/alice", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [], + "published": "2025-11-21T14:40:08Z", + "content": "two elements in inReplyTo", + "inReplyTo": [ + "https://fedidev.fun/assets/note1.jsonap", + "https://fedidev.fun/assets/note2.jsonap" + ] + } +} diff --git a/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_in_reply_to_test.exs b/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_in_reply_to_test.exs new file mode 100644 index 000000000..b720a5441 --- /dev/null +++ b/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_in_reply_to_test.exs @@ -0,0 +1,72 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.Transmogrifier.FedidevFunInReplyToTest do + use Oban.Testing, repo: Pleroma.Repo + use Pleroma.DataCase + + alias Pleroma.Object + alias Pleroma.User + alias Pleroma.Web.ActivityPub.Transmogrifier + + import Pleroma.Factory + + setup_all do + Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) + :ok + end + + defp ensure_remote_actor!(ap_id) when is_binary(ap_id) do + case User.get_cached_by_ap_id(ap_id) do + %User{} -> + :ok + + _ -> + insert(:user, + local: false, + ap_id: ap_id, + follower_address: ap_id <> "/followers", + following_address: ap_id <> "/following", + featured_address: ap_id <> "/collections/featured", + inbox: ap_id <> "/inbox" + ) + + :ok + end + end + + defp ingest_fixture!(rel_path) do + data = rel_path |> File.read!() |> Jason.decode!() + ensure_remote_actor!(data["actor"]) + + assert {:ok, activity} = Transmogrifier.handle_incoming(data) + object = Object.normalize(activity.data["object"], fetch: false) + + {activity, object} + end + + test "normalizes Link inReplyTo to a string" do + {_activity, object} = ingest_fixture!("test/fixtures/fedidev.fun/in_reply_to_link.json") + assert object.data["inReplyTo"] == "https://fedidev.fun/assets/note1.jsonap" + end + + test "normalizes list inReplyTo to the first object ID" do + {_activity, object} = ingest_fixture!("test/fixtures/fedidev.fun/in_reply_to_list.json") + assert object.data["inReplyTo"] == "https://fedidev.fun/assets/note1.jsonap" + end + + test "drops invalid map shapes in inReplyTo" do + {_activity, object} = + ingest_fixture!("test/fixtures/fedidev.fun/in_reply_to_collection.json") + + assert object.data["inReplyTo"] == nil + end + + test "accepts embedded objects in inReplyTo" do + {_activity, object} = + ingest_fixture!("test/fixtures/fedidev.fun/in_reply_to_embedded_note.json") + + assert object.data["inReplyTo"] == "https://fedidev.fun/assets/note1.jsonap" + end +end diff --git a/test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs b/test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs index 403c98a2d..f28d4f771 100644 --- a/test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs +++ b/test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs @@ -572,11 +572,11 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.NoteHandlingTest do Map.put(data["object"], "inReplyTo", ["https://shitposter.club/notice/2827873"]) modified_object = Transmogrifier.fix_in_reply_to(object_with_reply) - assert modified_object["inReplyTo"] == ["https://shitposter.club/notice/2827873"] + assert modified_object["inReplyTo"] == "https://shitposter.club/notice/2827873" object_with_reply = Map.put(data["object"], "inReplyTo", []) modified_object = Transmogrifier.fix_in_reply_to(object_with_reply) - assert modified_object["inReplyTo"] == [] + assert modified_object["inReplyTo"] == nil end test "returns modified object when allowed incoming reply", %{data: data} do From e781a7ef9b231a62c9c3d93cde20841dca13abea Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sat, 17 Jan 2026 00:08:19 +0400 Subject: [PATCH 009/183] AP ingest: harden attachment parsing --- .../article_note_page_validator.ex | 5 +- .../web/activity_pub/transmogrifier.ex | 11 ++- .../fedidev.fun/attachment_embedded_note.json | 32 +++++++++ .../fedidev.fun/attachment_payto_link.json | 31 +++++++++ .../fedidev.fun/attachment_url_list.json | 40 +++++++++++ .../fedidev_fun_attachments_test.exs | 69 +++++++++++++++++++ 6 files changed, 182 insertions(+), 6 deletions(-) create mode 100644 test/fixtures/fedidev.fun/attachment_embedded_note.json create mode 100644 test/fixtures/fedidev.fun/attachment_payto_link.json create mode 100644 test/fixtures/fedidev.fun/attachment_url_list.json create mode 100644 test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_attachments_test.exs diff --git a/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex b/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex index b0cdee552..f2df2250f 100644 --- a/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex @@ -100,10 +100,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidator do defp fix_replies(data), do: Map.delete(data, "replies") - def fix_attachments(%{"attachment" => attachment} = data) when is_map(attachment), - do: Map.put(data, "attachment", [attachment]) - - def fix_attachments(data), do: data + def fix_attachments(data), do: Transmogrifier.fix_attachments(data) defp fix(data) do data diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 43d991a22..cc84cf8af 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -293,6 +293,12 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do |> Map.drop(["conversation"]) end + defp valid_http_url?(href) when is_binary(href) do + match?({:ok, _}, ObjectValidators.Uri.cast(href)) + end + + defp valid_http_url?(_), do: false + def fix_attachments(%{"attachment" => attachment} = object) when is_list(attachment) do attachments = Enum.map(attachment, fn data -> @@ -305,7 +311,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do media_type = cond do - is_map(url) && url =~ Pleroma.Constants.mime_regex() -> + is_map(url) && is_bitstring(url["mediaType"]) && + url["mediaType"] =~ Pleroma.Constants.mime_regex() -> url["mediaType"] is_bitstring(data["mediaType"]) && data["mediaType"] =~ Pleroma.Constants.mime_regex() -> @@ -326,7 +333,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do true -> nil end - if href do + if href && valid_http_url?(href) do attachment_url = %{ "href" => href, diff --git a/test/fixtures/fedidev.fun/attachment_embedded_note.json b/test/fixtures/fedidev.fun/attachment_embedded_note.json new file mode 100644 index 000000000..8e83762dc --- /dev/null +++ b/test/fixtures/fedidev.fun/attachment_embedded_note.json @@ -0,0 +1,32 @@ +{ + "@context": [ + "https://www.w3.org/ns/activitystreams" + ], + "type": "Create", + "id": "https://fedidev.fun/activity/attachment-embedded-note", + "actor": "https://fedidev.fun/users/alice", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [], + "published": "2025-11-21T14:43:01Z", + "object": { + "type": "Note", + "id": "https://fedidev.fun/objects/note-attachment-embedded-note", + "attributedTo": "https://fedidev.fun/users/alice", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [], + "published": "2025-11-21T14:43:01Z", + "content": "Note with embedded Note attachment", + "attachment": { + "type": "Note", + "attributedTo": "https://fedidev.fun/users/alice", + "name": "attached note", + "content": "This is just a note", + "published": "2024-03-06T07:23:56Z" + } + } +} + diff --git a/test/fixtures/fedidev.fun/attachment_payto_link.json b/test/fixtures/fedidev.fun/attachment_payto_link.json new file mode 100644 index 000000000..1da324cb3 --- /dev/null +++ b/test/fixtures/fedidev.fun/attachment_payto_link.json @@ -0,0 +1,31 @@ +{ + "@context": [ + "https://www.w3.org/ns/activitystreams" + ], + "type": "Create", + "id": "https://fedidev.fun/activity/attachment-payto-link", + "actor": "https://fedidev.fun/users/alice", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [], + "published": "2025-11-21T14:43:00Z", + "object": { + "type": "Note", + "id": "https://fedidev.fun/objects/note-attachment-payto-link", + "attributedTo": "https://fedidev.fun/users/alice", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [], + "published": "2025-11-21T14:43:00Z", + "content": "Payment Link", + "attachment": { + "type": "Link", + "name": "Donate", + "href": "payto://iban/DE75512108001245126199", + "rel": "payment" + } + } +} + diff --git a/test/fixtures/fedidev.fun/attachment_url_list.json b/test/fixtures/fedidev.fun/attachment_url_list.json new file mode 100644 index 000000000..0efaeafdb --- /dev/null +++ b/test/fixtures/fedidev.fun/attachment_url_list.json @@ -0,0 +1,40 @@ +{ + "@context": [ + "https://www.w3.org/ns/activitystreams" + ], + "type": "Create", + "id": "https://fedidev.fun/activity/attachment-url-list", + "actor": "https://fedidev.fun/users/alice", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [], + "published": "2025-11-21T14:43:02Z", + "object": { + "type": "Note", + "id": "https://fedidev.fun/objects/note-attachment-url-list", + "attributedTo": "https://fedidev.fun/users/alice", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [], + "published": "2025-11-21T14:43:02Z", + "content": "url is array of two Link objects", + "attachment": { + "type": "Image", + "url": [ + { + "type": "Link", + "href": "https://fedidev.fun/images/007.png", + "mediaType": "image/png" + }, + { + "type": "Link", + "href": "https://fedidev.fun/images/008.jpg", + "mediaType": "image/jpeg" + } + ] + } + } +} + diff --git a/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_attachments_test.exs b/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_attachments_test.exs new file mode 100644 index 000000000..a13912b08 --- /dev/null +++ b/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_attachments_test.exs @@ -0,0 +1,69 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.Transmogrifier.FedidevFunAttachmentsTest do + use Oban.Testing, repo: Pleroma.Repo + use Pleroma.DataCase + + alias Pleroma.Object + alias Pleroma.User + alias Pleroma.Web.ActivityPub.Transmogrifier + + import Pleroma.Factory + + setup_all do + Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) + :ok + end + + defp ensure_remote_actor!(ap_id) when is_binary(ap_id) do + case User.get_cached_by_ap_id(ap_id) do + %User{} -> + :ok + + _ -> + insert(:user, + local: false, + ap_id: ap_id, + follower_address: ap_id <> "/followers", + following_address: ap_id <> "/following", + featured_address: ap_id <> "/collections/featured", + inbox: ap_id <> "/inbox" + ) + + :ok + end + end + + defp ingest_fixture!(rel_path) do + data = rel_path |> File.read!() |> Jason.decode!() + ensure_remote_actor!(data["actor"]) + + assert {:ok, activity} = Transmogrifier.handle_incoming(data) + object = Object.normalize(activity.data["object"], fetch: false) + + {activity, object} + end + + test "drops non-http(s) attachment href" do + {_activity, object} = ingest_fixture!("test/fixtures/fedidev.fun/attachment_payto_link.json") + assert object.data["attachment"] == [] + end + + test "drops embedded non-attachment objects" do + {_activity, object} = + ingest_fixture!("test/fixtures/fedidev.fun/attachment_embedded_note.json") + + assert object.data["attachment"] == [] + end + + test "accepts attachment url as list and keeps the first link" do + {_activity, object} = ingest_fixture!("test/fixtures/fedidev.fun/attachment_url_list.json") + + [attachment] = object.data["attachment"] + [%{"href" => href} | _] = attachment["url"] + + assert href == "https://fedidev.fun/images/007.png" + end +end From e9e3d255e5349f913ed88a36e6ff38ec93c8544b Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sat, 17 Jan 2026 00:12:48 +0400 Subject: [PATCH 010/183] AP ingest: infer tag type when missing --- .../object_validators/tag_validator.ex | 21 ++++++ .../fedidev.fun/tag_hashtag_missing_type.json | 30 +++++++++ .../fedidev.fun/tag_mention_missing_type.json | 31 +++++++++ .../fedidev_fun_tag_inference_test.exs | 65 +++++++++++++++++++ 4 files changed, 147 insertions(+) create mode 100644 test/fixtures/fedidev.fun/tag_hashtag_missing_type.json create mode 100644 test/fixtures/fedidev.fun/tag_mention_missing_type.json create mode 100644 test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_tag_inference_test.exs diff --git a/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex b/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex index dc2770189..a412c8dea 100644 --- a/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex @@ -80,6 +80,27 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.TagValidator do |> Map.put(:action, :ignore) end + def changeset(struct, data) when is_map(data) do + data = infer_type(data) + + if Map.has_key?(data, "type") do + changeset(struct, data) + else + struct + |> cast(data, []) + |> Map.put(:action, :ignore) + end + end + + defp infer_type(%{"type" => _} = data), do: data + defp infer_type(%{"name" => "#" <> _} = data), do: Map.put(data, "type", "Hashtag") + defp infer_type(%{"name" => "@" <> _} = data), do: Map.put(data, "type", "Mention") + + defp infer_type(%{"mediaType" => _media_type, "href" => _href} = data), + do: Map.put(data, "type", "Link") + + defp infer_type(data), do: data + def icon_changeset(struct, data) do struct |> cast(data, [:type, :url]) diff --git a/test/fixtures/fedidev.fun/tag_hashtag_missing_type.json b/test/fixtures/fedidev.fun/tag_hashtag_missing_type.json new file mode 100644 index 000000000..74f9d91b8 --- /dev/null +++ b/test/fixtures/fedidev.fun/tag_hashtag_missing_type.json @@ -0,0 +1,30 @@ +{ + "@context": [ + "https://www.w3.org/ns/activitystreams" + ], + "type": "Create", + "id": "https://fedidev.fun/activity/tag-hashtag-missing-type", + "actor": "https://fedidev.fun/users/alice", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [], + "published": "2025-11-21T14:41:00Z", + "object": { + "type": "Note", + "id": "https://fedidev.fun/objects/note-tag-hashtag-missing-type", + "attributedTo": "https://fedidev.fun/users/alice", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [], + "published": "2025-11-21T14:41:00Z", + "content": "hashtag without type", + "tag": [ + { + "name": "#Test" + } + ] + } +} + diff --git a/test/fixtures/fedidev.fun/tag_mention_missing_type.json b/test/fixtures/fedidev.fun/tag_mention_missing_type.json new file mode 100644 index 000000000..d290be6c2 --- /dev/null +++ b/test/fixtures/fedidev.fun/tag_mention_missing_type.json @@ -0,0 +1,31 @@ +{ + "@context": [ + "https://www.w3.org/ns/activitystreams" + ], + "type": "Create", + "id": "https://fedidev.fun/activity/tag-mention-missing-type", + "actor": "https://fedidev.fun/users/alice", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [], + "published": "2025-11-21T14:41:01Z", + "object": { + "type": "Note", + "id": "https://fedidev.fun/objects/note-tag-mention-missing-type", + "attributedTo": "https://fedidev.fun/users/alice", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [], + "published": "2025-11-21T14:41:01Z", + "content": "mention without type", + "tag": [ + { + "name": "@bob", + "href": "https://remote.example/users/bob" + } + ] + } +} + diff --git a/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_tag_inference_test.exs b/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_tag_inference_test.exs new file mode 100644 index 000000000..ef7feb8ad --- /dev/null +++ b/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_tag_inference_test.exs @@ -0,0 +1,65 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.Transmogrifier.FedidevFunTagInferenceTest do + use Oban.Testing, repo: Pleroma.Repo + use Pleroma.DataCase + + alias Pleroma.Object + alias Pleroma.User + alias Pleroma.Web.ActivityPub.Transmogrifier + + import Pleroma.Factory + + setup_all do + Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) + :ok + end + + defp ensure_remote_actor!(ap_id) when is_binary(ap_id) do + case User.get_cached_by_ap_id(ap_id) do + %User{} -> + :ok + + _ -> + insert(:user, + local: false, + ap_id: ap_id, + follower_address: ap_id <> "/followers", + following_address: ap_id <> "/following", + featured_address: ap_id <> "/collections/featured", + inbox: ap_id <> "/inbox" + ) + + :ok + end + end + + defp ingest_fixture!(rel_path) do + data = rel_path |> File.read!() |> Jason.decode!() + ensure_remote_actor!(data["actor"]) + + assert {:ok, activity} = Transmogrifier.handle_incoming(data) + object = Object.normalize(activity.data["object"], fetch: false) + + {activity, object} + end + + test "infers Hashtag type when missing" do + {_activity, object} = + ingest_fixture!("test/fixtures/fedidev.fun/tag_hashtag_missing_type.json") + + assert "test" in Object.hashtags(object) + end + + test "infers Mention type when missing" do + {_activity, object} = + ingest_fixture!("test/fixtures/fedidev.fun/tag_mention_missing_type.json") + + assert Enum.any?(object.data["tag"], fn + %{"type" => "Mention", "href" => "https://remote.example/users/bob"} -> true + _ -> false + end) + end +end From 9912a9f603b1d0816f95491b67a9ba12553123ce Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sat, 17 Jan 2026 01:33:21 +0400 Subject: [PATCH 011/183] AP ingest: accept Emoji icon variants --- .../object_validators/tag_validator.ex | 62 ++++++++++++++++-- .../web/activity_pub/transmogrifier.ex | 30 ++++++++- .../fedidev.fun/emoji_icon_string.json | 32 +++++++++ .../emoji_icon_url_missing_type.json | 34 ++++++++++ .../fedidev.fun/emoji_missing_icon.json | 31 +++++++++ .../transmogrifier/fedidev_fun_emoji_test.exs | 65 +++++++++++++++++++ 6 files changed, 246 insertions(+), 8 deletions(-) create mode 100644 test/fixtures/fedidev.fun/emoji_icon_string.json create mode 100644 test/fixtures/fedidev.fun/emoji_icon_url_missing_type.json create mode 100644 test/fixtures/fedidev.fun/emoji_missing_icon.json create mode 100644 test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_emoji_test.exs diff --git a/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex b/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex index a412c8dea..cdab8d78d 100644 --- a/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex @@ -59,12 +59,29 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.TagValidator do end def changeset(struct, %{"type" => "Emoji"} = data) do - data = Map.put(data, "name", String.trim(data["name"], ":")) + data = + data + |> Map.put("name", String.trim(data["name"], ":")) + |> normalize_emoji_icon() - struct - |> cast(data, [:type, :name, :updated, :id]) - |> cast_embed(:icon, with: &icon_changeset/2) - |> validate_required([:type, :name, :icon]) + case data["icon"] do + %{"url" => url} when is_binary(url) -> + if valid_http_url?(url) do + struct + |> cast(data, [:type, :name, :updated, :id]) + |> cast_embed(:icon, with: &icon_changeset/2) + |> validate_required([:type, :name, :icon]) + else + struct + |> cast(data, []) + |> Map.put(:action, :ignore) + end + + _ -> + struct + |> cast(data, []) + |> Map.put(:action, :ignore) + end end def changeset(struct, %{"type" => "Link"} = data) do @@ -101,6 +118,41 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.TagValidator do defp infer_type(data), do: data + defp normalize_emoji_icon(%{"icon" => icon} = data) when is_binary(icon) do + Map.put(data, "icon", %{"type" => "Image", "url" => icon}) + end + + defp normalize_emoji_icon(%{"icon" => icon} = data) when is_map(icon) do + case extract_icon_url(icon) do + url when is_binary(url) -> Map.put(data, "icon", %{"type" => "Image", "url" => url}) + _ -> Map.delete(data, "icon") + end + end + + defp normalize_emoji_icon(data), do: data + + defp extract_icon_url(%{"url" => url}) when is_binary(url), do: url + + defp extract_icon_url(%{"url" => %{"href" => href}}) when is_binary(href), do: href + + defp extract_icon_url(%{"url" => [first | _]}) do + cond do + is_binary(first) -> first + is_map(first) -> first["href"] + true -> nil + end + end + + defp extract_icon_url(%{"href" => href}) when is_binary(href), do: href + + defp extract_icon_url(_), do: nil + + defp valid_http_url?(url) when is_binary(url) do + match?({:ok, _}, ObjectValidators.Uri.cast(url)) + end + + defp valid_http_url?(_), do: false + def icon_changeset(struct, data) do struct |> cast(data, [:type, :url]) diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index cc84cf8af..55b70b881 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -386,6 +386,24 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def fix_url(object), do: object + defp emoji_icon_url(icon) when is_binary(icon), do: icon + + defp emoji_icon_url(%{"url" => url}) when is_binary(url), do: url + + defp emoji_icon_url(%{"url" => %{"href" => href}}) when is_binary(href), do: href + + defp emoji_icon_url(%{"url" => [first | _]}) do + cond do + is_binary(first) -> first + is_map(first) -> first["href"] || first["url"] || first["id"] + true -> nil + end + end + + defp emoji_icon_url(%{"href" => href}) when is_binary(href), do: href + + defp emoji_icon_url(_), do: nil + def fix_emoji(%{"tag" => tags} = object) when is_list(tags) do emoji = tags @@ -393,7 +411,10 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do |> Enum.reduce(%{}, fn data, mapping -> name = String.trim(data["name"], ":") - Map.put(mapping, name, data["icon"]["url"]) + case emoji_icon_url(data["icon"]) do + url when is_binary(url) -> Map.put(mapping, name, url) + _ -> mapping + end end) Map.put(object, "emoji", emoji) @@ -401,9 +422,12 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def fix_emoji(%{"tag" => %{"type" => "Emoji"} = tag} = object) do name = String.trim(tag["name"], ":") - emoji = %{name => tag["icon"]["url"]} - Map.put(object, "emoji", emoji) + with url when is_binary(url) <- emoji_icon_url(tag["icon"]) do + Map.put(object, "emoji", %{name => url}) + else + _ -> object + end end def fix_emoji(object), do: object diff --git a/test/fixtures/fedidev.fun/emoji_icon_string.json b/test/fixtures/fedidev.fun/emoji_icon_string.json new file mode 100644 index 000000000..0be3deb8c --- /dev/null +++ b/test/fixtures/fedidev.fun/emoji_icon_string.json @@ -0,0 +1,32 @@ +{ + "@context": [ + "https://www.w3.org/ns/activitystreams" + ], + "type": "Create", + "id": "https://fedidev.fun/activity/emoji-icon-string", + "actor": "https://fedidev.fun/users/alice", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [], + "published": "2025-11-21T14:40:00Z", + "object": { + "type": "Note", + "id": "https://fedidev.fun/objects/note-emoji-icon-string", + "attributedTo": "https://fedidev.fun/users/alice", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [], + "published": "2025-11-21T14:40:00Z", + "content": "emoji icon as string :cow:", + "tag": [ + { + "type": "Emoji", + "name": ":cow:", + "icon": "https://fedidev.fun/static/cow.png" + } + ] + } +} + diff --git a/test/fixtures/fedidev.fun/emoji_icon_url_missing_type.json b/test/fixtures/fedidev.fun/emoji_icon_url_missing_type.json new file mode 100644 index 000000000..2837872eb --- /dev/null +++ b/test/fixtures/fedidev.fun/emoji_icon_url_missing_type.json @@ -0,0 +1,34 @@ +{ + "@context": [ + "https://www.w3.org/ns/activitystreams" + ], + "type": "Create", + "id": "https://fedidev.fun/activity/emoji-icon-url-missing-type", + "actor": "https://fedidev.fun/users/alice", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [], + "published": "2025-11-21T14:40:01Z", + "object": { + "type": "Note", + "id": "https://fedidev.fun/objects/note-emoji-icon-url-missing-type", + "attributedTo": "https://fedidev.fun/users/alice", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [], + "published": "2025-11-21T14:40:01Z", + "content": "emoji icon url missing type :cow2:", + "tag": [ + { + "type": "Emoji", + "name": ":cow2:", + "icon": { + "url": "https://fedidev.fun/static/cow.png" + } + } + ] + } +} + diff --git a/test/fixtures/fedidev.fun/emoji_missing_icon.json b/test/fixtures/fedidev.fun/emoji_missing_icon.json new file mode 100644 index 000000000..69af01a73 --- /dev/null +++ b/test/fixtures/fedidev.fun/emoji_missing_icon.json @@ -0,0 +1,31 @@ +{ + "@context": [ + "https://www.w3.org/ns/activitystreams" + ], + "type": "Create", + "id": "https://fedidev.fun/activity/emoji-missing-icon", + "actor": "https://fedidev.fun/users/alice", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [], + "published": "2025-11-21T14:40:02Z", + "object": { + "type": "Note", + "id": "https://fedidev.fun/objects/note-emoji-missing-icon", + "attributedTo": "https://fedidev.fun/users/alice", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [], + "published": "2025-11-21T14:40:02Z", + "content": "emoji missing icon :cow3:", + "tag": [ + { + "type": "Emoji", + "name": ":cow3:" + } + ] + } +} + diff --git a/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_emoji_test.exs b/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_emoji_test.exs new file mode 100644 index 000000000..56c305352 --- /dev/null +++ b/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_emoji_test.exs @@ -0,0 +1,65 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.Transmogrifier.FedidevFunEmojiTest do + use Oban.Testing, repo: Pleroma.Repo + use Pleroma.DataCase + + alias Pleroma.Object + alias Pleroma.User + alias Pleroma.Web.ActivityPub.Transmogrifier + + import Pleroma.Factory + + setup_all do + Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) + :ok + end + + defp ensure_remote_actor!(ap_id) when is_binary(ap_id) do + case User.get_cached_by_ap_id(ap_id) do + %User{} -> + :ok + + _ -> + insert(:user, + local: false, + ap_id: ap_id, + follower_address: ap_id <> "/followers", + following_address: ap_id <> "/following", + featured_address: ap_id <> "/collections/featured", + inbox: ap_id <> "/inbox" + ) + + :ok + end + end + + defp ingest_fixture!(rel_path) do + data = rel_path |> File.read!() |> Jason.decode!() + ensure_remote_actor!(data["actor"]) + + assert {:ok, activity} = Transmogrifier.handle_incoming(data) + object = Object.normalize(activity.data["object"], fetch: false) + + {activity, object} + end + + test "ingests Emoji tag with string icon" do + {_activity, object} = ingest_fixture!("test/fixtures/fedidev.fun/emoji_icon_string.json") + assert object.data["emoji"]["cow"] == "https://fedidev.fun/static/cow.png" + end + + test "ingests Emoji tag with icon missing type" do + {_activity, object} = + ingest_fixture!("test/fixtures/fedidev.fun/emoji_icon_url_missing_type.json") + + assert object.data["emoji"]["cow2"] == "https://fedidev.fun/static/cow.png" + end + + test "ignores Emoji tag missing icon" do + {_activity, object} = ingest_fixture!("test/fixtures/fedidev.fun/emoji_missing_icon.json") + assert object.data["emoji"] == %{} + end +end From 3b3e369f83940db06da4030242c9ccc2ba8897cf Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sat, 17 Jan 2026 02:46:59 +0400 Subject: [PATCH 012/183] AP ingest: fill missing attributedTo from activity --- lib/pleroma/object/containment.ex | 30 ++++++++---- .../web/activity_pub/object_validator.ex | 2 + .../object_validators/common_fixes.ex | 11 +++++ .../fedidev.fun/missing_attributed_to.json | 25 ++++++++++ ...fedidev_fun_missing_attributed_to_test.exs | 49 +++++++++++++++++++ 5 files changed, 108 insertions(+), 9 deletions(-) create mode 100644 test/fixtures/fedidev.fun/missing_attributed_to.json create mode 100644 test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_missing_attributed_to_test.exs diff --git a/lib/pleroma/object/containment.ex b/lib/pleroma/object/containment.ex index 77fac12c0..b90d68636 100644 --- a/lib/pleroma/object/containment.ex +++ b/lib/pleroma/object/containment.ex @@ -15,12 +15,28 @@ defmodule Pleroma.Object.Containment do actor end + def get_actor(%{"actor" => nil, "attributedTo" => actor}) when not is_nil(actor) do + get_actor(%{"actor" => actor}) + end + + def get_actor(%{"actor" => nil}), do: nil + + def get_actor(%{"actor" => []}), do: nil + def get_actor(%{"actor" => actor}) when is_list(actor) do - if is_binary(Enum.at(actor, 0)) do - Enum.at(actor, 0) - else - Enum.find(actor, fn %{"type" => type} -> type in ["Person", "Service", "Application"] end) - |> Map.get("id") + case actor do + [first | _] when is_binary(first) -> + first + + _ -> + Enum.find_value(actor, fn + %{"type" => type, "id" => id} + when type in ["Person", "Service", "Application"] and is_binary(id) -> + id + + _ -> + nil + end) end end @@ -28,10 +44,6 @@ defmodule Pleroma.Object.Containment do id end - def get_actor(%{"actor" => nil, "attributedTo" => actor}) when not is_nil(actor) do - get_actor(%{"actor" => actor}) - end - def get_object(%{"object" => id}) when is_binary(id) do id end diff --git a/lib/pleroma/web/activity_pub/object_validator.ex b/lib/pleroma/web/activity_pub/object_validator.ex index 17652a0de..0254d6dc1 100644 --- a/lib/pleroma/web/activity_pub/object_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validator.ex @@ -118,6 +118,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do when objtype in ~w[Question Answer Audio Video Image Event Article Note Page] do with {:ok, object_data} <- object + |> CommonFixes.maybe_set_attributed_to_from_activity(create_activity) |> CommonFixes.maybe_add_language_from_activity(create_activity) |> cast_and_apply_and_stringify_with_history(), meta = Keyword.put(meta, :object_data, object_data), @@ -172,6 +173,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do {_, {:ok, object_data, _}} <- {:object_validation, object + |> CommonFixes.maybe_set_attributed_to_from_activity(update_activity) |> CommonFixes.maybe_add_language_from_activity(update_activity) |> validate(meta)}, meta = Keyword.put(meta, :object_data, object_data), diff --git a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex index 631f301af..c5defd6f4 100644 --- a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex +++ b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex @@ -91,6 +91,17 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes do |> Map.put("attributedTo", actor) end + def maybe_set_attributed_to_from_activity(object, activity) when is_map(object) do + if is_nil(object["attributedTo"]) and is_nil(object["actor"]) do + case Containment.get_actor(activity) do + actor when is_binary(actor) -> Map.put(object, "attributedTo", actor) + _ -> object + end + else + object + end + end + def fix_activity_context(data, %Object{data: %{"context" => object_context}}) do data |> Map.put("context", object_context) diff --git a/test/fixtures/fedidev.fun/missing_attributed_to.json b/test/fixtures/fedidev.fun/missing_attributed_to.json new file mode 100644 index 000000000..c2e644e84 --- /dev/null +++ b/test/fixtures/fedidev.fun/missing_attributed_to.json @@ -0,0 +1,25 @@ +{ + "@context": [ + "https://www.w3.org/ns/activitystreams" + ], + "type": "Create", + "id": "https://fedidev.fun/activity/missing-attributedto", + "actor": "https://fedidev.fun/users/alice", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [], + "published": "2025-11-21T14:42:00Z", + "object": { + "type": "Note", + "id": "https://fedidev.fun/objects/note-missing-attributedto", + "attributedTo": null, + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [], + "published": "2025-11-21T14:42:00Z", + "content": "object without attributedTo" + } +} + diff --git a/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_missing_attributed_to_test.exs b/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_missing_attributed_to_test.exs new file mode 100644 index 000000000..c08e20fc9 --- /dev/null +++ b/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_missing_attributed_to_test.exs @@ -0,0 +1,49 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.Transmogrifier.FedidevFunMissingAttributedToTest do + use Oban.Testing, repo: Pleroma.Repo + use Pleroma.DataCase + + alias Pleroma.Object + alias Pleroma.User + alias Pleroma.Web.ActivityPub.Transmogrifier + + import Pleroma.Factory + + setup_all do + Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) + :ok + end + + defp ensure_remote_actor!(ap_id) when is_binary(ap_id) do + case User.get_cached_by_ap_id(ap_id) do + %User{} -> + :ok + + _ -> + insert(:user, + local: false, + ap_id: ap_id, + follower_address: ap_id <> "/followers", + following_address: ap_id <> "/following", + featured_address: ap_id <> "/collections/featured", + inbox: ap_id <> "/inbox" + ) + + :ok + end + end + + test "inherits attributedTo from surrounding activity" do + data = File.read!("test/fixtures/fedidev.fun/missing_attributed_to.json") |> Jason.decode!() + ensure_remote_actor!(data["actor"]) + + assert {:ok, activity} = Transmogrifier.handle_incoming(data) + object = Object.normalize(activity.data["object"], fetch: false) + + assert object.data["attributedTo"] == data["actor"] + assert object.data["actor"] == data["actor"] + end +end From 3a87250e15f46b276171f93ba99b8475aa483ebf Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 3 Mar 2026 12:17:34 +0400 Subject: [PATCH 013/183] Add ParadeDB external search backend Introduce an optional search backend that indexes public/unlisted Notes into a separate ParadeDB (pg_search) Postgres and queries it for status search results. --- changelog.d/paradedb-search.add | 1 + config/config.exs | 8 + config/description.exs | 29 +++- docs/configuration/search.md | 34 ++++ lib/pleroma/application.ex | 23 ++- lib/pleroma/search/paradedb.ex | 213 ++++++++++++++++++++++++++ lib/pleroma/search/paradedb/client.ex | 24 +++ lib/pleroma/search/paradedb/repo.ex | 24 +++ test/pleroma/search/paradedb_test.exs | 168 ++++++++++++++++++++ test/support/mocks.ex | 2 + 10 files changed, 520 insertions(+), 6 deletions(-) create mode 100644 changelog.d/paradedb-search.add create mode 100644 lib/pleroma/search/paradedb.ex create mode 100644 lib/pleroma/search/paradedb/client.ex create mode 100644 lib/pleroma/search/paradedb/repo.ex create mode 100644 test/pleroma/search/paradedb_test.exs diff --git a/changelog.d/paradedb-search.add b/changelog.d/paradedb-search.add new file mode 100644 index 000000000..00a6e2361 --- /dev/null +++ b/changelog.d/paradedb-search.add @@ -0,0 +1 @@ +Add ParadeDB (pg_search) as an optional external search backend. diff --git a/config/config.exs b/config/config.exs index 683805fe3..7e374b22b 100644 --- a/config/config.exs +++ b/config/config.exs @@ -938,6 +938,14 @@ config :pleroma, Pleroma.Search.Meilisearch, private_key: nil, initial_indexing_chunk_size: 100_000 +config :pleroma, Pleroma.Search.ParadeDB, + url: nil, + table: "pleroma_search_documents" + +config :pleroma, Pleroma.Search.ParadeDB.Repo, + pool_size: 10, + prepare: :unnamed + config :pleroma, Pleroma.Application, background_migrators: true, internal_fetch: true, diff --git a/config/description.exs b/config/description.exs index c388d17c3..de454dfb2 100644 --- a/config/description.exs +++ b/config/description.exs @@ -3503,7 +3503,12 @@ config :pleroma, :config_description, [ key: :module, type: :keyword, description: "Selected search module.", - suggestion: [Pleroma.Search.DatabaseSearch, Pleroma.Search.Meilisearch] + suggestion: [ + Pleroma.Search.DatabaseSearch, + Pleroma.Search.Meilisearch, + Pleroma.Search.QdrantSearch, + Pleroma.Search.ParadeDB + ] } ] }, @@ -3536,6 +3541,28 @@ config :pleroma, :config_description, [ } ] }, + %{ + group: :pleroma, + key: Pleroma.Search.ParadeDB, + type: :group, + description: "ParadeDB settings.", + children: [ + %{ + key: :url, + type: :string, + description: + "ParadeDB database URL (separate Postgres instance). " <> + "Can also be set via `PARADEDB_DATABASE_URL`.", + suggestion: ["postgres://postgres:postgres@127.0.0.1:5432/paradedb"] + }, + %{ + key: :table, + type: :string, + description: "Table name used for search documents.", + suggestion: ["pleroma_search_documents"] + } + ] + }, %{ group: :pleroma, key: Pleroma.Language.LanguageDetector, diff --git a/docs/configuration/search.md b/docs/configuration/search.md index d34f84d4f..780904d10 100644 --- a/docs/configuration/search.md +++ b/docs/configuration/search.md @@ -145,3 +145,37 @@ This will clear **all** the posts from the search index. Note, that deleted post there is no need to actually clear the whole index, unless you want **all** of it gone. That said, the index does not hold any information that cannot be re-created from the database, it should also generally be a lot smaller than the size of your database. Still, the size depends on the amount of text in posts. + +## ParadeDB + +[ParadeDB](https://www.paradedb.com/) is a Postgres extension that provides BM25 full-text search. In Pleroma, it can be used as an +external search backend by pointing Pleroma at a separate Postgres instance with the `pg_search` extension installed. + +Pleroma will maintain a small `pleroma_search_documents` table in that database (via the existing search indexing queue) and run search +queries against it. + +To use it, set the search module to `Pleroma.Search.ParadeDB` and configure the ParadeDB database URL: + +> config :pleroma, Pleroma.Search, module: Pleroma.Search.ParadeDB +> +> config :pleroma, Pleroma.Search.ParadeDB, +> url: System.get_env("PARADEDB_DATABASE_URL") || "postgres://postgres:postgres@127.0.0.1:5432/paradedb", +> table: "pleroma_search_documents" + +Then, create the table and BM25 index once: + +=== "From Source" + ```sh + mix pleroma.search.indexer create_index + ``` + +After that, run an initial indexing pass to backfill existing posts: + +=== "From Source" + ```sh + mix pleroma.search.indexer index + ``` + +Note: Like the other external search backends, only public/unlisted Notes are indexed. + +Indexing runs via the Oban `search_indexing` queue, so ensure that queue is enabled (not paused) in your Oban configuration. diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 8e1c5de0d..8de71b816 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -80,12 +80,15 @@ defmodule Pleroma.Application do [ Pleroma.PromEx, Pleroma.LDAP, - Pleroma.Repo, - Config.TransferTask, - Pleroma.Emoji, - Pleroma.Web.Plugs.RateLimiter.Supervisor, - {Task.Supervisor, name: Pleroma.TaskSupervisor} + Pleroma.Repo ] ++ + search_children() ++ + [ + Config.TransferTask, + Pleroma.Emoji, + Pleroma.Web.Plugs.RateLimiter.Supervisor, + {Task.Supervisor, name: Pleroma.TaskSupervisor} + ] ++ cachex_children() ++ http_children(adapter) ++ [ @@ -159,6 +162,16 @@ defmodule Pleroma.Application do ] end + defp search_children do + case Config.get([Pleroma.Search, :module]) do + Pleroma.Search.ParadeDB -> + [Pleroma.Search.ParadeDB.Repo] + + _ -> + [] + end + end + defp emoji_packs_expiration, do: expiration(default: :timer.seconds(5 * 60), interval: :timer.seconds(60)) diff --git a/lib/pleroma/search/paradedb.ex b/lib/pleroma/search/paradedb.ex new file mode 100644 index 000000000..963d573e8 --- /dev/null +++ b/lib/pleroma/search/paradedb.ex @@ -0,0 +1,213 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Search.ParadeDB do + @behaviour Pleroma.Search.SearchBackend + + require Logger + + import Ecto.Query + + import Pleroma.Search.DatabaseSearch, + only: [ + maybe_fetch: 3, + maybe_restrict_author: 2, + maybe_restrict_blocked: 2, + maybe_restrict_local: 2 + ] + + import Pleroma.Search.Meilisearch, only: [object_to_search_data: 1] + + alias Pleroma.Activity + alias Pleroma.Config.Getting, as: Config + + @max_limit 40 + @default_table "pleroma_search_documents" + + defp client_impl do + Config.get([Pleroma.Search.ParadeDB, :client_impl]) || + Pleroma.Search.ParadeDB.Client.Postgres + end + + defp table do + Config.get([Pleroma.Search.ParadeDB, :table], @default_table) + end + + @impl true + def create_index do + with {:ok, _} <- client_impl().query("CREATE EXTENSION IF NOT EXISTS pg_search", []), + {:ok, _} <- client_impl().query(create_table_sql(table()), []), + {:ok, _} <- client_impl().query(create_object_id_index_sql(table()), []), + {:ok, _} <- client_impl().query(create_bm25_index_sql(table()), []) do + :ok + else + {:error, error} -> + Logger.error("ParadeDB create_index failed: #{inspect(error)}") + {:error, error} + end + end + + @impl true + def drop_index do + with {:ok, _} <- client_impl().query("DROP TABLE IF EXISTS #{table()}", []) do + :ok + else + {:error, error} -> + Logger.error("ParadeDB drop_index failed: #{inspect(error)}") + {:error, error} + end + end + + @impl true + def healthcheck_endpoints, do: nil + + @impl true + def add_to_index(activity) do + maybe_search_data = object_to_search_data(activity.object) + + if activity.data["type"] == "Create" and maybe_search_data do + actor_ap_id = activity.data["actor"] + published_at = DateTime.from_unix!(maybe_search_data.published) + + sql = + """ + INSERT INTO #{table()} (id, object_id, object_ap_id, actor_ap_id, content, published_at) + VALUES ($1, $2, $3, $4, $5, $6) + ON CONFLICT (id) DO UPDATE SET + object_id = EXCLUDED.object_id, + object_ap_id = EXCLUDED.object_ap_id, + actor_ap_id = EXCLUDED.actor_ap_id, + content = EXCLUDED.content, + published_at = EXCLUDED.published_at + """ + + params = [ + activity.id, + maybe_search_data.id, + maybe_search_data.ap, + actor_ap_id, + maybe_search_data.content, + published_at + ] + + case client_impl().query(sql, params) do + {:ok, _} -> + :ok + + {:error, error} -> + Logger.error("ParadeDB add_to_index failed: #{inspect(error)}") + {:error, error} + end + else + :ok + end + end + + @impl true + def remove_from_index(object) do + sql = "DELETE FROM #{table()} WHERE object_id = $1" + + case client_impl().query(sql, [object.id]) do + {:ok, _} -> + :ok + + {:error, error} -> + Logger.error("ParadeDB remove_from_index failed: #{inspect(error)}") + {:error, error} + end + end + + @impl true + def search(user, query, options \\ []) do + limit = options |> Keyword.get(:limit, @max_limit) |> min(@max_limit) + offset = Keyword.get(options, :offset, 0) + author = Keyword.get(options, :author) + + case search_ids(query, limit, offset, author) do + {:ok, []} -> + maybe_fetch([], user, query) + + {:ok, ids} -> + ids + |> fetch_activities(user, query, author) + + {:error, error} -> + Logger.error("ParadeDB search failed: #{inspect(error)}") + maybe_fetch([], user, query) + end + end + + defp fetch_activities(ids, user, query, author) do + dumped_ids = Enum.map(ids, &FlakeId.from_string/1) + + from(a in Activity, where: a.id in ^dumped_ids) + |> Activity.with_preloaded_object() + |> Activity.restrict_deactivated_users() + |> maybe_restrict_local(user) + |> maybe_restrict_author(author) + |> maybe_restrict_blocked(user) + |> order_by( + [a], + fragment( + "array_position(?, ?)", + ^dumped_ids, + a.id + ) + ) + |> Pleroma.Repo.all() + |> maybe_fetch(user, query) + end + + defp search_ids(query, limit, offset, author) do + base_sql = "SELECT id FROM #{table()} WHERE content &&& $1" + params = [query] + + {base_sql, params} = + case author do + %Pleroma.User{ap_id: ap_id} when is_binary(ap_id) -> + {base_sql <> " AND actor_ap_id = $2", params ++ [ap_id]} + + _ -> + {base_sql, params} + end + + limit_arg = length(params) + 1 + offset_arg = length(params) + 2 + + sql = + base_sql <> + " ORDER BY published_at DESC LIMIT $#{limit_arg} OFFSET $#{offset_arg}" + + params = params ++ [limit, offset] + + with {:ok, %{rows: rows}} <- client_impl().query(sql, params) do + {:ok, Enum.map(rows, fn [id] -> id end)} + end + end + + defp create_table_sql(table) do + """ + CREATE TABLE IF NOT EXISTS #{table} ( + id text PRIMARY KEY, + object_id bigint NOT NULL, + object_ap_id text NOT NULL, + actor_ap_id text, + content text NOT NULL, + published_at timestamptz + ) + """ + end + + defp create_object_id_index_sql(table) do + "CREATE INDEX IF NOT EXISTS #{table}_object_id_idx ON #{table} (object_id)" + end + + defp create_bm25_index_sql(table) do + """ + CREATE INDEX IF NOT EXISTS #{table}_bm25_idx ON #{table} + USING bm25 (id, content, (actor_ap_id::pdb.literal), published_at) + WITH (key_field = 'id') + """ + end +end diff --git a/lib/pleroma/search/paradedb/client.ex b/lib/pleroma/search/paradedb/client.ex new file mode 100644 index 000000000..d6dce0dd4 --- /dev/null +++ b/lib/pleroma/search/paradedb/client.ex @@ -0,0 +1,24 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Search.ParadeDB.Client do + @moduledoc false + + @type sql :: String.t() + @type params :: list() + @type result :: %{optional(:rows) => list(list())} + + @callback query(sql(), params()) :: {:ok, result()} | {:error, any()} +end + +defmodule Pleroma.Search.ParadeDB.Client.Postgres do + @moduledoc false + + @behaviour Pleroma.Search.ParadeDB.Client + + @impl true + def query(sql, params \\ []) do + Ecto.Adapters.SQL.query(Pleroma.Search.ParadeDB.Repo, sql, params) + end +end diff --git a/lib/pleroma/search/paradedb/repo.ex b/lib/pleroma/search/paradedb/repo.ex new file mode 100644 index 000000000..684271847 --- /dev/null +++ b/lib/pleroma/search/paradedb/repo.ex @@ -0,0 +1,24 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Search.ParadeDB.Repo do + use Ecto.Repo, + otp_app: :pleroma, + adapter: Ecto.Adapters.Postgres, + migration_timestamps: [type: :naive_datetime_usec] + + @doc """ + Loads the ParadeDB database URL from `PARADEDB_DATABASE_URL`. + + If the env var is not set, falls back to `config :pleroma, Pleroma.Search.ParadeDB, url: ...`. + """ + def init(_, opts) do + config = Application.get_env(:pleroma, Pleroma.Search.ParadeDB, []) + + url = System.get_env("PARADEDB_DATABASE_URL") || config[:url] + url = if is_nil(url), do: nil, else: to_string(url) + + {:ok, Keyword.put(opts, :url, url)} + end +end diff --git a/test/pleroma/search/paradedb_test.exs b/test/pleroma/search/paradedb_test.exs new file mode 100644 index 000000000..c55e2aa47 --- /dev/null +++ b/test/pleroma/search/paradedb_test.exs @@ -0,0 +1,168 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Search.ParadeDBTest do + use Pleroma.DataCase, async: true + use Oban.Testing, repo: Pleroma.Repo + + import Mox + import Pleroma.Factory + + alias Pleroma.Search.ParadeDB + alias Pleroma.Search.ParadeDB.ClientMock + alias Pleroma.UnstubbedConfigMock, as: Config + alias Pleroma.Web.CommonAPI + alias Pleroma.Workers.SearchIndexingWorker + + describe "ParadeDB" do + test "indexes a public post on creation" do + user = insert(:user) + + {:ok, activity} = + CommonAPI.post(user, %{ + status: "guys i just don't wanna leave the swamp", + visibility: "public" + }) + + args = %{"op" => "add_to_index", "activity" => activity.id} + + assert_enqueued(worker: SearchIndexingWorker, args: args) + + ClientMock + |> expect(:query, fn sql, params -> + assert sql =~ "INSERT INTO pleroma_search_documents" + + [activity_id, object_id, object_ap_id, actor_ap_id, content, published_at] = params + + assert activity_id == activity.id + assert is_integer(object_id) + assert is_binary(object_ap_id) + assert actor_ap_id == activity.data["actor"] + assert content == "guys i just don't wanna leave the swamp" + assert %DateTime{} = published_at + + send(self(), :inserted) + {:ok, %{}} + end) + + Config + |> expect(:get, 3, fn + [Pleroma.Search, :module], nil -> + ParadeDB + + [Pleroma.Search.ParadeDB, :client_impl], nil -> + ClientMock + + [Pleroma.Search.ParadeDB, :table], "pleroma_search_documents" -> + "pleroma_search_documents" + end) + + assert :ok = perform_job(SearchIndexingWorker, args) + assert_received(:inserted) + end + + test "doesn't index posts that are not public" do + user = insert(:user) + + Enum.each(["private", "direct"], fn visibility -> + {:ok, activity} = + CommonAPI.post(user, %{ + status: "guys i just don't wanna leave the swamp", + visibility: visibility + }) + + args = %{"op" => "add_to_index", "activity" => activity.id} + assert_enqueued(worker: SearchIndexingWorker, args: args) + + Config + |> expect(:get, fn + [Pleroma.Search, :module], nil -> + ParadeDB + end) + + assert :ok = perform_job(SearchIndexingWorker, args) + end) + end + + test "deletes posts from index when deleted locally" do + user = insert(:user) + + ClientMock + |> expect(:query, 2, fn sql, params -> + cond do + String.contains?(sql, "INSERT INTO pleroma_search_documents") -> + send(self(), :inserted) + {:ok, %{}} + + String.contains?(sql, "DELETE FROM pleroma_search_documents") -> + assert [object_id] = params + assert is_integer(object_id) + + send(self(), :deleted) + {:ok, %{}} + + true -> + flunk("Unexpected ParadeDB SQL: #{inspect(sql)}") + end + end) + + Config + |> expect(:get, 6, fn + [Pleroma.Search, :module], nil -> + ParadeDB + + [Pleroma.Search.ParadeDB, :client_impl], nil -> + ClientMock + + [Pleroma.Search.ParadeDB, :table], "pleroma_search_documents" -> + "pleroma_search_documents" + end) + + {:ok, activity} = + CommonAPI.post(user, %{ + status: "guys i just don't wanna leave the swamp", + visibility: "public" + }) + + args = %{"op" => "add_to_index", "activity" => activity.id} + assert_enqueued(worker: SearchIndexingWorker, args: args) + assert :ok = perform_job(SearchIndexingWorker, args) + assert_received(:inserted) + + {:ok, _} = CommonAPI.delete(activity.id, user) + + delete_args = %{"op" => "remove_from_index", "object" => activity.object.id} + assert_enqueued(worker: SearchIndexingWorker, args: delete_args) + assert :ok = perform_job(SearchIndexingWorker, delete_args) + assert_received(:deleted) + end + + test "search returns activities in backend order" do + user = insert(:user) + + {:ok, activity1} = CommonAPI.post(user, %{status: "first swamp", visibility: "public"}) + {:ok, activity2} = CommonAPI.post(user, %{status: "second swamp", visibility: "public"}) + + ClientMock + |> expect(:query, fn sql, params -> + assert sql =~ "SELECT id FROM pleroma_search_documents" + assert ["swamp", _limit, _offset] = params + + {:ok, %{rows: [[activity2.id], [activity1.id]]}} + end) + + Config + |> expect(:get, 2, fn + [Pleroma.Search.ParadeDB, :client_impl], nil -> + ClientMock + + [Pleroma.Search.ParadeDB, :table], "pleroma_search_documents" -> + "pleroma_search_documents" + end) + + assert [%{id: ^activity2.id}, %{id: ^activity1.id}] = + ParadeDB.search(nil, "swamp", limit: 40, offset: 0) + end + end +end diff --git a/test/support/mocks.ex b/test/support/mocks.ex index 7e1244220..84465509a 100644 --- a/test/support/mocks.ex +++ b/test/support/mocks.ex @@ -46,3 +46,5 @@ Mox.defmock(Pleroma.SignatureMock, for: Pleroma.Signature.API) Mox.defmock(Pleroma.Web.ActivityPub.TransmogrifierMock, for: Pleroma.Web.ActivityPub.Transmogrifier.API ) + +Mox.defmock(Pleroma.Search.ParadeDB.ClientMock, for: Pleroma.Search.ParadeDB.Client) From 911d84411dc4eabe8bbca8d90cb898ca9f52f4bd Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 3 Mar 2026 12:36:01 +0400 Subject: [PATCH 014/183] Add ParadeDB docker integration test Add a skipped-by-default integration test that boots a ParadeDB container, creates the BM25 index, indexes posts, and verifies search results against the real backend. --- .../search/paradedb_integration_test.exs | 209 ++++++++++++++++++ test/test_helper.exs | 2 +- 2 files changed, 210 insertions(+), 1 deletion(-) create mode 100644 test/pleroma/search/paradedb_integration_test.exs diff --git a/test/pleroma/search/paradedb_integration_test.exs b/test/pleroma/search/paradedb_integration_test.exs new file mode 100644 index 000000000..04e6a9d59 --- /dev/null +++ b/test/pleroma/search/paradedb_integration_test.exs @@ -0,0 +1,209 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Search.ParadeDBIntegrationTest do + use Pleroma.DataCase, async: false + + @moduletag :integration + @moduletag timeout: 180_000 + + import Mox + import Pleroma.Factory + + alias Pleroma.Activity + alias Pleroma.Search.ParadeDB + alias Pleroma.Web.CommonAPI + + @image "paradedb/paradedb:latest" + @db_user "postgres" + @db_password "postgres" + @db_name "paradedb" + @docker_startup_timeout_ms 60_000 + + setup do + Mox.stub_with(Pleroma.UnstubbedConfigMock, Pleroma.Config) + :ok + end + + setup_all do + with :ok <- ensure_docker_available() do + container = "pleroma-paradedb-it-#{System.unique_integer([:positive])}" + + on_exit(fn -> + _ = docker(["rm", "-f", container]) + end) + + {id, 0} = + docker([ + "run", + "--rm", + "--name", + container, + "-e", + "POSTGRES_USER=#{@db_user}", + "-e", + "POSTGRES_PASSWORD=#{@db_password}", + "-e", + "POSTGRES_DB=#{@db_name}", + "-p", + "5432", + "-d", + @image + ]) + + container_id = String.trim(id) + + host_port = + container + |> wait_for_port!(@docker_startup_timeout_ms) + + wait_for_pg_ready!(container, @docker_startup_timeout_ms) + + url = "postgres://#{@db_user}:#{@db_password}@127.0.0.1:#{host_port}/#{@db_name}" + + prev_env = System.get_env("PARADEDB_DATABASE_URL") + System.put_env("PARADEDB_DATABASE_URL", url) + + on_exit(fn -> + if is_nil(prev_env) do + System.delete_env("PARADEDB_DATABASE_URL") + else + System.put_env("PARADEDB_DATABASE_URL", prev_env) + end + end) + + {:ok, _pid} = start_supervised(Pleroma.Search.ParadeDB.Repo) + + table = "pleroma_search_documents_it_#{System.unique_integer([:positive])}" + Pleroma.Config.put([Pleroma.Search.ParadeDB, :table], table) + + {:ok, + %{ + container: container, + container_id: container_id, + url: url, + table: table + }} + end + end + + test "indexes and searches against real ParadeDB" do + :ok = ParadeDB.drop_index() + assert :ok = ParadeDB.create_index() + + user = insert(:user) + + {:ok, a1} = + CommonAPI.post(user, %{status: "paradedb integration first", visibility: "public"}) + + :timer.sleep(1100) + + {:ok, a2} = + CommonAPI.post(user, %{status: "paradedb integration second", visibility: "public"}) + + {:ok, private} = + CommonAPI.post(user, %{status: "paradedb integration private", visibility: "private"}) + + a1 = Activity.get_by_id_with_object(a1.id) + a2 = Activity.get_by_id_with_object(a2.id) + private = Activity.get_by_id_with_object(private.id) + + assert :ok = ParadeDB.add_to_index(a1) + assert :ok = ParadeDB.add_to_index(a2) + assert :ok = ParadeDB.add_to_index(private) + + results = ParadeDB.search(nil, "paradedb integration", limit: 40, offset: 0) + ids = Enum.map(results, & &1.id) + + assert a1.id in ids + assert a2.id in ids + refute private.id in ids + assert List.first(ids) == a2.id + + # Removing an object should remove it from the ParadeDB index. + assert :ok = ParadeDB.remove_from_index(a2.object) + + results_after_delete = ParadeDB.search(nil, "paradedb integration", limit: 40, offset: 0) + ids_after_delete = Enum.map(results_after_delete, & &1.id) + + assert a1.id in ids_after_delete + refute a2.id in ids_after_delete + end + + defp ensure_docker_available do + if is_nil(System.find_executable("docker")) do + {:skip, "Docker not installed"} + else + case docker(["info"]) do + {_out, 0} -> :ok + {out, _} -> {:skip, "Docker not available: #{String.trim(out)}"} + end + end + end + + defp wait_for_port!(container, timeout_ms) do + deadline = System.monotonic_time(:millisecond) + timeout_ms + + wait = fn wait -> + case docker(["port", container, "5432/tcp"]) do + {out, 0} when byte_size(out) > 0 -> + out + |> String.split("\n", trim: true) + |> List.first() + |> parse_host_port!() + + _ -> + if System.monotonic_time(:millisecond) > deadline do + raise "Timed out waiting for docker port mapping" + else + :timer.sleep(250) + wait.(wait) + end + end + end + + wait.(wait) + end + + defp parse_host_port!(mapping) do + mapping + |> String.trim() + |> String.split(":") + |> List.last() + |> String.to_integer() + end + + defp wait_for_pg_ready!(container, timeout_ms) do + deadline = System.monotonic_time(:millisecond) + timeout_ms + + wait = fn wait -> + case docker([ + "exec", + container, + "pg_isready", + "-U", + @db_user, + "-d", + @db_name + ]) do + {_out, 0} -> + :ok + + _ -> + if System.monotonic_time(:millisecond) > deadline do + raise "Timed out waiting for ParadeDB Postgres to become ready" + else + :timer.sleep(250) + wait.(wait) + end + end + end + + wait.(wait) + end + + defp docker(args) do + System.cmd("docker", args, stderr_to_stdout: true) + end +end diff --git a/test/test_helper.exs b/test/test_helper.exs index dc6c05a74..bbc00b7df 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -4,7 +4,7 @@ ExUnit.configure(capture_log: true, max_cases: System.schedulers_online()) -ExUnit.start(exclude: [:federated]) +ExUnit.start(exclude: [:federated, :integration]) if match?({:unix, :darwin}, :os.type()) do excluded = ExUnit.configuration() |> Keyword.get(:exclude, []) From efb75488d9f0f8cd8d9e629ec6c925420d7b174c Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 3 Mar 2026 12:49:23 +0400 Subject: [PATCH 015/183] Tighten ParadeDB integration test Remove warnings that break --warnings-as-errors, add TCP readiness check, and start the ParadeDB Repo with a small pool to reduce connection churn while booting the dockerized backend. --- .../search/paradedb_integration_test.exs | 26 +++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/test/pleroma/search/paradedb_integration_test.exs b/test/pleroma/search/paradedb_integration_test.exs index 04e6a9d59..1da131c6b 100644 --- a/test/pleroma/search/paradedb_integration_test.exs +++ b/test/pleroma/search/paradedb_integration_test.exs @@ -8,7 +8,6 @@ defmodule Pleroma.Search.ParadeDBIntegrationTest do @moduletag :integration @moduletag timeout: 180_000 - import Mox import Pleroma.Factory alias Pleroma.Activity @@ -59,6 +58,7 @@ defmodule Pleroma.Search.ParadeDBIntegrationTest do |> wait_for_port!(@docker_startup_timeout_ms) wait_for_pg_ready!(container, @docker_startup_timeout_ms) + wait_for_tcp_ready!(host_port, @docker_startup_timeout_ms) url = "postgres://#{@db_user}:#{@db_password}@127.0.0.1:#{host_port}/#{@db_name}" @@ -73,7 +73,7 @@ defmodule Pleroma.Search.ParadeDBIntegrationTest do end end) - {:ok, _pid} = start_supervised(Pleroma.Search.ParadeDB.Repo) + {:ok, _pid} = start_supervised({Pleroma.Search.ParadeDB.Repo, pool_size: 1}) table = "pleroma_search_documents_it_#{System.unique_integer([:positive])}" Pleroma.Config.put([Pleroma.Search.ParadeDB, :table], table) @@ -203,6 +203,28 @@ defmodule Pleroma.Search.ParadeDBIntegrationTest do wait.(wait) end + defp wait_for_tcp_ready!(port, timeout_ms) do + deadline = System.monotonic_time(:millisecond) + timeout_ms + + wait = fn wait -> + case :gen_tcp.connect(~c"127.0.0.1", port, [:binary, active: false], 500) do + {:ok, socket} -> + :gen_tcp.close(socket) + :ok + + {:error, _} -> + if System.monotonic_time(:millisecond) > deadline do + raise "Timed out waiting for TCP port #{port} to accept connections" + else + :timer.sleep(250) + wait.(wait) + end + end + end + + wait.(wait) + end + defp docker(args) do System.cmd("docker", args, stderr_to_stdout: true) end From dee7ac0415e0de1c1f2d44d1dcd1202d5c7d099c Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Wed, 4 Mar 2026 13:04:41 +0400 Subject: [PATCH 016/183] Fix ParadeDB key field handling Use uuid ids in the ParadeDB documents table to satisfy BM25 key_field requirements and adjust indexing/search to dump FlakeIds consistently. Also fix the ParadeDB backend unit test to avoid invalid pinning on struct fields. --- lib/pleroma/search/paradedb.ex | 74 +++++++++++++++++---------- test/pleroma/search/paradedb_test.exs | 14 +++-- 2 files changed, 58 insertions(+), 30 deletions(-) diff --git a/lib/pleroma/search/paradedb.ex b/lib/pleroma/search/paradedb.ex index 963d573e8..e5d3018d7 100644 --- a/lib/pleroma/search/paradedb.ex +++ b/lib/pleroma/search/paradedb.ex @@ -67,37 +67,43 @@ defmodule Pleroma.Search.ParadeDB do maybe_search_data = object_to_search_data(activity.object) if activity.data["type"] == "Create" and maybe_search_data do + dumped_activity_id = dump_activity_id(activity.id) actor_ap_id = activity.data["actor"] published_at = DateTime.from_unix!(maybe_search_data.published) - sql = - """ - INSERT INTO #{table()} (id, object_id, object_ap_id, actor_ap_id, content, published_at) - VALUES ($1, $2, $3, $4, $5, $6) - ON CONFLICT (id) DO UPDATE SET - object_id = EXCLUDED.object_id, - object_ap_id = EXCLUDED.object_ap_id, - actor_ap_id = EXCLUDED.actor_ap_id, - content = EXCLUDED.content, - published_at = EXCLUDED.published_at - """ + if is_nil(dumped_activity_id) do + Logger.error("ParadeDB add_to_index failed: invalid activity id #{inspect(activity.id)}") + :ok + else + sql = + """ + INSERT INTO #{table()} (id, object_id, object_ap_id, actor_ap_id, content, published_at) + VALUES ($1, $2, $3, $4, $5, $6) + ON CONFLICT (id) DO UPDATE SET + object_id = EXCLUDED.object_id, + object_ap_id = EXCLUDED.object_ap_id, + actor_ap_id = EXCLUDED.actor_ap_id, + content = EXCLUDED.content, + published_at = EXCLUDED.published_at + """ - params = [ - activity.id, - maybe_search_data.id, - maybe_search_data.ap, - actor_ap_id, - maybe_search_data.content, - published_at - ] + params = [ + dumped_activity_id, + maybe_search_data.id, + maybe_search_data.ap, + actor_ap_id, + maybe_search_data.content, + published_at + ] - case client_impl().query(sql, params) do - {:ok, _} -> - :ok + case client_impl().query(sql, params) do + {:ok, _} -> + :ok - {:error, error} -> - Logger.error("ParadeDB add_to_index failed: #{inspect(error)}") - {:error, error} + {:error, error} -> + Logger.error("ParadeDB add_to_index failed: #{inspect(error)}") + {:error, error} + end end else :ok @@ -139,7 +145,10 @@ defmodule Pleroma.Search.ParadeDB do end defp fetch_activities(ids, user, query, author) do - dumped_ids = Enum.map(ids, &FlakeId.from_string/1) + dumped_ids = + ids + |> Enum.map(&dump_activity_id/1) + |> Enum.reject(&is_nil/1) from(a in Activity, where: a.id in ^dumped_ids) |> Activity.with_preloaded_object() @@ -189,7 +198,7 @@ defmodule Pleroma.Search.ParadeDB do defp create_table_sql(table) do """ CREATE TABLE IF NOT EXISTS #{table} ( - id text PRIMARY KEY, + id uuid PRIMARY KEY, object_id bigint NOT NULL, object_ap_id text NOT NULL, actor_ap_id text, @@ -210,4 +219,15 @@ defmodule Pleroma.Search.ParadeDB do WITH (key_field = 'id') """ end + + defp dump_activity_id(<<_::binary-size(16)>> = uuid), do: uuid + + defp dump_activity_id(id) when is_binary(id) do + case FlakeId.Ecto.CompatType.dump(id) do + {:ok, <<_::binary-size(16)>> = uuid} -> uuid + _ -> nil + end + end + + defp dump_activity_id(_), do: nil end diff --git a/test/pleroma/search/paradedb_test.exs b/test/pleroma/search/paradedb_test.exs index c55e2aa47..3338c78ef 100644 --- a/test/pleroma/search/paradedb_test.exs +++ b/test/pleroma/search/paradedb_test.exs @@ -35,7 +35,9 @@ defmodule Pleroma.Search.ParadeDBTest do [activity_id, object_id, object_ap_id, actor_ap_id, content, published_at] = params - assert activity_id == activity.id + {:ok, dumped_activity_id} = FlakeId.Ecto.CompatType.dump(activity.id) + + assert activity_id == dumped_activity_id assert is_integer(object_id) assert is_binary(object_ap_id) assert actor_ap_id == activity.data["actor"] @@ -144,12 +146,18 @@ defmodule Pleroma.Search.ParadeDBTest do {:ok, activity1} = CommonAPI.post(user, %{status: "first swamp", visibility: "public"}) {:ok, activity2} = CommonAPI.post(user, %{status: "second swamp", visibility: "public"}) + activity1_id = activity1.id + activity2_id = activity2.id + + {:ok, dumped_activity1_id} = FlakeId.Ecto.CompatType.dump(activity1_id) + {:ok, dumped_activity2_id} = FlakeId.Ecto.CompatType.dump(activity2_id) + ClientMock |> expect(:query, fn sql, params -> assert sql =~ "SELECT id FROM pleroma_search_documents" assert ["swamp", _limit, _offset] = params - {:ok, %{rows: [[activity2.id], [activity1.id]]}} + {:ok, %{rows: [[dumped_activity2_id], [dumped_activity1_id]]}} end) Config @@ -161,7 +169,7 @@ defmodule Pleroma.Search.ParadeDBTest do "pleroma_search_documents" end) - assert [%{id: ^activity2.id}, %{id: ^activity1.id}] = + assert [%{id: ^activity2_id}, %{id: ^activity1_id}] = ParadeDB.search(nil, "swamp", limit: 40, offset: 0) end end From 74da3e01f7f0b27fcbbec313db363118c51388a8 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Thu, 5 Mar 2026 16:36:22 +0400 Subject: [PATCH 017/183] Harden ParadeDB search result safety Re-apply public Create visibility checks when resolving backend hits, improve UUID/FlakeId id parsing, and add tests for poisoned hits and UUID-string ids. Also tighten the docker integration test with localhost-only binding and config restoration. --- lib/pleroma/search/paradedb.ex | 41 ++++++++++-- .../search/paradedb_integration_test.exs | 13 +++- test/pleroma/search/paradedb_test.exs | 63 +++++++++++++++++++ 3 files changed, 111 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/search/paradedb.ex b/lib/pleroma/search/paradedb.ex index e5d3018d7..01cbc323f 100644 --- a/lib/pleroma/search/paradedb.ex +++ b/lib/pleroma/search/paradedb.ex @@ -6,6 +6,7 @@ defmodule Pleroma.Search.ParadeDB do @behaviour Pleroma.Search.SearchBackend require Logger + require Pleroma.Constants import Ecto.Query @@ -153,6 +154,7 @@ defmodule Pleroma.Search.ParadeDB do from(a in Activity, where: a.id in ^dumped_ids) |> Activity.with_preloaded_object() |> Activity.restrict_deactivated_users() + |> maybe_restrict_public(user) |> maybe_restrict_local(user) |> maybe_restrict_author(author) |> maybe_restrict_blocked(user) @@ -168,6 +170,25 @@ defmodule Pleroma.Search.ParadeDB do |> maybe_fetch(user, query) end + defp maybe_restrict_public(query, %Pleroma.User{}) do + intended_recipients = [ + Pleroma.Constants.as_public(), + Pleroma.Web.ActivityPub.Utils.as_local_public() + ] + + from(a in query, + where: fragment("?->>'type' = 'Create'", a.data), + where: fragment("? && ?", ^intended_recipients, a.recipients) + ) + end + + defp maybe_restrict_public(query, _user) do + from(a in query, + where: fragment("?->>'type' = 'Create'", a.data), + where: ^Pleroma.Constants.as_public() in a.recipients + ) + end + defp search_ids(query, limit, offset, author) do base_sql = "SELECT id FROM #{table()} WHERE content &&& $1" params = [query] @@ -186,7 +207,7 @@ defmodule Pleroma.Search.ParadeDB do sql = base_sql <> - " ORDER BY published_at DESC LIMIT $#{limit_arg} OFFSET $#{offset_arg}" + " ORDER BY published_at DESC, id DESC LIMIT $#{limit_arg} OFFSET $#{offset_arg}" params = params ++ [limit, offset] @@ -223,11 +244,23 @@ defmodule Pleroma.Search.ParadeDB do defp dump_activity_id(<<_::binary-size(16)>> = uuid), do: uuid defp dump_activity_id(id) when is_binary(id) do - case FlakeId.Ecto.CompatType.dump(id) do - {:ok, <<_::binary-size(16)>> = uuid} -> uuid - _ -> nil + case Ecto.UUID.dump(id) do + {:ok, <<_::binary-size(16)>> = uuid} -> + uuid + + :error -> + dump_flake_activity_id(id) end end defp dump_activity_id(_), do: nil + + defp dump_flake_activity_id(id) do + case FlakeId.Ecto.CompatType.dump(id) do + {:ok, <<_::binary-size(16)>> = uuid} -> uuid + _ -> nil + end + rescue + _ -> nil + end end diff --git a/test/pleroma/search/paradedb_integration_test.exs b/test/pleroma/search/paradedb_integration_test.exs index 1da131c6b..42d12c4d6 100644 --- a/test/pleroma/search/paradedb_integration_test.exs +++ b/test/pleroma/search/paradedb_integration_test.exs @@ -14,7 +14,7 @@ defmodule Pleroma.Search.ParadeDBIntegrationTest do alias Pleroma.Search.ParadeDB alias Pleroma.Web.CommonAPI - @image "paradedb/paradedb:latest" + @image System.get_env("PARADEDB_TEST_IMAGE", "paradedb/paradedb:latest") @db_user "postgres" @db_password "postgres" @db_name "paradedb" @@ -46,7 +46,7 @@ defmodule Pleroma.Search.ParadeDBIntegrationTest do "-e", "POSTGRES_DB=#{@db_name}", "-p", - "5432", + "127.0.0.1::5432", "-d", @image ]) @@ -75,9 +75,18 @@ defmodule Pleroma.Search.ParadeDBIntegrationTest do {:ok, _pid} = start_supervised({Pleroma.Search.ParadeDB.Repo, pool_size: 1}) + previous_table = Pleroma.Config.get([Pleroma.Search.ParadeDB, :table], nil) table = "pleroma_search_documents_it_#{System.unique_integer([:positive])}" Pleroma.Config.put([Pleroma.Search.ParadeDB, :table], table) + on_exit(fn -> + if is_nil(previous_table) do + Pleroma.Config.delete([Pleroma.Search.ParadeDB, :table]) + else + Pleroma.Config.put([Pleroma.Search.ParadeDB, :table], previous_table) + end + end) + {:ok, %{ container: container, diff --git a/test/pleroma/search/paradedb_test.exs b/test/pleroma/search/paradedb_test.exs index 3338c78ef..c9bb216df 100644 --- a/test/pleroma/search/paradedb_test.exs +++ b/test/pleroma/search/paradedb_test.exs @@ -172,5 +172,68 @@ defmodule Pleroma.Search.ParadeDBTest do assert [%{id: ^activity2_id}, %{id: ^activity1_id}] = ParadeDB.search(nil, "swamp", limit: 40, offset: 0) end + + test "search excludes non-public activities from backend hits" do + user = insert(:user) + + {:ok, public_activity} = + CommonAPI.post(user, %{status: "public swamp", visibility: "public"}) + + {:ok, private_activity} = + CommonAPI.post(user, %{status: "private swamp", visibility: "private"}) + + public_activity_id = public_activity.id + + {:ok, dumped_public_id} = FlakeId.Ecto.CompatType.dump(public_activity.id) + {:ok, dumped_private_id} = FlakeId.Ecto.CompatType.dump(private_activity.id) + + ClientMock + |> expect(:query, fn sql, params -> + assert sql =~ "SELECT id FROM pleroma_search_documents" + assert ["swamp", _limit, _offset] = params + + {:ok, %{rows: [[dumped_private_id], [dumped_public_id]]}} + end) + + Config + |> expect(:get, 2, fn + [Pleroma.Search.ParadeDB, :client_impl], nil -> + ClientMock + + [Pleroma.Search.ParadeDB, :table], "pleroma_search_documents" -> + "pleroma_search_documents" + end) + + assert [%{id: ^public_activity_id}] = ParadeDB.search(nil, "swamp", limit: 40, offset: 0) + end + + test "search supports backend ids returned as uuid strings" do + user = insert(:user) + + {:ok, activity} = CommonAPI.post(user, %{status: "uuid swamp", visibility: "public"}) + + activity_id = activity.id + {:ok, dumped_id} = FlakeId.Ecto.CompatType.dump(activity_id) + {:ok, uuid_string} = Ecto.UUID.load(dumped_id) + + ClientMock + |> expect(:query, fn sql, params -> + assert sql =~ "SELECT id FROM pleroma_search_documents" + assert ["swamp", _limit, _offset] = params + + {:ok, %{rows: [[uuid_string]]}} + end) + + Config + |> expect(:get, 2, fn + [Pleroma.Search.ParadeDB, :client_impl], nil -> + ClientMock + + [Pleroma.Search.ParadeDB, :table], "pleroma_search_documents" -> + "pleroma_search_documents" + end) + + assert [%{id: ^activity_id}] = ParadeDB.search(nil, "swamp", limit: 40, offset: 0) + end end end From 7efd296afbf49e5fb4ea8cdf659e1f0cdedd7de9 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Thu, 5 Mar 2026 17:12:03 +0400 Subject: [PATCH 018/183] Add fuzzy matching to ParadeDB search Make ParadeDB search typo-tolerant via configurable fuzzy distance and document the new setting. Add coverage for fuzzy query SQL and id parsing, and keep integration test stability under connection load. --- config/config.exs | 3 +- config/description.exs | 6 +++ docs/configuration/search.md | 5 +- lib/pleroma/search/paradedb.ex | 23 ++++++++- .../search/paradedb_integration_test.exs | 2 +- test/pleroma/search/paradedb_test.exs | 47 +++++++++++++++++-- 6 files changed, 79 insertions(+), 7 deletions(-) diff --git a/config/config.exs b/config/config.exs index 7e374b22b..01dd09a18 100644 --- a/config/config.exs +++ b/config/config.exs @@ -940,7 +940,8 @@ config :pleroma, Pleroma.Search.Meilisearch, config :pleroma, Pleroma.Search.ParadeDB, url: nil, - table: "pleroma_search_documents" + table: "pleroma_search_documents", + fuzzy_distance: 0 config :pleroma, Pleroma.Search.ParadeDB.Repo, pool_size: 10, diff --git a/config/description.exs b/config/description.exs index de454dfb2..9a00afdcd 100644 --- a/config/description.exs +++ b/config/description.exs @@ -3560,6 +3560,12 @@ config :pleroma, :config_description, [ type: :string, description: "Table name used for search documents.", suggestion: ["pleroma_search_documents"] + }, + %{ + key: :fuzzy_distance, + type: :integer, + description: "Edit-distance used for fuzzy token matching (0 disables fuzziness, max 2).", + suggestion: [0, 1, 2] } ] }, diff --git a/docs/configuration/search.md b/docs/configuration/search.md index 780904d10..c0d3f8462 100644 --- a/docs/configuration/search.md +++ b/docs/configuration/search.md @@ -160,7 +160,8 @@ To use it, set the search module to `Pleroma.Search.ParadeDB` and configure the > > config :pleroma, Pleroma.Search.ParadeDB, > url: System.get_env("PARADEDB_DATABASE_URL") || "postgres://postgres:postgres@127.0.0.1:5432/paradedb", -> table: "pleroma_search_documents" +> table: "pleroma_search_documents", +> fuzzy_distance: 0 Then, create the table and BM25 index once: @@ -178,4 +179,6 @@ After that, run an initial indexing pass to backfill existing posts: Note: Like the other external search backends, only public/unlisted Notes are indexed. +Set `fuzzy_distance` to `1` or `2` to allow typo-tolerant matching (`2` is the maximum ParadeDB supports). + Indexing runs via the Oban `search_indexing` queue, so ensure that queue is enabled (not paused) in your Oban configuration. diff --git a/lib/pleroma/search/paradedb.ex b/lib/pleroma/search/paradedb.ex index 01cbc323f..57e5599b5 100644 --- a/lib/pleroma/search/paradedb.ex +++ b/lib/pleroma/search/paradedb.ex @@ -35,6 +35,22 @@ defmodule Pleroma.Search.ParadeDB do Config.get([Pleroma.Search.ParadeDB, :table], @default_table) end + defp fuzzy_distance do + case Config.get([Pleroma.Search.ParadeDB, :fuzzy_distance], 0) do + value when is_integer(value) and value in 0..2 -> + value + + value when is_binary(value) -> + case Integer.parse(value) do + {parsed, ""} when parsed in 0..2 -> parsed + _ -> 0 + end + + _ -> + 0 + end + end + @impl true def create_index do with {:ok, _} <- client_impl().query("CREATE EXTENSION IF NOT EXISTS pg_search", []), @@ -190,7 +206,9 @@ defmodule Pleroma.Search.ParadeDB do end defp search_ids(query, limit, offset, author) do - base_sql = "SELECT id FROM #{table()} WHERE content &&& $1" + base_sql = + "SELECT id FROM #{table()} WHERE content &&& #{search_query_param_sql(fuzzy_distance())}" + params = [query] {base_sql, params} = @@ -216,6 +234,9 @@ defmodule Pleroma.Search.ParadeDB do end end + defp search_query_param_sql(0), do: "$1" + defp search_query_param_sql(distance), do: "$1::pdb.fuzzy(#{distance})" + defp create_table_sql(table) do """ CREATE TABLE IF NOT EXISTS #{table} ( diff --git a/test/pleroma/search/paradedb_integration_test.exs b/test/pleroma/search/paradedb_integration_test.exs index 42d12c4d6..743dcbaff 100644 --- a/test/pleroma/search/paradedb_integration_test.exs +++ b/test/pleroma/search/paradedb_integration_test.exs @@ -73,7 +73,7 @@ defmodule Pleroma.Search.ParadeDBIntegrationTest do end end) - {:ok, _pid} = start_supervised({Pleroma.Search.ParadeDB.Repo, pool_size: 1}) + {:ok, _pid} = start_supervised({Pleroma.Search.ParadeDB.Repo, pool_size: 5}) previous_table = Pleroma.Config.get([Pleroma.Search.ParadeDB, :table], nil) table = "pleroma_search_documents_it_#{System.unique_integer([:positive])}" diff --git a/test/pleroma/search/paradedb_test.exs b/test/pleroma/search/paradedb_test.exs index c9bb216df..c4fb57cb3 100644 --- a/test/pleroma/search/paradedb_test.exs +++ b/test/pleroma/search/paradedb_test.exs @@ -161,12 +161,15 @@ defmodule Pleroma.Search.ParadeDBTest do end) Config - |> expect(:get, 2, fn + |> expect(:get, 3, fn [Pleroma.Search.ParadeDB, :client_impl], nil -> ClientMock [Pleroma.Search.ParadeDB, :table], "pleroma_search_documents" -> "pleroma_search_documents" + + [Pleroma.Search.ParadeDB, :fuzzy_distance], 0 -> + 0 end) assert [%{id: ^activity2_id}, %{id: ^activity1_id}] = @@ -196,12 +199,15 @@ defmodule Pleroma.Search.ParadeDBTest do end) Config - |> expect(:get, 2, fn + |> expect(:get, 3, fn [Pleroma.Search.ParadeDB, :client_impl], nil -> ClientMock [Pleroma.Search.ParadeDB, :table], "pleroma_search_documents" -> "pleroma_search_documents" + + [Pleroma.Search.ParadeDB, :fuzzy_distance], 0 -> + 0 end) assert [%{id: ^public_activity_id}] = ParadeDB.search(nil, "swamp", limit: 40, offset: 0) @@ -225,15 +231,50 @@ defmodule Pleroma.Search.ParadeDBTest do end) Config - |> expect(:get, 2, fn + |> expect(:get, 3, fn [Pleroma.Search.ParadeDB, :client_impl], nil -> ClientMock [Pleroma.Search.ParadeDB, :table], "pleroma_search_documents" -> "pleroma_search_documents" + + [Pleroma.Search.ParadeDB, :fuzzy_distance], 0 -> + 0 end) assert [%{id: ^activity_id}] = ParadeDB.search(nil, "swamp", limit: 40, offset: 0) end + + test "search uses configured fuzzy distance" do + user = insert(:user) + + {:ok, activity} = CommonAPI.post(user, %{status: "runing shoes", visibility: "public"}) + + activity_id = activity.id + {:ok, dumped_id} = FlakeId.Ecto.CompatType.dump(activity_id) + + ClientMock + |> expect(:query, fn sql, params -> + assert sql =~ "SELECT id FROM pleroma_search_documents" + assert sql =~ "$1::pdb.fuzzy(1)" + assert ["running shoes", _limit, _offset] = params + + {:ok, %{rows: [[dumped_id]]}} + end) + + Config + |> expect(:get, 3, fn + [Pleroma.Search.ParadeDB, :client_impl], nil -> + ClientMock + + [Pleroma.Search.ParadeDB, :table], "pleroma_search_documents" -> + "pleroma_search_documents" + + [Pleroma.Search.ParadeDB, :fuzzy_distance], 0 -> + 1 + end) + + assert [%{id: ^activity_id}] = ParadeDB.search(nil, "running shoes", limit: 40, offset: 0) + end end end From 4e65bbf13469d8147dc780c7e655fefdb12bdebf Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Thu, 5 Mar 2026 17:34:48 +0400 Subject: [PATCH 019/183] Fix ParadeDB fuzzy query parameter casting Cast the bound search parameter through text before so Postgrex can handle the prepared statement type inference. Add matching test assertion updates. --- lib/pleroma/search/paradedb.ex | 2 +- test/pleroma/search/paradedb_test.exs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/search/paradedb.ex b/lib/pleroma/search/paradedb.ex index 57e5599b5..a6649a894 100644 --- a/lib/pleroma/search/paradedb.ex +++ b/lib/pleroma/search/paradedb.ex @@ -235,7 +235,7 @@ defmodule Pleroma.Search.ParadeDB do end defp search_query_param_sql(0), do: "$1" - defp search_query_param_sql(distance), do: "$1::pdb.fuzzy(#{distance})" + defp search_query_param_sql(distance), do: "($1::text)::pdb.fuzzy(#{distance})" defp create_table_sql(table) do """ diff --git a/test/pleroma/search/paradedb_test.exs b/test/pleroma/search/paradedb_test.exs index c4fb57cb3..3a3b8298f 100644 --- a/test/pleroma/search/paradedb_test.exs +++ b/test/pleroma/search/paradedb_test.exs @@ -256,7 +256,7 @@ defmodule Pleroma.Search.ParadeDBTest do ClientMock |> expect(:query, fn sql, params -> assert sql =~ "SELECT id FROM pleroma_search_documents" - assert sql =~ "$1::pdb.fuzzy(1)" + assert sql =~ "($1::text)::pdb.fuzzy(1)" assert ["running shoes", _limit, _offset] = params {:ok, %{rows: [[dumped_id]]}} From cae3c9a980945f107b2cd6a321b0d1d1e0d243ea Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Thu, 5 Mar 2026 17:39:34 +0400 Subject: [PATCH 020/183] Add ParadeDB fuzzy integration coverage Extend the docker-backed ParadeDB integration test to verify typo-tolerant status search with fuzzy_distance enabled, covering prepared-parameter fuzzy casts against a real backend. --- test/pleroma/search/paradedb_integration_test.exs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/test/pleroma/search/paradedb_integration_test.exs b/test/pleroma/search/paradedb_integration_test.exs index 743dcbaff..2687260cf 100644 --- a/test/pleroma/search/paradedb_integration_test.exs +++ b/test/pleroma/search/paradedb_integration_test.exs @@ -130,6 +130,20 @@ defmodule Pleroma.Search.ParadeDBIntegrationTest do refute private.id in ids assert List.first(ids) == a2.id + previous_fuzzy_distance = Pleroma.Config.get([Pleroma.Search.ParadeDB, :fuzzy_distance], 0) + Pleroma.Config.put([Pleroma.Search.ParadeDB, :fuzzy_distance], 1) + + on_exit(fn -> + Pleroma.Config.put([Pleroma.Search.ParadeDB, :fuzzy_distance], previous_fuzzy_distance) + end) + + fuzzy_results = ParadeDB.search(nil, "paradedb integraion", limit: 40, offset: 0) + fuzzy_ids = Enum.map(fuzzy_results, & &1.id) + + assert a1.id in fuzzy_ids + assert a2.id in fuzzy_ids + refute private.id in fuzzy_ids + # Removing an object should remove it from the ParadeDB index. assert :ok = ParadeDB.remove_from_index(a2.object) From 0e0931a3c23d8c966aef8ded80e82da6f3c4205e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?nicole=20miko=C5=82ajczyk?= Date: Fri, 19 Sep 2025 16:55:40 +0200 Subject: [PATCH 021/183] MastoAPI: /api/v1/admin/accounts, /api/v1/admin/reports MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: nicole mikołajczyk --- changelog.d/mastodon-admin.add | 1 + lib/pleroma/moderation_log.ex | 10 + lib/pleroma/user.ex | 6 + lib/pleroma/user/query.ex | 19 +- lib/pleroma/web/api_spec.ex | 7 + .../mastodon_admin/account_operation.ex | 258 ++++++++++++++++++ .../mastodon_admin/report_operation.ex | 150 ++++++++++ .../admin/controllers/account_controller.ex | 220 +++++++++++++++ .../admin/controllers/report_controller.ex | 107 ++++++++ .../mastodon_api/admin/views/account_view.ex | 64 +++++ .../mastodon_api/admin/views/report_view.ex | 56 ++++ lib/pleroma/web/router.ex | 40 +++ .../controllers/account_controller_test.exs | 175 ++++++++++++ .../controllers/report_controller_test.exs | 118 ++++++++ 14 files changed, 1230 insertions(+), 1 deletion(-) create mode 100644 changelog.d/mastodon-admin.add create mode 100644 lib/pleroma/web/api_spec/operations/mastodon_admin/account_operation.ex create mode 100644 lib/pleroma/web/api_spec/operations/mastodon_admin/report_operation.ex create mode 100644 lib/pleroma/web/mastodon_api/admin/controllers/account_controller.ex create mode 100644 lib/pleroma/web/mastodon_api/admin/controllers/report_controller.ex create mode 100644 lib/pleroma/web/mastodon_api/admin/views/account_view.ex create mode 100644 lib/pleroma/web/mastodon_api/admin/views/report_view.ex create mode 100644 test/pleroma/web/mastodon_api/admin/controllers/account_controller_test.exs create mode 100644 test/pleroma/web/mastodon_api/admin/controllers/report_controller_test.exs diff --git a/changelog.d/mastodon-admin.add b/changelog.d/mastodon-admin.add new file mode 100644 index 000000000..64f2ca987 --- /dev/null +++ b/changelog.d/mastodon-admin.add @@ -0,0 +1 @@ +Implement basics of Mastodon admin API \ No newline at end of file diff --git a/lib/pleroma/moderation_log.ex b/lib/pleroma/moderation_log.ex index 90219312c..260c139ec 100644 --- a/lib/pleroma/moderation_log.ex +++ b/lib/pleroma/moderation_log.ex @@ -346,6 +346,16 @@ defmodule Pleroma.ModerationLog do "@#{actor_nickname} approved users: #{users_to_nicknames_string(users)}" end + def get_log_entry_message(%ModerationLog{ + data: %{ + "actor" => %{"nickname" => actor_nickname}, + "action" => "reject", + "subject" => users + } + }) do + "@#{actor_nickname} rejected users: #{users_to_nicknames_string(users)}" + end + def get_log_entry_message(%ModerationLog{ data: %{ "actor" => %{"nickname" => actor_nickname}, diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 75da41da9..4b421f389 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -1964,6 +1964,12 @@ defmodule Pleroma.User do def approve(%User{} = user), do: {:ok, user} + def reject(%User{is_approved: false} = user) do + delete(user) + end + + def reject(%User{} = _user), do: {:error, "User is approved"} + def confirm(users) when is_list(users) do Repo.transaction(fn -> Enum.map(users, fn user -> diff --git a/lib/pleroma/user/query.ex b/lib/pleroma/user/query.ex index cd9586452..b877b52b2 100644 --- a/lib/pleroma/user/query.ex +++ b/lib/pleroma/user/query.ex @@ -63,7 +63,8 @@ defmodule Pleroma.User.Query do limit: pos_integer(), actor_types: [String.t()], birthday_day: pos_integer(), - birthday_month: pos_integer() + birthday_month: pos_integer(), + staff: boolean() } | map() @@ -179,6 +180,10 @@ defmodule Pleroma.User.Query do defp compose_query({:external, _}, query), do: location_query(query, false) + defp compose_query({:active, false}, query) do + where(query, [u], u.is_active == false or u.is_approved == false or u.is_confirmed == false) + end + defp compose_query({:active, _}, query) do where(query, [u], u.is_active == true) |> where([u], u.is_approved == true) @@ -202,6 +207,10 @@ defmodule Pleroma.User.Query do where(query, [u], u.is_confirmed != ^bool) end + defp compose_query({:need_approval, false}, query) do + where(query, [u], u.is_approved == true) + end + defp compose_query({:need_approval, _}, query) do where(query, [u], u.is_approved == false) end @@ -285,6 +294,14 @@ defmodule Pleroma.User.Query do |> where([u], fragment("date_part('month', ?)", u.birthday) == ^month) end + defp compose_query({:staff, false}, query) do + where(query, [u], u.is_admin == false and u.is_moderator == false) + end + + defp compose_query({:staff, _}, query) do + where(query, [u], u.is_admin == true or u.is_moderator == true) + end + defp compose_query(_unsupported_param, query), do: query defp location_query(query, local) do diff --git a/lib/pleroma/web/api_spec.ex b/lib/pleroma/web/api_spec.ex index 5006167ea..ff6363603 100644 --- a/lib/pleroma/web/api_spec.ex +++ b/lib/pleroma/web/api_spec.ex @@ -108,6 +108,13 @@ defmodule Pleroma.Web.ApiSpec do "Announcement management" ] }, + %{ + "name" => "Administration (Mastodon API)", + "tags" => [ + "User administration (Mastodon API)", + "Report management (Mastodon API)" + ] + }, %{"name" => "Applications", "tags" => ["Applications", "Push subscriptions"]}, %{ "name" => "Current account", diff --git a/lib/pleroma/web/api_spec/operations/mastodon_admin/account_operation.ex b/lib/pleroma/web/api_spec/operations/mastodon_admin/account_operation.ex new file mode 100644 index 000000000..23dae0306 --- /dev/null +++ b/lib/pleroma/web/api_spec/operations/mastodon_admin/account_operation.ex @@ -0,0 +1,258 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ApiSpec.MastodonAdmin.AccountOperation do + alias OpenApiSpex.Operation + alias OpenApiSpex.Schema + alias Pleroma.Web.ApiSpec.Schemas.Account + alias Pleroma.Web.ApiSpec.Schemas.ApiError + alias Pleroma.Web.ApiSpec.Schemas.FlakeID + + import Pleroma.Web.ApiSpec.Helpers + + def open_api_operation(action) do + operation = String.to_existing_atom("#{action}_operation") + apply(__MODULE__, operation, []) + end + + def index_operation do + %Operation{ + tags: ["User administration (Mastodon API)"], + summary: "View accounts by criteria", + operationId: "MastodonAdmin.AccountController.index", + description: "View accounts matching certain criteria for filtering, up to 40 at a time.", + security: [%{"oAuth" => ["admin:read:accounts"]}], + parameters: + [ + Operation.parameter(:local, :query, :boolean, "Filter for local accounts?"), + Operation.parameter(:remote, :query, :boolean, "Filter for remote accounts?"), + Operation.parameter( + :by_domain, + :query, + :string, + "Filter by the given domain (not implemented yet)" + ), + Operation.parameter( + :active, + :query, + :boolean, + "Filter for currently active accounts??" + ), + Operation.parameter( + :pending, + :query, + :boolean, + "Filter for currently pending accounts?" + ), + Operation.parameter( + :disabled, + :query, + :boolean, + "Filter for currently disabled accounts?" + ), + Operation.parameter( + :sensitized, + :query, + :boolean, + "Filter for currently sensitized accounts? (not implemented yet)" + ), + Operation.parameter( + :silenced, + :query, + :boolean, + "Filter for currently silenced accounts? (not implemented yet)" + ), + Operation.parameter( + :suspended, + :query, + :boolean, + "Filter for currently suspended accounts? (not implemented yet)" + ), + Operation.parameter(:username, :query, :string, "Username to search for"), + Operation.parameter(:display_name, :query, :string, "Display name to search for"), + Operation.parameter(:email, :query, :string, "Lookup a user with this email"), + Operation.parameter( + :ip, + :query, + :string, + "Lookup users by this IP address (not implemented yet)" + ), + Operation.parameter(:staff, :query, :boolean, "Filter for staff accounts?") + ] ++ + pagination_params(), + responses: %{ + 200 => + Operation.response("Account", "application/json", %Schema{ + title: "ArrayOfAccounts", + type: :array, + items: account() + }), + 401 => Operation.response("Error", "application/json", ApiError) + } + } + end + + def show_operation do + %Operation{ + tags: ["User administration (Mastodon API)"], + summary: "View a specific account", + operationId: "MastodonAdmin.AccountController.show", + description: "View admin-level information about the given account.", + security: [%{"oAuth" => ["admin:read:accounts"]}], + parameters: [ + Operation.parameter(:id, :path, :string, "ID of the account") + ], + responses: %{ + 200 => Operation.response("Account", "application/json", account()), + 401 => Operation.response("Error", "application/json", ApiError), + 404 => Operation.response("Error", "application/json", ApiError) + } + } + end + + def account_action_operation do + %Operation{ + tags: ["User administration (Mastodon API)"], + summary: "Perform an action against an account", + operationId: "MastodonAdmin.AccountController.account_action", + description: + "Perform an action against an account and log this action in the moderation history.", + security: [%{"oAuth" => ["admin:write:accounts"]}], + parameters: [ + Operation.parameter(:id, :path, :string, "ID of the account") + ], + requestBody: + request_body( + "Parameters", + %Schema{ + type: :object, + properties: %{ + type: %Schema{ + type: :string, + enum: ["none", "disable", "sensitive", "silence", "suspend"] + }, + report_id: %Schema{ + type: :string, + nullable: true, + description: "ID of an associated report that caused this action to be taken" + } + } + }, + required: true + ), + responses: %{ + 204 => no_content_response(), + 401 => Operation.response("Error", "application/json", ApiError) + } + } + end + + def delete_operation do + %Operation{ + tags: ["User administration (Mastodon API)"], + summary: "Delete a specific account", + operationId: "MastodonAdmin.AccountController.delete", + description: "Delete the given account.", + security: [%{"oAuth" => ["admin:write:accounts"]}], + parameters: [ + Operation.parameter(:id, :path, :string, "ID of the account") + ], + responses: %{ + 200 => Operation.response("Account", "application/json", account()), + 401 => Operation.response("Error", "application/json", ApiError), + 404 => Operation.response("Error", "application/json", ApiError) + } + } + end + + def enable_operation do + %Operation{ + tags: ["User administration (Mastodon API)"], + summary: "Re-enable account", + operationId: "MastodonAdmin.AccountController.enable", + description: "Re-enable a local account whose login is currently disabled.", + security: [%{"oAuth" => ["admin:write:accounts"]}], + parameters: [ + Operation.parameter(:id, :path, :string, "ID of the account") + ], + responses: %{ + 200 => Operation.response("Account", "application/json", account()), + 401 => Operation.response("Error", "application/json", ApiError), + 404 => Operation.response("Error", "application/json", ApiError) + } + } + end + + def approve_operation do + %Operation{ + tags: ["User administration (Mastodon API)"], + summary: "Approve pending account", + operationId: "MastodonAdmin.AccountController.approve", + description: "Approve the given local account if it is currently pending approval.", + parameters: [ + Operation.parameter(:id, :path, :string, "ID of the account") + ], + responses: %{ + 200 => Operation.response("Account", "application/json", account()), + 401 => Operation.response("Error", "application/json", ApiError), + 404 => Operation.response("Error", "application/json", ApiError) + } + } + end + + def reject_operation do + %Operation{ + tags: ["User administration (Mastodon API)"], + summary: "Reject pending account", + operationId: "MastodonAdmin.AccountController.reject", + description: "Reject the given local account if it is currently pending approval.", + parameters: [ + Operation.parameter(:id, :path, :string, "ID of the account") + ], + responses: %{ + 200 => Operation.response("Account", "application/json", account()), + 400 => Operation.response("Error", "application/json", ApiError), + 401 => Operation.response("Error", "application/json", ApiError), + 404 => Operation.response("Error", "application/json", ApiError) + } + } + end + + def account do + %Schema{ + title: "Admin::Account", + description: "Admin-level information about a given account.", + type: :object, + properties: %{ + id: FlakeID, + username: %Schema{type: :string}, + domain: %Schema{type: :string, nullable: true}, + created_at: %Schema{type: :string, format: "date-time"}, + email: %Schema{type: :string, format: "email", nullable: true}, + ip: %Schema{type: :string, nullable: true}, + ips: %Schema{ + type: :array, + items: %Schema{ + type: :object, + properties: %{ + ip: %Schema{type: :string}, + used_at: %Schema{type: :string, format: "date-time"} + } + } + }, + locale: %Schema{type: :string, format: "date-time", nullable: true}, + invite_request: %Schema{type: :string, format: "date-time", nullable: true}, + role: %Schema{type: :string, nullable: true}, + confirmed: %Schema{type: :boolean}, + approved: %Schema{type: :boolean}, + disabled: %Schema{type: :boolean}, + silenced: %Schema{type: :boolean, nullable: true}, + suspended: %Schema{type: :boolean, nullable: true}, + account: Account, + created_by_application_id: %Schema{type: :string, nullable: true}, + invited_by_account_id: %Schema{type: :string, nullable: true} + } + } + end +end diff --git a/lib/pleroma/web/api_spec/operations/mastodon_admin/report_operation.ex b/lib/pleroma/web/api_spec/operations/mastodon_admin/report_operation.ex new file mode 100644 index 000000000..274ce3bd9 --- /dev/null +++ b/lib/pleroma/web/api_spec/operations/mastodon_admin/report_operation.ex @@ -0,0 +1,150 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ApiSpec.MastodonAdmin.ReportOperation do + alias OpenApiSpex.Operation + alias OpenApiSpex.Schema + alias Pleroma.Web.ApiSpec.MastodonAdmin.AccountOperation + alias Pleroma.Web.ApiSpec.Schemas.ApiError + alias Pleroma.Web.ApiSpec.Schemas.FlakeID + alias Pleroma.Web.ApiSpec.Schemas.Status + + import Pleroma.Web.ApiSpec.Helpers + + def open_api_operation(action) do + operation = String.to_existing_atom("#{action}_operation") + apply(__MODULE__, operation, []) + end + + def index_operation do + %Operation{ + tags: ["Report management (Mastodon API)"], + summary: "View all reports", + operationId: "MastodonAdmin.ReportController.index", + description: + "View all reports. Pagination may be done with HTTP Link header in the response.", + security: [%{"oAuth" => ["admin:read:reports"]}], + parameters: + [ + Operation.parameter(:resolved, :query, :boolean, "Filter for resolved reports"), + Operation.parameter(:account_id, :query, :string, "Filter by author account id"), + Operation.parameter( + :target_account_id, + :query, + :string, + "Filter by report target account id (not implemented)" + ) + ] ++ + pagination_params(), + responses: %{ + 200 => + Operation.response("Reports", "application/json", %Schema{ + title: "ArrayOfReports", + type: :array, + items: report() + }), + 401 => Operation.response("Error", "application/json", ApiError) + } + } + end + + def show_operation do + %Operation{ + tags: ["Report management (Mastodon API)"], + summary: "View a single report", + operationId: "MastodonAdmin.ReportController.show", + description: "View information about the report with the given ID.", + security: [%{"oAuth" => ["admin:read:reports"]}], + parameters: [ + Operation.parameter(:id, :path, :string, "ID of the report") + ], + responses: %{ + 200 => Operation.response("Report", "application/json", report()), + 401 => Operation.response("Error", "application/json", ApiError), + 404 => Operation.response("Error", "application/json", ApiError) + } + } + end + + def resolve_operation do + %Operation{ + tags: ["Report management (Mastodon API)"], + summary: "Mark as resolved", + operationId: "MastodonAdmin.ReportController.resolve", + description: "Mark a report as resolved with no further action taken.", + security: [%{"oAuth" => ["admin:write:reports"]}], + parameters: [ + Operation.parameter(:id, :path, :string, "ID of the report") + ], + responses: %{ + 200 => Operation.response("Report", "application/json", report()), + 400 => Operation.response("Error", "application/json", ApiError), + 401 => Operation.response("Error", "application/json", ApiError) + } + } + end + + def reopen_operation do + %Operation{ + tags: ["Report management (Mastodon API)"], + summary: "Re-open report", + operationId: "MastodonAdmin.ReportController.reopen", + description: "Reopen a currently closed report.", + security: [%{"oAuth" => ["admin:write:reports"]}], + parameters: [ + Operation.parameter(:id, :path, :string, "ID of the report") + ], + responses: %{ + 200 => Operation.response("Report", "application/json", report()), + 400 => Operation.response("Error", "application/json", ApiError), + 401 => Operation.response("Error", "application/json", ApiError) + } + } + end + + defp report do + %Schema{ + title: "Admin::Report", + type: :object, + properties: %{ + id: FlakeID, + action_taken: %Schema{type: :boolean}, + action_taken_at: %Schema{type: :string, format: "date-time", nullable: true}, + category: %Schema{type: :string, enum: ["spam", "violation", "other"]}, + comment: %Schema{type: :string, nullable: true}, + forwarded: %Schema{type: :boolean, nullable: true}, + created_at: %Schema{type: :string, format: "date-time"}, + updated_at: %Schema{type: :string, format: "date-time"}, + account: AccountOperation.account(), + target_account: AccountOperation.account(), + assigned_account: %Schema{ + oneOf: [ + AccountOperation.account(), + %Schema{type: :null} + ] + }, + action_taken_by_account: %Schema{ + oneOf: [ + AccountOperation.account(), + %Schema{type: :null} + ] + }, + statuses: %Schema{ + type: :array, + items: Status + }, + rules: %Schema{ + type: :array, + items: %Schema{ + type: :object, + properties: %{ + id: %Schema{type: :integer}, + text: %Schema{type: :string} + } + } + } + } + } + end +end diff --git a/lib/pleroma/web/mastodon_api/admin/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/admin/controllers/account_controller.ex new file mode 100644 index 000000000..a1eac7fe3 --- /dev/null +++ b/lib/pleroma/web/mastodon_api/admin/controllers/account_controller.ex @@ -0,0 +1,220 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.MastodonAPI.Admin.AccountController do + use Pleroma.Web, :controller + + import Pleroma.Web.ControllerHelper, + only: [ + add_link_headers: 2, + assign_account_by_id: 2, + json_response: 3 + ] + + alias Pleroma.Activity + alias Pleroma.ModerationLog + alias Pleroma.Pagination + alias Pleroma.User + alias Pleroma.Web.ActivityPub.Builder + alias Pleroma.Web.ActivityPub.Pipeline + alias Pleroma.Web.CommonAPI + alias Pleroma.Web.Plugs.OAuthScopesPlug + + @filter_params ~W( + local external active needing_approval deactivated nickname name email staff + ) + + plug(Pleroma.Web.ApiSpec.CastAndValidate) + + plug( + OAuthScopesPlug, + %{scopes: ["admin:read:accounts"]} + when action in [:index, :show] + ) + + plug( + OAuthScopesPlug, + %{scopes: ["admin:write:accounts"]} + when action in [ + :delete, + :enable, + :account_action, + :approve, + :reject + ] + ) + + plug( + :assign_account_by_id + when action in [ + :show, + :delete, + :enable, + :account_action, + :approve, + :reject + ] + ) + + defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.MastodonAdmin.AccountOperation + + def index(conn, params) do + users = + params + |> build_criteria() + |> User.Query.build() + |> Pagination.fetch_paginated(params) + + conn + |> add_link_headers(users) + |> render("index.json", users: users) + end + + def show(%{assigns: %{user: _admin, account: user}} = conn, _params) do + render(conn, "show.json", user: user) + end + + def account_action( + %{assigns: %{user: admin, account: user}, body_params: %{type: type} = body_params} = + conn, + _params + ) do + {:ok, _user} = handle_account_action(user, admin, type) + + resolve_report(admin, body_params) + + json_response(conn, :no_content, "") + end + + def delete(%{assigns: %{user: admin, account: user}} = conn, _params) do + {:ok, delete_data, _} = Builder.delete(admin, user.ap_id) + Pipeline.common_pipeline(delete_data, local: true) + + ModerationLog.insert_log(%{ + actor: admin, + subject: [user], + action: "delete" + }) + + render(conn, "show.json", user: user) + end + + def enable(%{assigns: %{user: admin, account: user}} = conn, _params) do + {:ok, user} = User.set_activation(user, true) + + ModerationLog.insert_log(%{ + actor: admin, + subject: [user], + action: "activate" + }) + + render(conn, "show.json", user: user) + end + + def approve(%{assigns: %{user: admin, account: user}} = conn, _params) do + {:ok, user} = User.approve(user) + + ModerationLog.insert_log(%{ + actor: admin, + subject: [user], + action: "approve" + }) + + render(conn, "show.json", user: user) + end + + def reject(%{assigns: %{user: admin, account: user}} = conn, _params) do + with {:ok, _} <- User.reject(user) do + ModerationLog.insert_log(%{ + actor: admin, + subject: [user], + action: "reject" + }) + + render(conn, "show.json", user: user) + else + {:error, error} -> + json_response(conn, :bad_request, %{error: error}) + end + end + + defp handle_account_action(%User{local: true} = user, admin, "disable") do + ModerationLog.insert_log(%{ + actor: admin, + subject: [user], + action: "deactivate" + }) + + User.set_activation(user, false) + end + + defp handle_account_action(user, _admin, _type) do + {:ok, user} + end + + defp build_criteria(params) do + %{} + |> maybe_filter_local(params) + |> maybe_filter_external(params) + |> maybe_filter_active(params) + |> maybe_filter_needing_approval(params) + |> maybe_filter_deactivated(params) + |> maybe_filter_nickname(params) + |> maybe_filter_name(params) + |> maybe_filter_email(params) + |> maybe_filter_staff(params) + end + + defp maybe_filter_local(criteria, %{local: true} = _params), + do: Map.put(criteria, :local, true) + + defp maybe_filter_local(criteria, %{local: false} = _params), + do: Map.put(criteria, :external, true) + + defp maybe_filter_external(criteria, %{remote: true} = _params), + do: Map.put(criteria, :external, true) + + defp maybe_filter_external(criteria, %{remote: false} = _params), + do: Map.put(criteria, :local, true) + + defp maybe_filter_active(criteria, %{active: active} = _params), + do: Map.put(criteria, :active, active) + + defp maybe_filter_needing_approval(criteria, %{pending: need_approval} = _params), + do: Map.put(criteria, :need_approval, need_approval) + + defp maybe_filter_deactivated(criteria, %{disabled: deactivated} = _params), + do: Map.put(criteria, :deactivated, deactivated) + + defp maybe_filter_nickname(criteria, %{username: nickname} = _params), + do: Map.put(criteria, :nickname, nickname) + + defp maybe_filter_name(criteria, %{display_name: name} = _params), + do: Map.put(criteria, :name, name) + + defp maybe_filter_email(criteria, %{email: email} = _params), + do: Map.put(criteria, :email, email) + + defp maybe_filter_staff(criteria, %{staff: staff} = _params), + do: Map.put(criteria, :staff, staff) + + for filter_param <- @filter_params do + defp unquote(:"maybe_filter_#{filter_param}")(criteria, _params), do: criteria + end + + defp resolve_report(admin, %{report_id: id}) do + with {:ok, activity} <- CommonAPI.update_report_state(id, "resolved"), + report <- Activity.get_by_id_with_user_actor(activity.id) do + ModerationLog.insert_log(%{ + action: "report_update", + actor: admin, + subject: activity, + subject_actor: report.user_actor + }) + end + end + + defp resolve_report(_admin, _params) do + end +end diff --git a/lib/pleroma/web/mastodon_api/admin/controllers/report_controller.ex b/lib/pleroma/web/mastodon_api/admin/controllers/report_controller.ex new file mode 100644 index 000000000..217a97cc9 --- /dev/null +++ b/lib/pleroma/web/mastodon_api/admin/controllers/report_controller.ex @@ -0,0 +1,107 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.MastodonAPI.Admin.ReportController do + use Pleroma.Web, :controller + + import Pleroma.Web.ControllerHelper, + only: [ + add_link_headers: 2, + json_response: 3 + ] + + alias Pleroma.Activity + alias Pleroma.ModerationLog + alias Pleroma.Pagination + alias Pleroma.User + alias Pleroma.Web.ActivityPub.ActivityPub + alias Pleroma.Web.AdminAPI.Report + alias Pleroma.Web.CommonAPI + alias Pleroma.Web.Plugs.OAuthScopesPlug + + plug(Pleroma.Web.ApiSpec.CastAndValidate) + + plug(OAuthScopesPlug, %{scopes: ["admin:read:reports"]} when action in [:index, :show]) + + plug(OAuthScopesPlug, %{scopes: ["admin:write:reports"]} when action in [:resolve, :reopen]) + + defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.MastodonAdmin.ReportOperation + + def index(conn, params) do + opts = + %{} + |> Map.put(:type, "Flag") + |> Map.put(:skip_preload, true) + |> Map.put(:preload_report_notes, true) + |> Map.put(:total, true) + |> restrict_state(params) + |> restrict_actor(params) + + reports = + ActivityPub.fetch_activities_query([], opts) + |> Pagination.fetch_paginated(params) + + conn + |> add_link_headers(reports) + |> render("index.json", reports: reports) + end + + def show(conn, %{id: id}) do + with %Activity{} = report <- Activity.get_report(id) do + render(conn, "show.json", Report.extract_report_info(report)) + else + _ -> {:error, :not_found} + end + end + + def resolve(%{assigns: %{user: admin}} = conn, %{id: id}) do + with {:ok, activity} <- CommonAPI.update_report_state(id, "resolved"), + report <- Activity.get_by_id_with_user_actor(activity.id) do + ModerationLog.insert_log(%{ + action: "report_update", + actor: admin, + subject: activity, + subject_actor: report.user_actor + }) + + render(conn, "show.json", Report.extract_report_info(activity)) + else + {:error, error} -> + json_response(conn, :bad_request, %{error: error}) + end + end + + def reopen(%{assigns: %{user: admin}} = conn, %{id: id}) do + with {:ok, activity} <- CommonAPI.update_report_state(id, "open"), + report <- Activity.get_by_id_with_user_actor(activity.id) do + ModerationLog.insert_log(%{ + action: "report_update", + actor: admin, + subject: activity, + subject_actor: report.user_actor + }) + + render(conn, "show.json", Report.extract_report_info(report)) + else + {:error, error} -> + json_response(conn, :bad_request, %{error: error}) + end + end + + defp restrict_state(opts, %{resolved: true}), do: Map.put(opts, :state, "resolved") + + defp restrict_state(opts, %{resolved: false}), do: Map.put(opts, :state, "open") + + defp restrict_state(opts, _params), do: opts + + defp restrict_actor(opts, %{account_id: actor}) do + with %User{ap_id: ap_id} <- User.get_by_id(actor) do + Map.put(opts, :actor_id, ap_id) + else + _ -> Map.put(opts, :actor_id, actor) + end + end + + defp restrict_actor(opts, _params), do: opts +end diff --git a/lib/pleroma/web/mastodon_api/admin/views/account_view.ex b/lib/pleroma/web/mastodon_api/admin/views/account_view.ex new file mode 100644 index 000000000..20ccfd51c --- /dev/null +++ b/lib/pleroma/web/mastodon_api/admin/views/account_view.ex @@ -0,0 +1,64 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.MastodonAPI.Admin.AccountView do + use Pleroma.Web, :view + + alias Pleroma.User + alias Pleroma.Web.CommonAPI.Utils + alias Pleroma.Web.MastodonAPI + + def render("index.json", %{users: users}) do + render_many(users, __MODULE__, "show.json", as: :user) + end + + def render("show.json", %{user: user}) do + account = + MastodonAPI.AccountView.render("show.json", %{user: user, skip_visibility_check: true}) + + %{ + id: user.id, + username: username_from_nickname(user.nickname), + domain: domain_from_nickname(user.nickname), + created_at: Utils.to_masto_date(user.inserted_at), + email: user.email, + ip: nil, + ips: [], + locale: nil, + invite_request: user.registration_reason, + role: role(user), + confirmed: user.is_confirmed, + approved: user.is_approved, + disabled: !user.is_active, + silenced: nil, + suspended: nil, + account: account + } + end + + defp username_from_nickname(string) when is_binary(string) do + hd(String.split(string, "@")) + end + + defp username_from_nickname(_), do: nil + + defp domain_from_nickname(string) when is_binary(string) do + String.split(string, "@") + |> Enum.at(1, nil) + end + + defp domain_from_nickname(_), do: nil + + defp role(%User{is_admin: true}) do + "admin" + end + + defp role(%User{is_moderator: true}) do + "moderator" + end + + defp role(_user) do + nil + end +end diff --git a/lib/pleroma/web/mastodon_api/admin/views/report_view.ex b/lib/pleroma/web/mastodon_api/admin/views/report_view.ex new file mode 100644 index 000000000..664f58b03 --- /dev/null +++ b/lib/pleroma/web/mastodon_api/admin/views/report_view.ex @@ -0,0 +1,56 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.MastodonAPI.Admin.ReportView do + use Pleroma.Web, :view + + alias Pleroma.HTML + alias Pleroma.Web.AdminAPI.Report + alias Pleroma.Web.CommonAPI.Utils + alias Pleroma.Web.MastodonAPI.Admin.AccountView + alias Pleroma.Web.MastodonAPI.StatusView + + def render("index.json", %{reports: reports}) do + reports + |> Enum.map(&Report.extract_report_info/1) + |> Enum.map(&render(__MODULE__, "show.json", &1)) + end + + def render("show.json", %{ + report: report, + user: account, + account: target_account, + statuses: statuses + }) do + created_at = Utils.to_masto_date(report.data["published"]) + + content = + unless is_nil(report.data["content"]) do + HTML.filter_tags(report.data["content"]) + else + nil + end + + %{ + id: report.id, + action_taken: report.data["state"] != "open", + action_taken_at: nil, + category: "other", + comment: content, + forwarded: nil, + created_at: created_at, + updated_at: created_at, + account: AccountView.render("show.json", %{user: account}), + target_account: AccountView.render("show.json", %{user: target_account}), + assigned_account: nil, + action_taken_by_account: nil, + statuses: + StatusView.render("index.json", %{ + activities: statuses, + as: :activity + }), + rules: [] + } + end +end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 20ac1c67b..a8ba84036 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -418,6 +418,46 @@ defmodule Pleroma.Web.Router do delete("/chats/:id/messages/:message_id", ChatController, :delete_message) end + # Mastodon AdminAPI: admins and mods (staff) can perform these actions (if privileged by role) + scope "/api/v1/admin", Pleroma.Web.MastodonAPI.Admin do + pipe_through([:require_privileged_role_users_read]) + + get("/accounts", AccountController, :index) + get("/accounts/:id", AccountController, :show) + end + + # Mastodon AdminAPI: admins and mods (staff) can perform these actions (if privileged by role) + scope "/api/v1/admin", Pleroma.Web.MastodonAPI.Admin do + pipe_through(:require_privileged_role_users_delete) + + delete("/accounts/:id", AccountController, :delete) + end + + # Mastodon AdminAPI: admins and mods (staff) can perform these actions (if privileged by role) + scope "/api/v1/admin", Pleroma.Web.MastodonAPI.Admin do + pipe_through([:require_privileged_role_users_manage_activation_state]) + + post("/accounts/:id/action", AccountController, :account_action) + post("/accounts/:id/enable", AccountController, :enable) + end + + # Mastodon AdminAPI: admins and mods (staff) can perform these actions (if privileged by role) + scope "/api/v1/admin", Pleroma.Web.MastodonAPI.Admin do + pipe_through([:require_privileged_role_users_manage_invites]) + post("/accounts/:id/approve", AccountController, :approve) + post("/accounts/:id/reject", AccountController, :reject) + end + + # Mastodon AdminAPI: admins and mods (staff) can perform these actions (if privileged by role) + scope "/api/v1/admin", Pleroma.Web.MastodonAPI.Admin do + pipe_through([:require_privileged_role_reports_manage_reports]) + + get("/reports", ReportController, :index) + get("/reports/:id", ReportController, :show) + post("/reports/:id/resolve", ReportController, :resolve) + post("/reports/:id/reopen", ReportController, :reopen) + end + # AdminAPI: admins and mods (staff) can perform these actions (if privileged by role) scope "/api/v1/pleroma/admin", Pleroma.Web.AdminAPI do pipe_through(:require_privileged_role_emoji_manage_emoji) diff --git a/test/pleroma/web/mastodon_api/admin/controllers/account_controller_test.exs b/test/pleroma/web/mastodon_api/admin/controllers/account_controller_test.exs new file mode 100644 index 000000000..7280691d4 --- /dev/null +++ b/test/pleroma/web/mastodon_api/admin/controllers/account_controller_test.exs @@ -0,0 +1,175 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.MastodonAPI.Admin.AccountControllerTest do + use Pleroma.Web.ConnCase + use Oban.Testing, repo: Pleroma.Repo + + import Pleroma.Factory + + alias Pleroma.Repo + alias Pleroma.User + alias Pleroma.Web.CommonAPI + + setup_all do + Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) + + :ok + end + + setup do + admin = insert(:user, is_admin: true) + token = insert(:oauth_admin_token, user: admin) + + conn = + build_conn() + |> assign(:user, admin) + |> assign(:token, token) + + {:ok, %{admin: admin, token: token, conn: conn}} + end + + describe "GET /api/v1/admin/accounts" do + test "search by display name", %{conn: conn} do + %{id: id} = insert(:user, name: "Display name") + insert(:user, name: "Other name") + + assert [%{"id" => ^id}] = + conn + |> get("/api/v1/admin/accounts?display_name=Display") + |> json_response_and_validate_schema(200) + end + end + + describe "GET /api/v1/admin/accounts/:id" do + test "show admin-level information", %{conn: conn} do + %{id: id} = + insert(:user, + email: "email@example.com", + is_confirmed: false, + is_moderator: true + ) + + assert %{ + "id" => ^id, + "email" => "email@example.com", + "confirmed" => false, + "role" => "moderator" + } = + conn + |> get("/api/v1/admin/accounts/#{id}") + |> json_response_and_validate_schema(200) + end + end + + describe "DELETE /api/v1/admin/accounts/:id" do + test "delete account", %{conn: conn} do + %{id: id} = user = insert(:user) + + conn + |> delete("/api/v1/admin/accounts/#{id}") + |> json_response_and_validate_schema(200) + + user = Repo.reload!(user) + + assert %{is_active: false} = user + end + end + + describe "POST /api/v1/admin/accounts/:id/action" do + test "disable account", %{conn: conn} do + %{id: id} = user = insert(:user) + + conn + |> put_req_header("content-type", "application/json") + |> post("/api/v1/admin/accounts/#{id}/action", %{ + "type" => "disable" + }) + |> json_response_and_validate_schema(204) + + user = Repo.reload!(user) + + assert %{is_active: false} = user + end + + test "perform action with assigned report", %{conn: conn} do + [reporter, target_user] = insert_pair(:user) + + {:ok, %{id: report_id} = report} = + CommonAPI.report(reporter, %{ + account_id: target_user.id + }) + + %{id: id} = insert(:user) + + conn + |> put_req_header("content-type", "application/json") + |> post("/api/v1/admin/accounts/#{id}/action", %{ + "type" => "none", + "report_id" => report_id + }) + |> json_response_and_validate_schema(204) + + report = Repo.reload!(report) + + assert %{data: %{"state" => "resolved"}} = report + end + end + + describe "POST /api/v1/admin/accounts/:id/enable" do + test "enable account", %{conn: conn} do + %{id: id} = user = insert(:user) + User.set_activation(user, false) + + conn + |> post("/api/v1/admin/accounts/#{id}/enable") + |> json_response_and_validate_schema(200) + + user = Repo.reload!(user) + + assert %{is_active: true} = user + end + end + + describe "POST /api/v1/admin/accounts/:id/approve" do + test "approve account", %{conn: conn} do + %{id: id} = user = insert(:user, is_approved: false) + + conn + |> post("/api/v1/admin/accounts/#{id}/approve") + |> json_response_and_validate_schema(200) + + user = Repo.reload!(user) + + assert %{is_approved: true} = user + end + end + + describe "POST /api/v1/admin/accounts/:id/rejct" do + test "reject account", %{conn: conn} do + %{id: id} = user = insert(:user, is_approved: false) + + conn + |> post("/api/v1/admin/accounts/#{id}/reject") + |> json_response_and_validate_schema(200) + + user = Repo.reload!(user) + + assert %{is_active: false} = user + end + + test "do not allow rejecting already accepted accounts", %{conn: conn} do + %{id: id} = user = insert(:user, is_approved: true) + + assert %{"error" => "User is approved"} == + conn + |> post("/api/v1/admin/accounts/#{id}/reject") + |> json_response_and_validate_schema(400) + + user = Repo.reload!(user) + + assert %{is_approved: true} = user + end + end +end diff --git a/test/pleroma/web/mastodon_api/admin/controllers/report_controller_test.exs b/test/pleroma/web/mastodon_api/admin/controllers/report_controller_test.exs new file mode 100644 index 000000000..a0cc5f459 --- /dev/null +++ b/test/pleroma/web/mastodon_api/admin/controllers/report_controller_test.exs @@ -0,0 +1,118 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.MastodonAPI.Admin.ReportControllerTest do + use Pleroma.Web.ConnCase + use Oban.Testing, repo: Pleroma.Repo + + import Pleroma.Factory + + alias Pleroma.Web.CommonAPI + + setup_all do + Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) + + :ok + end + + setup do + admin = insert(:user, is_admin: true) + token = insert(:oauth_admin_token, user: admin) + + conn = + build_conn() + |> assign(:user, admin) + |> assign(:token, token) + + {:ok, %{admin: admin, token: token, conn: conn}} + end + + describe "GET /api/v1/admin/reports" do + test "get reports by state", %{conn: conn} do + [reporter, target_user] = insert_pair(:user) + activity = insert(:note_activity, user: target_user) + + {:ok, %{id: report_id1}} = + CommonAPI.report(reporter, %{ + account_id: target_user.id, + comment: "this user uses an app called Hitler Tusky", + status_ids: [activity.id] + }) + + {:ok, %{id: report_id2}} = + CommonAPI.report(reporter, %{ + account_id: target_user.id, + status_ids: [activity.id] + }) + + CommonAPI.update_report_state(report_id2, "resolved") + + assert [%{"id" => ^report_id1}] = + conn + |> get("/api/v1/admin/reports?resolved=false") + |> json_response_and_validate_schema(200) + + assert [%{"id" => ^report_id2}] = + conn + |> get("/api/v1/admin/reports?resolved=true") + |> json_response_and_validate_schema(200) + end + end + + describe "GET /api/v1/admin/reports/:id" do + test "get report by id", %{conn: conn} do + [reporter, target_user] = insert_pair(:user) + activity = insert(:note_activity, user: target_user) + + {:ok, %{id: report_id}} = + CommonAPI.report(reporter, %{ + account_id: target_user.id, + status_ids: [activity.id] + }) + + assert %{"id" => ^report_id} = + conn + |> get("/api/v1/admin/reports/#{report_id}") + |> json_response_and_validate_schema(200) + end + end + + describe "POST /api/v1/admin/reports/:id/resolve" do + test "resolve a report", %{conn: conn} do + [reporter, target_user] = insert_pair(:user) + activity = insert(:note_activity, user: target_user) + + {:ok, %{id: report_id}} = + CommonAPI.report(reporter, %{ + account_id: target_user.id, + status_ids: [activity.id] + }) + + assert %{"id" => ^report_id, "action_taken" => true} = + conn + |> post("/api/v1/admin/reports/#{report_id}/resolve") + |> json_response_and_validate_schema(200) + end + end + + describe "POST /api/v1/admin/reports/:id/reopen" do + test "reopen a report", %{conn: conn} do + [reporter, target_user] = insert_pair(:user) + activity = insert(:note_activity, user: target_user) + + {:ok, %{id: report_id}} = + CommonAPI.report(reporter, %{ + account_id: target_user.id, + status_ids: [activity.id] + }) + + CommonAPI.update_report_state(report_id, "resolved") + + assert %{"id" => ^report_id, "action_taken" => false} = + conn + |> post("/api/v1/admin/reports/#{report_id}/reopen") + |> json_response_and_validate_schema(200) + end + end +end From 2c14121389d0dadd0609695b941881eb5d93cbae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?nicole=20miko=C5=82ajczyk?= Date: Fri, 19 Sep 2025 16:59:36 +0200 Subject: [PATCH 022/183] MastoAPI: /api/v2/admin/accounts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: nicole mikołajczyk --- lib/pleroma/user/query.ex | 5 + .../mastodon_admin/account_operation.ex | 118 +++++++++++++++--- .../mastodon_admin/report_operation.ex | 12 +- .../admin/controllers/account_controller.ex | 48 ++++++- lib/pleroma/web/router.ex | 7 ++ 5 files changed, 164 insertions(+), 26 deletions(-) diff --git a/lib/pleroma/user/query.ex b/lib/pleroma/user/query.ex index b877b52b2..33f19c454 100644 --- a/lib/pleroma/user/query.ex +++ b/lib/pleroma/user/query.ex @@ -302,6 +302,11 @@ defmodule Pleroma.User.Query do where(query, [u], u.is_admin == true or u.is_moderator == true) end + defp compose_query({:domain, domain}, query) do + query + |> where([u], like(u.nickname, ^"%@#{domain}")) + end + defp compose_query(_unsupported_param, query), do: query defp location_query(query, local) do diff --git a/lib/pleroma/web/api_spec/operations/mastodon_admin/account_operation.ex b/lib/pleroma/web/api_spec/operations/mastodon_admin/account_operation.ex index 23dae0306..19ccf97fb 100644 --- a/lib/pleroma/web/api_spec/operations/mastodon_admin/account_operation.ex +++ b/lib/pleroma/web/api_spec/operations/mastodon_admin/account_operation.ex @@ -19,7 +19,7 @@ defmodule Pleroma.Web.ApiSpec.MastodonAdmin.AccountOperation do def index_operation do %Operation{ tags: ["User administration (Mastodon API)"], - summary: "View accounts by criteria", + summary: "View accounts by criteria (v1)", operationId: "MastodonAdmin.AccountController.index", description: "View accounts matching certain criteria for filtering, up to 40 at a time.", security: [%{"oAuth" => ["admin:read:accounts"]}], @@ -27,12 +27,6 @@ defmodule Pleroma.Web.ApiSpec.MastodonAdmin.AccountOperation do [ Operation.parameter(:local, :query, :boolean, "Filter for local accounts?"), Operation.parameter(:remote, :query, :boolean, "Filter for remote accounts?"), - Operation.parameter( - :by_domain, - :query, - :string, - "Filter by the given domain (not implemented yet)" - ), Operation.parameter( :active, :query, @@ -51,12 +45,6 @@ defmodule Pleroma.Web.ApiSpec.MastodonAdmin.AccountOperation do :boolean, "Filter for currently disabled accounts?" ), - Operation.parameter( - :sensitized, - :query, - :boolean, - "Filter for currently sensitized accounts? (not implemented yet)" - ), Operation.parameter( :silenced, :query, @@ -69,14 +57,31 @@ defmodule Pleroma.Web.ApiSpec.MastodonAdmin.AccountOperation do :boolean, "Filter for currently suspended accounts? (not implemented yet)" ), - Operation.parameter(:username, :query, :string, "Username to search for"), - Operation.parameter(:display_name, :query, :string, "Display name to search for"), + Operation.parameter( + :sensitized, + :query, + :boolean, + "Filter for accounts force-marked as sensitive? (not implemented yet)" + ), + Operation.parameter(:username, :query, :string, "Search for the given username"), + Operation.parameter( + :display_name, + :query, + :string, + "Search for the given display name" + ), + Operation.parameter( + :by_domain, + :query, + :string, + "Filter by the given domain" + ), Operation.parameter(:email, :query, :string, "Lookup a user with this email"), Operation.parameter( :ip, :query, :string, - "Lookup users by this IP address (not implemented yet)" + "Lookup users with this IP address (not implemented yet)" ), Operation.parameter(:staff, :query, :boolean, "Filter for staff accounts?") ] ++ @@ -93,6 +98,87 @@ defmodule Pleroma.Web.ApiSpec.MastodonAdmin.AccountOperation do } end + def index2_operation do + %Operation{ + tags: ["User administration (Mastodon API)"], + summary: "View accounts by criteria (v2)", + operationId: "MastodonAdmin.AccountController.index2", + description: "View accounts matching certain criteria for filtering, up to 40 at a time.", + security: [%{"oAuth" => ["admin:read:accounts"]}], + parameters: + [ + Operation.parameter( + :origin, + :query, + %Schema{type: :string, enum: ["local", "remote"]}, + "Filter for local or remote accounts" + ), + Operation.parameter( + :status, + :query, + %Schema{ + type: :string, + enum: ["active", "inactive", "pending", "disabled", "silenced", "suspended"] + }, + "Filter for active, pending, disabled, silenced or suspended accounts" + ), + Operation.parameter( + :permissions, + :query, + :string, + "Filter for accounts with staff permissions (users that can manage reports). (not implemented yet)" + ), + Operation.parameter( + :role_ids, + :query, + %Schema{ + oneOf: [ + %Schema{type: :array, items: %Schema{type: :string}}, + %Schema{type: :string} + ] + }, + "Filter for users with these roles. (not implemented yet)" + ), + Operation.parameter( + :invited_by, + :query, + :string, + "Lookup users invited by the account with this ID. (not implemented yet)" + ), + Operation.parameter(:username, :query, :string, "Search for the given username"), + Operation.parameter( + :display_name, + :query, + :string, + "Search for the given display name" + ), + Operation.parameter( + :by_domain, + :query, + :string, + "Filter by the given domain" + ), + Operation.parameter(:email, :query, :string, "Lookup a user with this email"), + Operation.parameter( + :ip, + :query, + :string, + "Lookup users with this IP address (not implemented yet)" + ) + ] ++ + pagination_params(), + responses: %{ + 200 => + Operation.response("Account", "application/json", %Schema{ + title: "ArrayOfAccounts", + type: :array, + items: account() + }), + 401 => Operation.response("Error", "application/json", ApiError) + } + } + end + def show_operation do %Operation{ tags: ["User administration (Mastodon API)"], diff --git a/lib/pleroma/web/api_spec/operations/mastodon_admin/report_operation.ex b/lib/pleroma/web/api_spec/operations/mastodon_admin/report_operation.ex index 274ce3bd9..487e7853e 100644 --- a/lib/pleroma/web/api_spec/operations/mastodon_admin/report_operation.ex +++ b/lib/pleroma/web/api_spec/operations/mastodon_admin/report_operation.ex @@ -119,16 +119,12 @@ defmodule Pleroma.Web.ApiSpec.MastodonAdmin.ReportOperation do account: AccountOperation.account(), target_account: AccountOperation.account(), assigned_account: %Schema{ - oneOf: [ - AccountOperation.account(), - %Schema{type: :null} - ] + type: AccountOperation.account(), + nullable: true }, action_taken_by_account: %Schema{ - oneOf: [ - AccountOperation.account(), - %Schema{type: :null} - ] + type: AccountOperation.account(), + nullable: true }, statuses: %Schema{ type: :array, diff --git a/lib/pleroma/web/mastodon_api/admin/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/admin/controllers/account_controller.ex index a1eac7fe3..833bf1db5 100644 --- a/lib/pleroma/web/mastodon_api/admin/controllers/account_controller.ex +++ b/lib/pleroma/web/mastodon_api/admin/controllers/account_controller.ex @@ -22,7 +22,7 @@ defmodule Pleroma.Web.MastodonAPI.Admin.AccountController do alias Pleroma.Web.Plugs.OAuthScopesPlug @filter_params ~W( - local external active needing_approval deactivated nickname name email staff + local external active needing_approval deactivated nickname name domain email staff origin status ) plug(Pleroma.Web.ApiSpec.CastAndValidate) @@ -30,7 +30,7 @@ defmodule Pleroma.Web.MastodonAPI.Admin.AccountController do plug( OAuthScopesPlug, %{scopes: ["admin:read:accounts"]} - when action in [:index, :show] + when action in [:index, :index2, :show] ) plug( @@ -71,6 +71,18 @@ defmodule Pleroma.Web.MastodonAPI.Admin.AccountController do |> render("index.json", users: users) end + def index2(conn, params) do + users = + params + |> build_criteria_v2() + |> User.Query.build() + |> Pagination.fetch_paginated(params) + + conn + |> add_link_headers(users) + |> render("index.json", users: users) + end + def show(%{assigns: %{user: _admin, account: user}} = conn, _params) do render(conn, "show.json", user: user) end @@ -162,10 +174,21 @@ defmodule Pleroma.Web.MastodonAPI.Admin.AccountController do |> maybe_filter_deactivated(params) |> maybe_filter_nickname(params) |> maybe_filter_name(params) + |> maybe_filter_domain(params) |> maybe_filter_email(params) |> maybe_filter_staff(params) end + defp build_criteria_v2(params) do + %{} + |> maybe_filter_origin(params) + |> maybe_filter_status(params) + |> maybe_filter_nickname(params) + |> maybe_filter_name(params) + |> maybe_filter_domain(params) + |> maybe_filter_email(params) + end + defp maybe_filter_local(criteria, %{local: true} = _params), do: Map.put(criteria, :local, true) @@ -178,6 +201,12 @@ defmodule Pleroma.Web.MastodonAPI.Admin.AccountController do defp maybe_filter_external(criteria, %{remote: false} = _params), do: Map.put(criteria, :local, true) + defp maybe_filter_origin(criteria, %{origin: "local"} = _params), + do: Map.put(criteria, :local, true) + + defp maybe_filter_origin(criteria, %{origin: "remote"} = _params), + do: Map.put(criteria, :external, true) + defp maybe_filter_active(criteria, %{active: active} = _params), do: Map.put(criteria, :active, active) @@ -187,12 +216,27 @@ defmodule Pleroma.Web.MastodonAPI.Admin.AccountController do defp maybe_filter_deactivated(criteria, %{disabled: deactivated} = _params), do: Map.put(criteria, :deactivated, deactivated) + defp maybe_filter_status(criteria, %{status: "active"} = _params), + do: Map.put(criteria, :active, true) + + defp maybe_filter_status(criteria, %{status: "inactive"} = _params), + do: Map.put(criteria, :active, false) + + defp maybe_filter_status(criteria, %{status: "pending"} = _params), + do: Map.put(criteria, :need_approval, true) + + defp maybe_filter_status(criteria, %{status: "disabled"} = _params), + do: Map.put(criteria, :deactivated, true) + defp maybe_filter_nickname(criteria, %{username: nickname} = _params), do: Map.put(criteria, :nickname, nickname) defp maybe_filter_name(criteria, %{display_name: name} = _params), do: Map.put(criteria, :name, name) + defp maybe_filter_domain(criteria, %{by_domain: domain} = _params), + do: Map.put(criteria, :domain, domain) + defp maybe_filter_email(criteria, %{email: email} = _params), do: Map.put(criteria, :email, email) diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index a8ba84036..87568b481 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -458,6 +458,13 @@ defmodule Pleroma.Web.Router do post("/reports/:id/reopen", ReportController, :reopen) end + # Mastodon AdminAPI: admins and mods (staff) can perform these actions (if privileged by role) + scope "/api/v2/admin", Pleroma.Web.MastodonAPI.Admin do + pipe_through([:require_privileged_role_users_read]) + + get("/accounts", AccountController, :index2) + end + # AdminAPI: admins and mods (staff) can perform these actions (if privileged by role) scope "/api/v1/pleroma/admin", Pleroma.Web.AdminAPI do pipe_through(:require_privileged_role_emoji_manage_emoji) From 87e6cd3dc009f977fec7fddb664c73cc525d9fe1 Mon Sep 17 00:00:00 2001 From: mkljczk Date: Tue, 18 Mar 2025 00:34:37 +0100 Subject: [PATCH 023/183] Adapt tests for v2/admin/accounts from Mastodon MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: mkljczk Signed-off-by: nicole mikołajczyk --- .../controllers/account_controller_test.exs | 90 +++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/test/pleroma/web/mastodon_api/admin/controllers/account_controller_test.exs b/test/pleroma/web/mastodon_api/admin/controllers/account_controller_test.exs index 7280691d4..e06b3a70a 100644 --- a/test/pleroma/web/mastodon_api/admin/controllers/account_controller_test.exs +++ b/test/pleroma/web/mastodon_api/admin/controllers/account_controller_test.exs @@ -42,6 +42,96 @@ defmodule Pleroma.Web.MastodonAPI.Admin.AccountControllerTest do end end + # Adapted from + # https://github.com/mastodon/mastodon/blob/main/spec/requests/api/v2/admin/accounts_spec.rb + describe "GET /api/v2/admin/accounts" do + setup do + remote_account = insert(:user, nickname: "remote@example.org", local: false) + other_remote_account = insert(:user, nickname: "other@foo.bar", local: false) + # suspended_account = insert(:user) + # suspended_remote = insert(:user) + disabled_account = insert(:user, is_active: false) + pending_account = insert(:user, is_approved: false) + admin_account = insert(:user, is_admin: true) + + {:ok, + %{ + remote_account: remote_account, + other_remote_account: other_remote_account, + disabled_account: disabled_account, + pending_account: pending_account, + admin_account: admin_account + }} + end + + # test "returns the correct accounts when called with status active + # and origin local and permissions staff", %{ + # conn: conn, + # admin_account: %{id: admin_account_id} + # } do + # assert [%{"id" => ^admin_account_id}] = + # conn + # |> get("/api/v2/admin/accounts?status=active&origin=local&permissions=staff") + # |> json_response_and_validate_schema(200) + # end + + test "returns the correct accounts when called with by_domain value and origin remote", %{ + conn: conn, + remote_account: %{id: remote_account_id} + } do + assert [%{"id" => ^remote_account_id}] = + conn + |> get("/api/v2/admin/accounts?by_domain=example.org&origin=remote") + |> json_response_and_validate_schema(200) + end + + # test "returns the correct accounts when called with status suspended", %{ + # conn: conn, + # suspended_account: %{id: suspended_account_id} + # } do + # assert [%{"id" => ^suspended_account_id}] = + # conn + # |> get("/api/v2/admin/accounts?status=suspended") + # |> json_response_and_validate_schema(200) + # end + + test "returns the correct accounts when called with status disabled", %{ + conn: conn, + disabled_account: %{id: disabled_account_id} + } do + assert [%{"id" => ^disabled_account_id}] = + conn + |> get("/api/v2/admin/accounts?status=disabled") + |> json_response_and_validate_schema(200) + end + + test "returns the correct accounts when called with status pending", %{ + conn: conn, + pending_account: %{id: pending_account_id} + } do + assert [%{"id" => ^pending_account_id}] = + conn + |> get("/api/v2/admin/accounts?status=pending") + |> json_response_and_validate_schema(200) + end + + test "sets the correct pagination headers with limit param", %{ + conn: conn, + admin_account: %{id: admin_account_id} + } do + response = + conn + |> get("/api/v2/admin/accounts?limit=1") + + next_url = + ~r{<.+?(?/api[^>]+)>; rel=\"next\"} + |> Regex.named_captures(get_resp_header(response, "link") |> Enum.at(0)) + |> Map.get("link") + + next_url =~ "&limit=1&max_id=#{admin_account_id}" + end + end + describe "GET /api/v1/admin/accounts/:id" do test "show admin-level information", %{conn: conn} do %{id: id} = From 8626a337f807d6ab97840419148f187d081d18bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?nicole=20miko=C5=82ajczyk?= Date: Fri, 27 Feb 2026 22:34:18 +0100 Subject: [PATCH 024/183] Mastodon admin API v2 improvements MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: nicole mikołajczyk --- .../api_spec/operations/mastodon_admin/account_operation.ex | 6 +++--- .../api_spec/operations/mastodon_admin/report_operation.ex | 4 ++-- .../web/mastodon_api/admin/controllers/report_controller.ex | 2 +- .../admin/controllers/account_controller_test.exs | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/pleroma/web/api_spec/operations/mastodon_admin/account_operation.ex b/lib/pleroma/web/api_spec/operations/mastodon_admin/account_operation.ex index 19ccf97fb..e9143364c 100644 --- a/lib/pleroma/web/api_spec/operations/mastodon_admin/account_operation.ex +++ b/lib/pleroma/web/api_spec/operations/mastodon_admin/account_operation.ex @@ -31,7 +31,7 @@ defmodule Pleroma.Web.ApiSpec.MastodonAdmin.AccountOperation do :active, :query, :boolean, - "Filter for currently active accounts??" + "Filter for currently active accounts?" ), Operation.parameter( :pending, @@ -327,8 +327,8 @@ defmodule Pleroma.Web.ApiSpec.MastodonAdmin.AccountOperation do } } }, - locale: %Schema{type: :string, format: "date-time", nullable: true}, - invite_request: %Schema{type: :string, format: "date-time", nullable: true}, + locale: %Schema{type: :string, nullable: true}, + invite_request: %Schema{type: :string, nullable: true}, role: %Schema{type: :string, nullable: true}, confirmed: %Schema{type: :boolean}, approved: %Schema{type: :boolean}, diff --git a/lib/pleroma/web/api_spec/operations/mastodon_admin/report_operation.ex b/lib/pleroma/web/api_spec/operations/mastodon_admin/report_operation.ex index 487e7853e..3360cb843 100644 --- a/lib/pleroma/web/api_spec/operations/mastodon_admin/report_operation.ex +++ b/lib/pleroma/web/api_spec/operations/mastodon_admin/report_operation.ex @@ -119,11 +119,11 @@ defmodule Pleroma.Web.ApiSpec.MastodonAdmin.ReportOperation do account: AccountOperation.account(), target_account: AccountOperation.account(), assigned_account: %Schema{ - type: AccountOperation.account(), + oneOf: [AccountOperation.account()], nullable: true }, action_taken_by_account: %Schema{ - type: AccountOperation.account(), + oneOf: [AccountOperation.account()], nullable: true }, statuses: %Schema{ diff --git a/lib/pleroma/web/mastodon_api/admin/controllers/report_controller.ex b/lib/pleroma/web/mastodon_api/admin/controllers/report_controller.ex index 217a97cc9..2d89d13bc 100644 --- a/lib/pleroma/web/mastodon_api/admin/controllers/report_controller.ex +++ b/lib/pleroma/web/mastodon_api/admin/controllers/report_controller.ex @@ -65,7 +65,7 @@ defmodule Pleroma.Web.MastodonAPI.Admin.ReportController do subject_actor: report.user_actor }) - render(conn, "show.json", Report.extract_report_info(activity)) + render(conn, "show.json", Report.extract_report_info(report)) else {:error, error} -> json_response(conn, :bad_request, %{error: error}) diff --git a/test/pleroma/web/mastodon_api/admin/controllers/account_controller_test.exs b/test/pleroma/web/mastodon_api/admin/controllers/account_controller_test.exs index e06b3a70a..113b242dd 100644 --- a/test/pleroma/web/mastodon_api/admin/controllers/account_controller_test.exs +++ b/test/pleroma/web/mastodon_api/admin/controllers/account_controller_test.exs @@ -236,7 +236,7 @@ defmodule Pleroma.Web.MastodonAPI.Admin.AccountControllerTest do end end - describe "POST /api/v1/admin/accounts/:id/rejct" do + describe "POST /api/v1/admin/accounts/:id/reject" do test "reject account", %{conn: conn} do %{id: id} = user = insert(:user, is_approved: false) From a72f9cff232c6afd7aff8ab41b21d82779d8a086 Mon Sep 17 00:00:00 2001 From: Phantasm Date: Thu, 16 Oct 2025 15:03:54 +0200 Subject: [PATCH 025/183] Elixir 1.19: Fix typing violation on struct updates in Pleroma.Marker MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit warning: a struct for Pleroma.Marker is expected on struct update: %Pleroma.Marker{marker | user: user} but got type: dynamic() where "marker" was given the type: # type: dynamic() # from: lib/pleroma/marker.ex {:ok, marker} when defining the variable "marker", you must also pattern match on "%Pleroma.Marker{}". hint: given pattern matching is enough to catch typing errors, you may optionally convert the struct update into a map update. For example, instead of: user = some_function() %User{user | name: "John Doe"} it is enough to write: %User{} = user = some_function() %{user | name: "John Doe"} typing violation found at: │ 81 │ {:ok, marker} -> %__MODULE__{marker | user: user} │ ~ │ └─ lib/pleroma/marker.ex:81:24: Pleroma.Marker.get_marker/2 --- lib/pleroma/marker.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/marker.ex b/lib/pleroma/marker.ex index 68b054e4d..4f645240e 100644 --- a/lib/pleroma/marker.ex +++ b/lib/pleroma/marker.ex @@ -78,7 +78,7 @@ defmodule Pleroma.Marker do defp get_marker(user, timeline) do case Repo.find_resource(get_query(user, timeline)) do - {:ok, marker} -> %__MODULE__{marker | user: user} + {:ok, %__MODULE__{} = marker} -> %__MODULE__{marker | user: user} _ -> %__MODULE__{timeline: timeline, user_id: user.id} end end From 0fc046a75e0499a068cbdee82f9c559407a7521b Mon Sep 17 00:00:00 2001 From: Phantasm Date: Thu, 16 Oct 2025 15:06:02 +0200 Subject: [PATCH 026/183] Elixir 1.19: Fix typing violation on struct updates in MFA.Changeset MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit warning: a struct for Pleroma.MFA.Settings is expected on struct update: %Pleroma.MFA.Settings{settings | enabled: false} but got type: dynamic() where "settings" was given the type: # type: dynamic() # from: lib/pleroma/mfa/changeset.ex:11:14 settings = Pleroma.MFA.fetch_settings(Ecto.Changeset.apply_changes(changeset)) when defining the variable "settings", you must also pattern match on "%Pleroma.MFA.Settings{}". hint: given pattern matching is enough to catch typing errors, you may optionally convert the struct update into a map update. For example, instead of: user = some_function() %User{user | name: "John Doe"} it is enough to write: %User{} = user = some_function() %{user | name: "John Doe"} typing violation found at: │ 17 │ put_change(changeset, %Settings{settings | enabled: false}) │ ~ │ └─ lib/pleroma/mfa/changeset.ex:17:29: Pleroma.MFA.Changeset.disable/2 --- warning: a struct for Pleroma.MFA.Settings is expected on struct update: %Pleroma.MFA.Settings{ settings | totp: %Pleroma.MFA.Settings.TOTP{confirmed: false, delivery_type: "app", secret: nil} } but got type: dynamic() where "settings" was given the type: # type: dynamic() # from: lib/pleroma/mfa/changeset.ex:23:74 %Pleroma.User{multi_factor_authentication_settings: settings} = user when defining the variable "settings", you must also pattern match on "%Pleroma.MFA.Settings{}". hint: given pattern matching is enough to catch typing errors, you may optionally convert the struct update into a map update. For example, instead of: user = some_function() %User{user | name: "John Doe"} it is enough to write: %User{} = user = some_function() %{user | name: "John Doe"} typing violation found at: │ 25 │ |> put_change(%Settings{settings | totp: %Settings.TOTP{}}) │ ~ │ └─ lib/pleroma/mfa/changeset.ex:25:19: Pleroma.MFA.Changeset.disable_totp/1 --- lib/pleroma/mfa/changeset.ex | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/mfa/changeset.ex b/lib/pleroma/mfa/changeset.ex index 3ec3cfe91..890cb2193 100644 --- a/lib/pleroma/mfa/changeset.ex +++ b/lib/pleroma/mfa/changeset.ex @@ -8,7 +8,7 @@ defmodule Pleroma.MFA.Changeset do alias Pleroma.User def disable(%Ecto.Changeset{} = changeset, force \\ false) do - settings = + %Settings{} = settings = changeset |> Ecto.Changeset.apply_changes() |> MFA.fetch_settings() @@ -20,20 +20,20 @@ defmodule Pleroma.MFA.Changeset do end end - def disable_totp(%User{multi_factor_authentication_settings: settings} = user) do + def disable_totp(%User{multi_factor_authentication_settings: %Settings{} = settings} = user) do user |> put_change(%Settings{settings | totp: %Settings.TOTP{}}) end - def confirm_totp(%User{multi_factor_authentication_settings: settings} = user) do - totp_settings = %Settings.TOTP{settings.totp | confirmed: true} + def confirm_totp(%User{multi_factor_authentication_settings: %Settings{} = settings} = user) do + totp_settings = %Settings.TOTP{%Settings.TOTP{} = settings.totp | confirmed: true} user |> put_change(%Settings{settings | totp: totp_settings, enabled: true}) end def setup_totp(%User{} = user, attrs) do - mfa_settings = MFA.fetch_settings(user) + %Settings{} = mfa_settings = MFA.fetch_settings(user) totp_settings = %Settings.TOTP{} @@ -46,7 +46,7 @@ defmodule Pleroma.MFA.Changeset do def cast_backup_codes(%User{} = user, codes) do user |> put_change(%Settings{ - user.multi_factor_authentication_settings + %Settings{} = user.multi_factor_authentication_settings | backup_codes: codes }) end From 0d3085937ca728c572f19798d21dce12dcf7b89e Mon Sep 17 00:00:00 2001 From: Phantasm Date: Thu, 16 Oct 2025 15:09:03 +0200 Subject: [PATCH 027/183] Elixir 1.19: Fix typing violation on struct updates in Pleroma.Upload MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit warning: a struct for Pleroma.Upload is expected on struct update: %Pleroma.Upload{ upload | path: case upload.path do x when x === false or x === nil -> <> x -> x end } but got type: dynamic() where "upload" was given the type: # type: dynamic() # from: lib/pleroma/upload.ex:95:24 {:ok, upload} <- prepare_upload(upload, opts) when defining the variable "upload", you must also pattern match on "%Pleroma.Upload{}". hint: given pattern matching is enough to catch typing errors, you may optionally convert the struct update into a map update. For example, instead of: user = some_function() %User{user | name: "John Doe"} it is enough to write: %User{} = user = some_function() %{user | name: "John Doe"} typing violation found at: │ 96 │ upload = %__MODULE__{upload | path: upload.path || "#{upload.id}/#{upload.name}"}, │ ~ │ └─ lib/pleroma/upload.ex:96:19: Pleroma.Upload.store/2 --- lib/pleroma/upload.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/upload.ex b/lib/pleroma/upload.ex index 06d8005bc..350ff6cb3 100644 --- a/lib/pleroma/upload.ex +++ b/lib/pleroma/upload.ex @@ -93,7 +93,7 @@ defmodule Pleroma.Upload do def store(upload, opts \\ []) do opts = get_opts(opts) - with {:ok, upload} <- prepare_upload(upload, opts), + with {:ok, %__MODULE__{} = upload} <- prepare_upload(upload, opts), upload = %__MODULE__{upload | path: upload.path || "#{upload.id}/#{upload.name}"}, {:ok, upload} <- Pleroma.Upload.Filter.filter(opts.filters, upload), description = get_description(upload), From 6f62cb0ed38c5d2bbdb6e304a287553bad8584f1 Mon Sep 17 00:00:00 2001 From: Phantasm Date: Thu, 16 Oct 2025 15:09:51 +0200 Subject: [PATCH 028/183] Elixir 1.19: Fix typing violation on struct updates in Web.ApiSpec.Cast* MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit warning: a struct for Plug.Conn is expected on struct update: %Plug.Conn{conn | query_params: query_params} but got type: dynamic() where "conn" was given the type: # type: dynamic() # from: lib/pleroma/web/api_spec/cast_and_validate.ex:109:43 conn when defining the variable "conn", you must also pattern match on "%Plug.Conn{}". hint: given pattern matching is enough to catch typing errors, you may optionally convert the struct update into a map update. For example, instead of: user = some_function() %User{user | name: "John Doe"} it is enough to write: %User{} = user = some_function() %{user | name: "John Doe"} typing violation found at: │ 133 │ conn = %Conn{conn | query_params: query_params} │ ~ │ └─ lib/pleroma/web/api_spec/cast_and_validate.ex:133:16: Pleroma.Web.ApiSpec.CastAndValidate.cast_and_validate/6 --- lib/pleroma/web/api_spec/cast_and_validate.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/api_spec/cast_and_validate.ex b/lib/pleroma/web/api_spec/cast_and_validate.ex index 672d1c4a1..57ea8e9b3 100644 --- a/lib/pleroma/web/api_spec/cast_and_validate.ex +++ b/lib/pleroma/web/api_spec/cast_and_validate.ex @@ -106,7 +106,7 @@ defmodule Pleroma.Web.ApiSpec.CastAndValidate do OpenApiSpex.cast_and_validate(spec, operation, conn, content_type, cast_opts) end - defp cast_and_validate(spec, operation, conn, content_type, false = _strict, cast_opts) do + defp cast_and_validate(spec, operation, %Conn{} = conn, content_type, false = _strict, cast_opts) do case OpenApiSpex.cast_and_validate(spec, operation, conn, content_type) do {:ok, conn} -> {:ok, conn} From 7c79e4416f78b317d8a3bda4cacd23f3d955df46 Mon Sep 17 00:00:00 2001 From: Phantasm Date: Thu, 16 Oct 2025 15:11:33 +0200 Subject: [PATCH 029/183] Elixir 1.19: Fix typing violation on struct updates in Web.ApiSpec.Rend* MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit warning: a struct for OpenApiSpex.Cast.Error is expected on struct update: %OpenApiSpex.Cast.Error{err | name: err.value} but got type: dynamic(%{..., name: nil, reason: :invalid_enum}) where "err" was given the type: # type: dynamic(%{..., name: nil, reason: :invalid_enum}) # from: lib/pleroma/web/api_spec/render_error.ex:20:45 %{name: nil, reason: :invalid_enum} = err when defining the variable "err", you must also pattern match on "%OpenApiSpex.Cast.Error{}". hint: given pattern matching is enough to catch typing errors, you may optionally convert the struct update into a map update. For example, instead of: user = some_function() %User{user | name: "John Doe"} it is enough to write: %User{} = user = some_function() %{user | name: "John Doe"} typing violation found at: │ 21 │ %OpenApiSpex.Cast.Error{err | name: err.value} │ ~ │ └─ lib/pleroma/web/api_spec/render_error.ex:21:11: Pleroma.Web.ApiSpec.RenderError.call/2 --- lib/pleroma/web/api_spec/render_error.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/api_spec/render_error.ex b/lib/pleroma/web/api_spec/render_error.ex index 3539af6e4..acf510774 100644 --- a/lib/pleroma/web/api_spec/render_error.ex +++ b/lib/pleroma/web/api_spec/render_error.ex @@ -17,10 +17,10 @@ defmodule Pleroma.Web.ApiSpec.RenderError do def call(conn, errors) do errors = Enum.map(errors, fn - %{name: nil, reason: :invalid_enum} = err -> + %OpenApiSpex.Cast.Error{name: nil, reason: :invalid_enum} = err -> %OpenApiSpex.Cast.Error{err | name: err.value} - %{name: nil} = err -> + %OpenApiSpex.Cast.Error{name: nil} = err -> %OpenApiSpex.Cast.Error{err | name: List.last(err.path)} err -> From 6a926bf605b6e33c0fe9242fc0b2c1b8f68b655d Mon Sep 17 00:00:00 2001 From: Phantasm Date: Thu, 16 Oct 2025 15:12:41 +0200 Subject: [PATCH 030/183] Elixir 1.19: Fix typing violation on struct updates in CommonAPI.Activity* MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit warning: a struct for Pleroma.Web.CommonAPI.ActivityDraft is expected on struct update: %Pleroma.Web.CommonAPI.ActivityDraft{draft | object: object} but got type: dynamic() where "draft" was given the type: # type: dynamic() # from: lib/pleroma/web/common_api/activity_draft.ex:91:22 draft when defining the variable "draft", you must also pattern match on "%Pleroma.Web.CommonAPI.ActivityDraft{}". hint: given pattern matching is enough to catch typing errors, you may optionally convert the struct update into a map update. For example, instead of: user = some_function() %User{user | name: "John Doe"} it is enough to write: %User{} = user = some_function() %{user | name: "John Doe"} typing violation found at: │ 102 │ %__MODULE__{draft | object: object} │ ~ │ └─ lib/pleroma/web/common_api/activity_draft.ex:102:5: Pleroma.Web.CommonAPI.ActivityDraft.listen_object/1 --- lib/pleroma/web/common_api/activity_draft.ex | 46 ++++++++++---------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/lib/pleroma/web/common_api/activity_draft.ex b/lib/pleroma/web/common_api/activity_draft.ex index c0b98508c..c5959276a 100644 --- a/lib/pleroma/web/common_api/activity_draft.ex +++ b/lib/pleroma/web/common_api/activity_draft.ex @@ -88,7 +88,7 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do |> validate() end - defp listen_object(draft) do + defp listen_object(%__MODULE__{} = draft) do object = draft.params |> Map.take([:album, :artist, :title, :length]) @@ -102,20 +102,20 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do %__MODULE__{draft | object: object} end - defp put_params(draft, params) do + defp put_params(%__MODULE__{} = draft, params) do params = Map.put_new(params, :in_reply_to_status_id, params[:in_reply_to_id]) %__MODULE__{draft | params: params} end - defp status(%{params: %{status: status}} = draft) do + defp status(%__MODULE__{params: %{status: status}} = draft) do %__MODULE__{draft | status: String.trim(status)} end - defp summary(%{params: params} = draft) do + defp summary(%__MODULE__{params: params} = draft) do %__MODULE__{draft | summary: Map.get(params, :spoiler_text, "")} end - defp full_payload(%{status: status, summary: summary} = draft) do + defp full_payload(%__MODULE__{status: status, summary: summary} = draft) do full_payload = String.trim(status <> summary) case Utils.validate_character_limit(full_payload, draft.attachments) do @@ -124,7 +124,7 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do end end - defp attachments(%{params: params} = draft) do + defp attachments(%__MODULE__{params: params} = draft) do attachments = Utils.attachments_from_ids(params, draft.user) draft = %__MODULE__{draft | attachments: attachments} @@ -134,9 +134,9 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do end end - defp in_reply_to(%{params: %{in_reply_to_status_id: ""}} = draft), do: draft + defp in_reply_to(%__MODULE__{params: %{in_reply_to_status_id: ""}} = draft), do: draft - defp in_reply_to(%{params: %{in_reply_to_status_id: id}} = draft) when is_binary(id) do + defp in_reply_to(%__MODULE__{params: %{in_reply_to_status_id: id}} = draft) when is_binary(id) do # If a post was deleted all its activities (except the newly added Delete) are purged too, # thus lookup by Create db ID will yield nil just as if it never existed in the first place. # @@ -166,13 +166,13 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do end end - defp in_reply_to(%{params: %{in_reply_to_status_id: %Activity{} = in_reply_to}} = draft) do + defp in_reply_to(%__MODULE__{params: %{in_reply_to_status_id: %Activity{} = in_reply_to}} = draft) do %__MODULE__{draft | in_reply_to: in_reply_to} end defp in_reply_to(draft), do: draft - defp quote_post(%{params: %{quoted_status_id: id}} = draft) when not_empty_string(id) do + defp quote_post(%__MODULE__{params: %{quoted_status_id: id}} = draft) when not_empty_string(id) do case Activity.get_by_id_with_object(id) do %Activity{} = activity -> %__MODULE__{draft | quote_post: activity} @@ -188,12 +188,12 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do defp quote_post(draft), do: draft - defp in_reply_to_conversation(draft) do + defp in_reply_to_conversation(%__MODULE__{} = draft) do in_reply_to_conversation = Participation.get(draft.params[:in_reply_to_conversation_id]) %__MODULE__{draft | in_reply_to_conversation: in_reply_to_conversation} end - defp visibility(%{params: params} = draft) do + defp visibility(%__MODULE__{params: params} = draft) do case CommonAPI.get_visibility(params, draft.in_reply_to, draft.in_reply_to_conversation) do {visibility, "direct"} when visibility != "direct" -> add_error(draft, dgettext("errors", "The message visibility must be direct")) @@ -226,14 +226,14 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do defp quoting_visibility(draft), do: draft - defp expires_at(draft) do + defp expires_at(%__MODULE__{} = draft) do case CommonAPI.check_expiry_date(draft.params[:expires_in]) do {:ok, expires_at} -> %__MODULE__{draft | expires_at: expires_at} {:error, message} -> add_error(draft, message) end end - defp poll(draft) do + defp poll(%__MODULE__{} = draft) do case Utils.make_poll_data(draft.params) do {:ok, {poll, poll_emoji}} -> %__MODULE__{draft | extra: poll, emoji: Map.merge(draft.emoji, poll_emoji)} @@ -243,7 +243,7 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do end end - defp content(%{mentions: mentions} = draft) do + defp content(%__MODULE__{mentions: mentions} = draft) do {content_html, mentioned_users, tags} = Utils.make_content_html(draft) mentioned_ap_ids = @@ -257,22 +257,22 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do %__MODULE__{draft | content_html: content_html, mentions: mentions, tags: tags} end - defp to_and_cc(draft) do + defp to_and_cc(%__MODULE__{} = draft) do {to, cc} = Utils.get_to_and_cc(draft) %__MODULE__{draft | to: to, cc: cc} end - defp context(draft) do + defp context(%__MODULE__{} = draft) do context = Utils.make_context(draft.in_reply_to, draft.in_reply_to_conversation) %__MODULE__{draft | context: context} end - defp sensitive(draft) do + defp sensitive(%__MODULE__{} = draft) do sensitive = draft.params[:sensitive] %__MODULE__{draft | sensitive: sensitive} end - defp language(draft) do + defp language(%__MODULE__{} = draft) do language = with language <- draft.params[:language], true <- good_locale_code?(language) do @@ -284,7 +284,7 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do %__MODULE__{draft | language: language} end - defp object(draft) do + defp object(%__MODULE__{} = draft) do emoji = Map.merge(Pleroma.Emoji.Formatter.get_emoji_map(draft.full_payload), draft.emoji) # Sometimes people create posts with subject containing emoji, @@ -328,12 +328,12 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do %__MODULE__{draft | object: object} end - defp preview?(draft) do + defp preview?(%__MODULE__{} = draft) do preview? = Pleroma.Web.Utils.Params.truthy_param?(draft.params[:preview]) %__MODULE__{draft | preview?: preview?} end - defp changes(draft) do + defp changes(%__MODULE__{} = draft) do direct? = draft.visibility == "direct" additional = %{"cc" => draft.cc, "directMessage" => direct?} @@ -359,7 +359,7 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do defp with_valid(%{valid?: true} = draft, func), do: func.(draft) defp with_valid(draft, _func), do: draft - defp add_error(draft, message) do + defp add_error(%__MODULE__{} = draft, message) do %__MODULE__{draft | valid?: false, errors: [message | draft.errors]} end From 9040434886526e082bf675e46e8cadb3f64e7fe9 Mon Sep 17 00:00:00 2001 From: Phantasm Date: Fri, 9 Jan 2026 16:40:19 +0100 Subject: [PATCH 031/183] Elixir 1.19: Fix typing violations in ActivityPubTest --- test/pleroma/web/activity_pub/activity_pub_test.exs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/pleroma/web/activity_pub/activity_pub_test.exs b/test/pleroma/web/activity_pub/activity_pub_test.exs index 37ec87a9b..13146619a 100644 --- a/test/pleroma/web/activity_pub/activity_pub_test.exs +++ b/test/pleroma/web/activity_pub/activity_pub_test.exs @@ -1524,8 +1524,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do %{test_file: test_file} end - test "strips / from filename", %{test_file: file} do - file = %Plug.Upload{file | filename: "../../../../../nested/bad.jpg"} + test "strips / from filename", %{test_file: %Plug.Upload{} = file} do + file = %{file | filename: "../../../../../nested/bad.jpg"} {:ok, %Object{} = object} = ActivityPub.upload(file) [%{"href" => href}] = object.data["url"] assert Regex.match?(~r"/bad.jpg$", href) @@ -1786,10 +1786,10 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do {:ok, list} = Pleroma.List.create(%{title: "foo"}, user) {:ok, list} = Pleroma.List.follow(list, member) - {:ok, activity} = CommonAPI.post(user, %{status: "foobar", visibility: "list:#{list.id}"}) + {:ok, %Activity{} = activity} = CommonAPI.post(user, %{status: "foobar", visibility: "list:#{list.id}"}) activity = Repo.preload(activity, :bookmark) - activity = %Activity{activity | thread_muted?: !!activity.thread_muted?} + activity = %{activity | thread_muted?: !!activity.thread_muted?} assert ActivityPub.fetch_activities([], %{user: user}) == [activity] end @@ -1989,7 +1989,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do assert User.following?(follower, old_user) assert User.following?(follower_move_opted_out, old_user) - assert {:ok, activity} = ActivityPub.move(old_user, new_user) + assert {:ok, %Activity{} = activity} = ActivityPub.move(old_user, new_user) assert %Activity{ actor: ^old_ap_id, @@ -2021,7 +2021,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do assert User.following?(follower_move_opted_out, old_user) refute User.following?(follower_move_opted_out, new_user) - activity = %Activity{activity | object: nil} + activity = %{activity | object: nil} assert [%Notification{activity: ^activity}] = Notification.for_user(follower) From 349d858fdade27c7a016a55ac69a435dadb59d2c Mon Sep 17 00:00:00 2001 From: Phantasm Date: Fri, 9 Jan 2026 16:42:26 +0100 Subject: [PATCH 032/183] Elixir 1.19: Fix typing violation in MediaControllerTest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit warning: a struct for Plug.Upload is expected on struct update: %Plug.Upload{image | filename: "../../../../../nested/file.jpg"} but got type: dynamic() where "image" was given the type: # type: dynamic() # from: test/pleroma/web/mastodon_api/controllers/media_controller_test.exs:132:42 %{conn: conn, image: image} when defining the variable "image", you must also pattern match on "%Plug.Upload{}". hint: given pattern matching is enough to catch typing errors, you may optionally convert the struct update into a map update. For example, instead of: user = some_function() %User{user | name: "John Doe"} it is enough to write: %User{} = user = some_function() %{user | name: "John Doe"} typing violation found at: │ 133 │ image = %Plug.Upload{ │ ~ │ └─ test/pleroma/web/mastodon_api/controllers/media_controller_test.exs:133:15: Pleroma.Web.MastodonAPI.MediaControllerTest."test Upload media Do not allow nested filename"/1 --- .../web/mastodon_api/controllers/media_controller_test.exs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/pleroma/web/mastodon_api/controllers/media_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/media_controller_test.exs index ae86078d7..fc083fd0e 100644 --- a/test/pleroma/web/mastodon_api/controllers/media_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/media_controller_test.exs @@ -129,8 +129,8 @@ defmodule Pleroma.Web.MastodonAPI.MediaControllerTest do assert :ok == File.rm(Path.absname("test/tmp/large_binary.data")) end - test "Do not allow nested filename", %{conn: conn, image: image} do - image = %Plug.Upload{ + test "Do not allow nested filename", %{conn: conn, image: %Plug.Upload{} = image} do + image = %{ image | filename: "../../../../../nested/file.jpg" } From f3d9880f07e70f4e83e25e13aab537e2ea3c89ba Mon Sep 17 00:00:00 2001 From: Phantasm Date: Fri, 9 Jan 2026 16:44:37 +0100 Subject: [PATCH 033/183] Elixir 1.19: Fix typing violation in RepoTest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit warning: a struct for Pleroma.Web.OAuth.Token is expected on struct update: %Pleroma.Web.OAuth.Token{Pleroma.Factory.insert(:oauth_token) | user: user} but got type: dynamic() you must assign "Pleroma.Factory.insert(:oauth_token)" to variable and pattern match on "%Pleroma.Web.OAuth.Token{}". hint: given pattern matching is enough to catch typing errors, you may optionally convert the struct update into a map update. For example, instead of: user = some_function() %User{user | name: "John Doe"} it is enough to write: %User{} = user = some_function() %{user | name: "John Doe"} typing violation found at: │ 27 │ token = %Pleroma.Web.OAuth.Token{insert(:oauth_token) | user: user} │ ~ │ └─ test/pleroma/repo_test.exs:27:15: Pleroma.RepoTest."test get_assoc/2 get assoc from preloaded data"/1 --- test/pleroma/repo_test.exs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/pleroma/repo_test.exs b/test/pleroma/repo_test.exs index 9c0f5d028..721175bda 100644 --- a/test/pleroma/repo_test.exs +++ b/test/pleroma/repo_test.exs @@ -24,7 +24,8 @@ defmodule Pleroma.RepoTest do describe "get_assoc/2" do test "get assoc from preloaded data" do user = %User{name: "Agent Smith"} - token = %Pleroma.Web.OAuth.Token{insert(:oauth_token) | user: user} + %Pleroma.Web.OAuth.Token{} = token = insert(:oauth_token) + token = %{token | user: user} assert Repo.get_assoc(token, :user) == {:ok, user} end From d1ce6f9833036aacc585963046043f1a5c4f5b14 Mon Sep 17 00:00:00 2001 From: Phantasm Date: Fri, 9 Jan 2026 16:48:55 +0100 Subject: [PATCH 034/183] Elixir 1.19: Fix typing violation in MarkerTest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit warning: a struct for Pleroma.Marker is expected on struct update: %Pleroma.Marker{refresh_record(marker) | unread_count: 2} but got type: dynamic() where "marker" was given the type: # type: dynamic() # from: test/pleroma/marker_test.exs:35:14 marker = Pleroma.Factory.insert(:marker, user: user) you must assign "refresh_record(marker)" to variable and pattern match on "%Pleroma.Marker{}". hint: given pattern matching is enough to catch typing errors, you may optionally convert the struct update into a map update. For example, instead of: user = some_function() %User{user | name: "John Doe"} it is enough to write: %User{} = user = some_function() %{user | name: "John Doe"} typing violation found at: │ 43 │ ) == [%Marker{refresh_record(marker) | unread_count: 2}] │ ~ │ └─ test/pleroma/marker_test.exs:43:20: Pleroma.MarkerTest."test get_markers/2 returns user markers"/1 --- test/pleroma/marker_test.exs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/pleroma/marker_test.exs b/test/pleroma/marker_test.exs index 819dde9be..7f573ac7a 100644 --- a/test/pleroma/marker_test.exs +++ b/test/pleroma/marker_test.exs @@ -36,11 +36,12 @@ defmodule Pleroma.MarkerTest do insert(:notification, user: user, activity: insert(:note_activity)) insert(:notification, user: user, activity: insert(:note_activity)) insert(:marker, timeline: "home", user: user) + %Marker{} = refreshed_marker = refresh_record(marker) assert Marker.get_markers( user, ["notifications"] - ) == [%Marker{refresh_record(marker) | unread_count: 2}] + ) == [%{refreshed_marker | unread_count: 2}] end end From 5777d8e8d3c33bc70bb3cda6b7bb72592a310e16 Mon Sep 17 00:00:00 2001 From: Phantasm Date: Fri, 9 Jan 2026 17:01:36 +0100 Subject: [PATCH 035/183] Elixir 1.19: Fix deprecation warning when invoking ParallelCompiler warning: you must pass return_diagnostics: true when invoking Kernel.ParallelCompiler functions (elixir 1.19.5) lib/kernel/parallel_compiler.ex:324: Kernel.ParallelCompiler.spawn_workers/3 (pleroma 2.10.0-7-ga7a74d5e-elixir-1-19+test) lib/pleroma/html.ex:13: Pleroma.HTML.compile_scrubbers/0 (pleroma 2.10.0-7-ga7a74d5e-elixir-1-19+test) lib/pleroma/application.ex:48: Pleroma.Application.start/2 (kernel 10.5) application_master.erl:299: :application_master.start_it_old/4 warning: you must pass return_diagnostics: true when invoking Kernel.ParallelCompiler functions (elixir 1.19.5) lib/kernel/parallel_compiler.ex:324: Kernel.ParallelCompiler.spawn_workers/3 (pleroma 2.10.0-7-ga7a74d5e-elixir-1-19+test) lib/pleroma/application.ex:121: Pleroma.Application.load_custom_modules/0 (pleroma 2.10.0-7-ga7a74d5e-elixir-1-19+test) lib/pleroma/application.ex:60: Pleroma.Application.start/2 (kernel 10.5) application_master.erl:299: :application_master.start_it_old/4 --- lib/pleroma/utils.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/utils.ex b/lib/pleroma/utils.ex index 73001c987..61d122a47 100644 --- a/lib/pleroma/utils.ex +++ b/lib/pleroma/utils.ex @@ -17,7 +17,7 @@ defmodule Pleroma.Utils do dir |> File.ls!() |> Enum.map(&Path.join(dir, &1)) - |> Kernel.ParallelCompiler.compile() + |> Kernel.ParallelCompiler.compile(return_diagnostics: true) end @doc """ From 94ea3dffd1c49fadc6a80c34127ea59c7fd87311 Mon Sep 17 00:00:00 2001 From: Phantasm Date: Fri, 9 Jan 2026 17:05:31 +0100 Subject: [PATCH 036/183] Elixir 1.19: Fix ConfigDBTest regex tests It is not possible match regexes anymore as this worked by accident previously. Instead, at least check that the sources of the regex (the regex itself) match. Notice the +1 difference in the regex Reference below. 1) test to_elixir_types/1 complex keyword with sigil (Pleroma.ConfigDBTest) test/pleroma/config_db_test.exs:460 Assertion with == failed code: assert ConfigDB.to_elixir_types([ %{"tuple" => [":federated_timeline_removal", []]}, %{"tuple" => [":reject", ["~r/comp[lL][aA][iI][nN]er/"]]}, %{"tuple" => [":replace", []]} ]) == [federated_timeline_removal: [], reject: [~r/comp[lL][aA][iI][nN]er/], replace: []] left: [federated_timeline_removal: [], reject: [%Regex{opts: [], re_pattern: {:re_pattern, 0, 0, 0, #Reference<0.230935836.591265794.259515>}, source: "comp[lL][aA][iI][nN]er"}], replace: []] right: [federated_timeline_removal: [], reject: [%Regex{opts: [], re_pattern: {:re_pattern, 0, 0, 0, #Reference<0.230935836.591265794.259516>}, source: "comp[lL][aA][iI][nN]er"}], replace: []] stacktrace: test/pleroma/config_db_test.exs:461: (test) --- test/pleroma/config_db_test.exs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/test/pleroma/config_db_test.exs b/test/pleroma/config_db_test.exs index d68e4e6fa..59beba48e 100644 --- a/test/pleroma/config_db_test.exs +++ b/test/pleroma/config_db_test.exs @@ -273,24 +273,24 @@ defmodule Pleroma.ConfigDBTest do end test "sigil" do - assert ConfigDB.to_elixir_types("~r[comp[lL][aA][iI][nN]er]") == ~r/comp[lL][aA][iI][nN]er/ + assert ConfigDB.to_elixir_types("~r[comp[lL][aA][iI][nN]er]").source == ~r/comp[lL][aA][iI][nN]er/.source end test "link sigil" do - assert ConfigDB.to_elixir_types("~r/https:\/\/example.com/") == ~r/https:\/\/example.com/ + assert ConfigDB.to_elixir_types("~r/https:\/\/example.com/").source == ~r/https:\/\/example.com/.source end test "link sigil with um modifiers" do - assert ConfigDB.to_elixir_types("~r/https:\/\/example.com/um") == - ~r/https:\/\/example.com/um + assert ConfigDB.to_elixir_types("~r/https:\/\/example.com/um").source == + ~r/https:\/\/example.com/um.source end test "link sigil with i modifier" do - assert ConfigDB.to_elixir_types("~r/https:\/\/example.com/i") == ~r/https:\/\/example.com/i + assert ConfigDB.to_elixir_types("~r/https:\/\/example.com/i").source == ~r/https:\/\/example.com/i.source end test "link sigil with s modifier" do - assert ConfigDB.to_elixir_types("~r/https:\/\/example.com/s") == ~r/https:\/\/example.com/s + assert ConfigDB.to_elixir_types("~r/https:\/\/example.com/s").source == ~r/https:\/\/example.com/s.source end test "raise if valid delimiter not found" do @@ -460,11 +460,11 @@ defmodule Pleroma.ConfigDBTest do test "complex keyword with sigil" do assert ConfigDB.to_elixir_types([ %{"tuple" => [":federated_timeline_removal", []]}, - %{"tuple" => [":reject", ["~r/comp[lL][aA][iI][nN]er/"]]}, + %{"tuple" => [":reject", [~r/comp[lL][aA][iI][nN]er/.source]]}, %{"tuple" => [":replace", []]} ]) == [ federated_timeline_removal: [], - reject: [~r/comp[lL][aA][iI][nN]er/], + reject: [~r/comp[lL][aA][iI][nN]er/.source], replace: [] ] end From cbacc0f146eaad98eb31574142a076658ae98f3f Mon Sep 17 00:00:00 2001 From: Phantasm Date: Fri, 9 Jan 2026 17:13:10 +0100 Subject: [PATCH 037/183] Elixir 1.19: Fix MRFTest regex tests It is no longer possible to match regexes. Instead at least match that the sources of the regexes (regexes themselves) are the same. Notice the +1 Reference number below. 2) test subdomain_match/2 wildcard domains with one subdomain (Pleroma.Web.ActivityPub.MRFTest) test/pleroma/web/activity_pub/mrf_test.exs:36 Assertion with == failed code: assert regexes == [~r/^(.*\.)*unsafe.tld$/i] left: [%Regex{opts: [:caseless], re_pattern: {:re_pattern, 1, 0, 0, #Reference<0.378940835.3277193222.129648>}, source: "^(.*\\.)*unsafe.tld$"}] right: [%Regex{opts: [:caseless], re_pattern: {:re_pattern, 1, 0, 0, #Reference<0.378940835.3277193222.129649>}, source: "^(.*\\.)*unsafe.tld$"}] stacktrace: test/pleroma/web/activity_pub/mrf_test.exs:39: (test) --- test/pleroma/web/activity_pub/mrf_test.exs | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/test/pleroma/web/activity_pub/mrf_test.exs b/test/pleroma/web/activity_pub/mrf_test.exs index 25548e3da..6656a01b8 100644 --- a/test/pleroma/web/activity_pub/mrf_test.exs +++ b/test/pleroma/web/activity_pub/mrf_test.exs @@ -11,17 +11,21 @@ defmodule Pleroma.Web.ActivityPub.MRFTest do alias Pleroma.Web.ActivityPub.MRF test "subdomains_regex/1" do - assert MRF.subdomains_regex(["unsafe.tld", "*.unsafe.tld"]) == [ - ~r/^unsafe.tld$/i, - ~r/^(.*\.)*unsafe.tld$/i + regexes = MRF.subdomains_regex(["unsafe.tld", "*.unsafe.tld"]) + matchable_regexes = Enum.map(regexes, fn r -> r.source end) + + assert matchable_regexes == [ + ~r/^unsafe.tld$/i.source, + ~r/^(.*\.)*unsafe.tld$/i.source ] end describe "subdomain_match/2" do test "common domains" do regexes = MRF.subdomains_regex(["unsafe.tld", "unsafe2.tld"]) + matchable_regexes = Enum.map(regexes, fn r -> r.source end) - assert regexes == [~r/^unsafe.tld$/i, ~r/^unsafe2.tld$/i] + assert matchable_regexes == [~r/^unsafe.tld$/i.source, ~r/^unsafe2.tld$/i.source] assert MRF.subdomain_match?(regexes, "unsafe.tld") assert MRF.subdomain_match?(regexes, "unsafe2.tld") @@ -31,8 +35,9 @@ defmodule Pleroma.Web.ActivityPub.MRFTest do test "wildcard domains with one subdomain" do regexes = MRF.subdomains_regex(["*.unsafe.tld"]) + matchable_regexes = Enum.map(regexes, fn r -> r.source end) - assert regexes == [~r/^(.*\.)*unsafe.tld$/i] + assert matchable_regexes == [~r/^(.*\.)*unsafe.tld$/i.source] assert MRF.subdomain_match?(regexes, "unsafe.tld") assert MRF.subdomain_match?(regexes, "sub.unsafe.tld") @@ -42,8 +47,9 @@ defmodule Pleroma.Web.ActivityPub.MRFTest do test "wildcard domains with two subdomains" do regexes = MRF.subdomains_regex(["*.unsafe.tld"]) + matchable_regexes = Enum.map(regexes, fn r -> r.source end) - assert regexes == [~r/^(.*\.)*unsafe.tld$/i] + assert matchable_regexes == [~r/^(.*\.)*unsafe.tld$/i.source] assert MRF.subdomain_match?(regexes, "unsafe.tld") assert MRF.subdomain_match?(regexes, "sub.sub.unsafe.tld") @@ -53,8 +59,9 @@ defmodule Pleroma.Web.ActivityPub.MRFTest do test "matches are case-insensitive" do regexes = MRF.subdomains_regex(["UnSafe.TLD", "UnSAFE2.Tld"]) + matchable_regexes = Enum.map(regexes, fn r -> r.source end) - assert regexes == [~r/^UnSafe.TLD$/i, ~r/^UnSAFE2.Tld$/i] + assert matchable_regexes == [~r/^UnSafe.TLD$/i.source, ~r/^UnSAFE2.Tld$/i.source] assert MRF.subdomain_match?(regexes, "UNSAFE.TLD") assert MRF.subdomain_match?(regexes, "UNSAFE2.TLD") From 3c7cd8b043237fb0ece9c314b73e335dfd28ffcb Mon Sep 17 00:00:00 2001 From: Phantasm Date: Fri, 9 Jan 2026 17:28:06 +0100 Subject: [PATCH 038/183] Elixir 1.19: Fix Mastodon StatusControllerTest DateTime difference --- .../web/mastodon_api/controllers/status_controller_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs index 298e92366..8477ae03f 100644 --- a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs @@ -3068,7 +3068,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do |> json_response_and_validate_schema(:ok) {:ok, a_expires_at, 0} = DateTime.from_iso8601(a_expires_at) - assert DateTime.diff(expires_at, a_expires_at) == 0 + assert DateTime.diff(DateTime.truncate(expires_at, :second), DateTime.truncate(a_expires_at, :second)) == 0 %{conn: conn} = oauth_access(["read:statuses"]) From dd2f0a32d8a0ab918cf840b937158e5b6d9418b3 Mon Sep 17 00:00:00 2001 From: Phantasm Date: Fri, 9 Jan 2026 20:16:30 +0100 Subject: [PATCH 039/183] lint --- lib/pleroma/mfa/changeset.ex | 7 ++++--- lib/pleroma/web/api_spec/cast_and_validate.ex | 9 ++++++++- lib/pleroma/web/common_api/activity_draft.ex | 10 +++++++--- test/pleroma/config_db_test.exs | 12 ++++++++---- test/pleroma/web/activity_pub/activity_pub_test.exs | 3 ++- .../controllers/status_controller_test.exs | 6 +++++- 6 files changed, 34 insertions(+), 13 deletions(-) diff --git a/lib/pleroma/mfa/changeset.ex b/lib/pleroma/mfa/changeset.ex index 890cb2193..2045c3a7c 100644 --- a/lib/pleroma/mfa/changeset.ex +++ b/lib/pleroma/mfa/changeset.ex @@ -8,7 +8,8 @@ defmodule Pleroma.MFA.Changeset do alias Pleroma.User def disable(%Ecto.Changeset{} = changeset, force \\ false) do - %Settings{} = settings = + %Settings{} = + settings = changeset |> Ecto.Changeset.apply_changes() |> MFA.fetch_settings() @@ -26,7 +27,7 @@ defmodule Pleroma.MFA.Changeset do end def confirm_totp(%User{multi_factor_authentication_settings: %Settings{} = settings} = user) do - totp_settings = %Settings.TOTP{%Settings.TOTP{} = settings.totp | confirmed: true} + totp_settings = %Settings.TOTP{(%Settings.TOTP{} = settings.totp) | confirmed: true} user |> put_change(%Settings{settings | totp: totp_settings, enabled: true}) @@ -46,7 +47,7 @@ defmodule Pleroma.MFA.Changeset do def cast_backup_codes(%User{} = user, codes) do user |> put_change(%Settings{ - %Settings{} = user.multi_factor_authentication_settings + (%Settings{} = user.multi_factor_authentication_settings) | backup_codes: codes }) end diff --git a/lib/pleroma/web/api_spec/cast_and_validate.ex b/lib/pleroma/web/api_spec/cast_and_validate.ex index 57ea8e9b3..95bd4d9cf 100644 --- a/lib/pleroma/web/api_spec/cast_and_validate.ex +++ b/lib/pleroma/web/api_spec/cast_and_validate.ex @@ -106,7 +106,14 @@ defmodule Pleroma.Web.ApiSpec.CastAndValidate do OpenApiSpex.cast_and_validate(spec, operation, conn, content_type, cast_opts) end - defp cast_and_validate(spec, operation, %Conn{} = conn, content_type, false = _strict, cast_opts) do + defp cast_and_validate( + spec, + operation, + %Conn{} = conn, + content_type, + false = _strict, + cast_opts + ) do case OpenApiSpex.cast_and_validate(spec, operation, conn, content_type) do {:ok, conn} -> {:ok, conn} diff --git a/lib/pleroma/web/common_api/activity_draft.ex b/lib/pleroma/web/common_api/activity_draft.ex index c5959276a..6c84ef656 100644 --- a/lib/pleroma/web/common_api/activity_draft.ex +++ b/lib/pleroma/web/common_api/activity_draft.ex @@ -136,7 +136,8 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do defp in_reply_to(%__MODULE__{params: %{in_reply_to_status_id: ""}} = draft), do: draft - defp in_reply_to(%__MODULE__{params: %{in_reply_to_status_id: id}} = draft) when is_binary(id) do + defp in_reply_to(%__MODULE__{params: %{in_reply_to_status_id: id}} = draft) + when is_binary(id) do # If a post was deleted all its activities (except the newly added Delete) are purged too, # thus lookup by Create db ID will yield nil just as if it never existed in the first place. # @@ -166,13 +167,16 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do end end - defp in_reply_to(%__MODULE__{params: %{in_reply_to_status_id: %Activity{} = in_reply_to}} = draft) do + defp in_reply_to( + %__MODULE__{params: %{in_reply_to_status_id: %Activity{} = in_reply_to}} = draft + ) do %__MODULE__{draft | in_reply_to: in_reply_to} end defp in_reply_to(draft), do: draft - defp quote_post(%__MODULE__{params: %{quoted_status_id: id}} = draft) when not_empty_string(id) do + defp quote_post(%__MODULE__{params: %{quoted_status_id: id}} = draft) + when not_empty_string(id) do case Activity.get_by_id_with_object(id) do %Activity{} = activity -> %__MODULE__{draft | quote_post: activity} diff --git a/test/pleroma/config_db_test.exs b/test/pleroma/config_db_test.exs index 59beba48e..98b56a146 100644 --- a/test/pleroma/config_db_test.exs +++ b/test/pleroma/config_db_test.exs @@ -273,11 +273,13 @@ defmodule Pleroma.ConfigDBTest do end test "sigil" do - assert ConfigDB.to_elixir_types("~r[comp[lL][aA][iI][nN]er]").source == ~r/comp[lL][aA][iI][nN]er/.source + assert ConfigDB.to_elixir_types("~r[comp[lL][aA][iI][nN]er]").source == + ~r/comp[lL][aA][iI][nN]er/.source end test "link sigil" do - assert ConfigDB.to_elixir_types("~r/https:\/\/example.com/").source == ~r/https:\/\/example.com/.source + assert ConfigDB.to_elixir_types("~r/https:\/\/example.com/").source == + ~r/https:\/\/example.com/.source end test "link sigil with um modifiers" do @@ -286,11 +288,13 @@ defmodule Pleroma.ConfigDBTest do end test "link sigil with i modifier" do - assert ConfigDB.to_elixir_types("~r/https:\/\/example.com/i").source == ~r/https:\/\/example.com/i.source + assert ConfigDB.to_elixir_types("~r/https:\/\/example.com/i").source == + ~r/https:\/\/example.com/i.source end test "link sigil with s modifier" do - assert ConfigDB.to_elixir_types("~r/https:\/\/example.com/s").source == ~r/https:\/\/example.com/s.source + assert ConfigDB.to_elixir_types("~r/https:\/\/example.com/s").source == + ~r/https:\/\/example.com/s.source end test "raise if valid delimiter not found" do diff --git a/test/pleroma/web/activity_pub/activity_pub_test.exs b/test/pleroma/web/activity_pub/activity_pub_test.exs index 13146619a..9aafc41a5 100644 --- a/test/pleroma/web/activity_pub/activity_pub_test.exs +++ b/test/pleroma/web/activity_pub/activity_pub_test.exs @@ -1786,7 +1786,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do {:ok, list} = Pleroma.List.create(%{title: "foo"}, user) {:ok, list} = Pleroma.List.follow(list, member) - {:ok, %Activity{} = activity} = CommonAPI.post(user, %{status: "foobar", visibility: "list:#{list.id}"}) + {:ok, %Activity{} = activity} = + CommonAPI.post(user, %{status: "foobar", visibility: "list:#{list.id}"}) activity = Repo.preload(activity, :bookmark) activity = %{activity | thread_muted?: !!activity.thread_muted?} diff --git a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs index 8477ae03f..11e96a6ac 100644 --- a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs @@ -3068,7 +3068,11 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do |> json_response_and_validate_schema(:ok) {:ok, a_expires_at, 0} = DateTime.from_iso8601(a_expires_at) - assert DateTime.diff(DateTime.truncate(expires_at, :second), DateTime.truncate(a_expires_at, :second)) == 0 + + assert DateTime.diff( + DateTime.truncate(expires_at, :second), + DateTime.truncate(a_expires_at, :second) + ) == 0 %{conn: conn} = oauth_access(["read:statuses"]) From 926cad3a76972835d340a7cabb0e90b8c01f51eb Mon Sep 17 00:00:00 2001 From: Phantasm Date: Sat, 10 Jan 2026 16:42:09 +0100 Subject: [PATCH 040/183] Elixir 1.19: Refactor Logger configuration to what Elixir 1.15+ expects Elixir 1.19 started emitting warnings about our use of: config :logger, :console and config :logger, backends: [] which got split in 1.15 to :default_handler, :default_formatter, and LoggerBackends.add(backend) respectively. On Pleroma startup it is now necessary to add the additional logger backends at runtime (done with configure_logger() in application.ex). New config key :pleroma, :logger_backends: [] has replaced the older backends array. The logger_backends package is now used to facilitate usage of other logger backends besides the default one (:logger_std_h from OTP). Warnings were added to Pleroma startup to warn about the usage of :logger, backends and additional logic was introduced to make addition of ExSyslogger to admin's configuration less dumb. Ref: https://hexdocs.pm/logger/1.15.0/Logger.html#module-backends-and-backwards-compatibility --- config/config.exs | 9 +++++---- config/dev.exs | 2 +- config/prod.exs | 3 +-- config/test.exs | 6 +++--- docs/configuration/cheatsheet.md | 25 +++++++++++++------------ lib/pleroma/application.ex | 32 ++++++++++++++++++++++++++++++++ 6 files changed, 55 insertions(+), 22 deletions(-) diff --git a/config/config.exs b/config/config.exs index 5bf2c5c2e..08311146e 100644 --- a/config/config.exs +++ b/config/config.exs @@ -132,11 +132,12 @@ config :pleroma, Pleroma.Web.Endpoint, "SameSite=Lax" ] -# Configures Elixir's Logger -config :logger, backends: [:console] +# config :logger, :console has been changed in 1.15+ and split into +# :default_handler and :default_formatter. It started emitting warnings in 1.19. +# See https://hexdocs.pm/logger/1.19.0/Logger.html#module-backends-and-backwards-compatibility +config :logger, :default_handler, level: :debug -config :logger, :console, - level: :debug, +config :logger, :default_formatter, format: "\n$time $metadata[$level] $message\n", metadata: [:actor, :path, :type, :user] diff --git a/config/dev.exs b/config/dev.exs index 14cf4a6dc..e86bdb7fd 100644 --- a/config/dev.exs +++ b/config/dev.exs @@ -36,7 +36,7 @@ config :pleroma, Pleroma.Emails.Mailer, adapter: Swoosh.Adapters.Local # different ports. # Do not include timestamps in development logs -config :logger, Logger.Backends.Console, format: "$metadata[$level] $message\n" +config :logger, :default_formatter, format: "$metadata[$level] $message\n" # Set a higher stacktrace during development. Avoid configuring such # in production as building large stacktraces may be expensive. diff --git a/config/prod.exs b/config/prod.exs index 2d252bf02..1dfbbe937 100644 --- a/config/prod.exs +++ b/config/prod.exs @@ -20,8 +20,7 @@ config :pleroma, Pleroma.Web.Endpoint, config :phoenix, serve_endpoints: true # Do not print debug messages in production -config :logger, Logger.Backends.Console, level: :info -config :logger, :console, level: :info +config :logger, :default_handler, level: :info config :logger, :ex_syslogger, level: :info # ## SSL Support diff --git a/config/test.exs b/config/test.exs index 9652b7d4b..bac5e4df3 100644 --- a/config/test.exs +++ b/config/test.exs @@ -15,9 +15,9 @@ config :pleroma, Pleroma.Captcha, method: Pleroma.Captcha.Mock # Print only warnings and errors during test -config :logger, :console, - level: :warning, - format: "\n[$level] $message\n" +config :logger, :default_handler, level: :warning + +config :logger, :default_formatter, format: "\n[$level] $message\n" config :pleroma, :auth, oauth_consumer_strategies: [] diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index 9efa1c8b3..2ab2652b8 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -858,12 +858,14 @@ Web Push Notifications configuration. You can use the mix task `mix web_push.gen * ``private_key``: VAPID private key ## :logger -* `backends`: `:console` is used to send logs to stdout, `{ExSyslogger, :ex_syslogger}` to log to syslog +* Logging to console/stdout is done by default, use `{ExSyslogger, :ex_syslogger}` to log to syslog An example to enable ONLY ExSyslogger (f/ex in ``prod.secret.exs``) with info and debug suppressed: ```elixir -config :logger, - backends: [{ExSyslogger, :ex_syslogger}] +config :pleroma, + logger_backends: [{ExSyslogger, :ex_syslogger}] + +config :logger, default_handler: false config :logger, :ex_syslogger, level: :warning @@ -871,8 +873,8 @@ config :logger, :ex_syslogger, Another example, keeping console output and adding the pid to syslog output: ```elixir -config :logger, - backends: [:console, {ExSyslogger, :ex_syslogger}] +config :pleroma, + logger_backends: [{ExSyslogger, :ex_syslogger}] config :logger, :ex_syslogger, level: :warning, @@ -883,23 +885,22 @@ See: [logger’s documentation](https://hexdocs.pm/logger/Logger.html) and [ex_s An example of logging info to local syslog, but debug to console: ```elixir -config :logger, - backends: [ {ExSyslogger, :ex_syslogger}, :console ], - level: :info +config :pleroma, + logger_backends: [ {ExSyslogger, :ex_syslogger}] config :logger, :ex_syslogger, level: :info, ident: "pleroma", format: "$metadata[$level] $message" -config :logger, :console, - level: :debug, +config :logger, :default_handler, + level: :debug + +config :logger, :default_formatter, format: "\n$time $metadata[$level] $message\n", metadata: [:request_id] ``` - - ## Database options ### RUM indexing for full text search diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 8e1c5de0d..7355a501a 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -43,6 +43,7 @@ defmodule Pleroma.Application do # every time the application is restarted, so we disable module # conflicts at runtime Code.compiler_options(ignore_module_conflict: true) + configure_logger() Pleroma.Telemetry.Logger.attach() Config.Holder.save_default() Pleroma.HTML.compile_scrubbers() @@ -113,6 +114,37 @@ defmodule Pleroma.Application do Supervisor.start_link(children, opts) end + def configure_logger do + if Application.get_env(:logger, :backends) do + Logger.warning( + "'config :logger, backends: [...]' is deprecated syntax due to changes in Elixir. Use 'config :pleroma, :logger_backends: [...]' instead." + ) + end + + Config.get([:logger_backends], []) + |> Enum.each(fn backend -> + backend = backend_to_logger(backend) + + case LoggerBackends.add(backend) do + {:ok, _} -> + Logger.debug("Successfully added logger backend: #{inspect(backend)}") + + {:error, reason} -> + Logger.error("Failed to add logger backend #{inspect(backend)}: #{inspect(reason)}") + end + end) + end + + defp backend_to_logger({:ex_syslogger = backend, name}) do + Logger.warning( + "Configuration {:#{backend}, :#{name}} is incorrect. Use {ExSyslogger, :#{name}} instead!" + ) + + {ExSyslogger, name} + end + + defp backend_to_logger(backend), do: backend + def load_custom_modules do dir = Config.get([:modules, :runtime_dir]) From 0393f0f92dcd68f7615b98139a35055d272e242a Mon Sep 17 00:00:00 2001 From: Phantasm Date: Mon, 12 Jan 2026 18:03:24 +0100 Subject: [PATCH 041/183] Elixir 1.19: Move Logger backends under a specific config key --- docs/configuration/cheatsheet.md | 12 ++++++------ lib/pleroma/application.ex | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index 2ab2652b8..01d81f6ca 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -862,8 +862,8 @@ Web Push Notifications configuration. You can use the mix task `mix web_push.gen An example to enable ONLY ExSyslogger (f/ex in ``prod.secret.exs``) with info and debug suppressed: ```elixir -config :pleroma, - logger_backends: [{ExSyslogger, :ex_syslogger}] +config :pleroma, :logger, + backends: [{ExSyslogger, :ex_syslogger}] config :logger, default_handler: false @@ -873,8 +873,8 @@ config :logger, :ex_syslogger, Another example, keeping console output and adding the pid to syslog output: ```elixir -config :pleroma, - logger_backends: [{ExSyslogger, :ex_syslogger}] +config :pleroma, :logger, + backends: [{ExSyslogger, :ex_syslogger}] config :logger, :ex_syslogger, level: :warning, @@ -885,8 +885,8 @@ See: [logger’s documentation](https://hexdocs.pm/logger/Logger.html) and [ex_s An example of logging info to local syslog, but debug to console: ```elixir -config :pleroma, - logger_backends: [ {ExSyslogger, :ex_syslogger}] +config :pleroma, :logger, + backends: [ {ExSyslogger, :ex_syslogger}] config :logger, :ex_syslogger, level: :info, diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 7355a501a..ce5b4f8d6 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -117,11 +117,11 @@ defmodule Pleroma.Application do def configure_logger do if Application.get_env(:logger, :backends) do Logger.warning( - "'config :logger, backends: [...]' is deprecated syntax due to changes in Elixir. Use 'config :pleroma, :logger_backends: [...]' instead." + "'config :logger, backends: [...]' is deprecated syntax due to changes in Elixir. Use 'config :pleroma, :logger, backends: [...]' instead." ) end - Config.get([:logger_backends], []) + Config.get([:logger, :backends], []) |> Enum.each(fn backend -> backend = backend_to_logger(backend) From 3790e6786cce376cbabf2588beed27d0daf9c55c Mon Sep 17 00:00:00 2001 From: Phantasm Date: Mon, 12 Jan 2026 18:21:46 +0100 Subject: [PATCH 042/183] Only warn about bad clear_config usage if warned usage is met --- test/support/helpers.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/support/helpers.ex b/test/support/helpers.ex index 7fa6c31a4..259180e8f 100644 --- a/test/support/helpers.ex +++ b/test/support/helpers.ex @@ -73,7 +73,8 @@ defmodule Pleroma.Tests.Helpers do defmacro clear_config(config_path, temp_setting) do # NOTE: `clear_config([section, key], value)` != `clear_config([section], key: value)` (!) # Displaying a warning to prevent unintentional clearing of all but one keys in section - if Keyword.keyword?(temp_setting) and length(temp_setting) == 1 do + if is_list(config_path) and length(config_path) == 1 and Keyword.keyword?(temp_setting) and + Keyword.keys(temp_setting) == [:key] do Logger.warning( "Please change `clear_config([section], key: value)` to `clear_config([section, key], value)`" ) From eee54a9226b1b4642bb2a71daa0921ba0022ce48 Mon Sep 17 00:00:00 2001 From: Phantasm Date: Mon, 12 Jan 2026 18:24:11 +0100 Subject: [PATCH 043/183] Elixir 1.19: Add tests for new Logger configuration --- test/pleroma/application_test.exs | 37 +++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 test/pleroma/application_test.exs diff --git a/test/pleroma/application_test.exs b/test/pleroma/application_test.exs new file mode 100644 index 000000000..bff44f4ff --- /dev/null +++ b/test/pleroma/application_test.exs @@ -0,0 +1,37 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2026 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.ApplicationTest do + use Pleroma.DataCase + + import ExUnit.CaptureLog + + # clear_config/2 can only manipulate env under :pleroma. + describe "config :logger, backends: []" do + setup do + Application.put_env(:logger, :backends, [:console, {ExSyslogger, :ex_syslogger}]) + + on_exit(fn -> + Application.delete_env(:logger, :backends) end) + end + + test "warns on deprecated syntax" do + assert capture_log(fn -> Pleroma.Application.configure_logger() end) =~ + "'config :logger, backends: [...]' is deprecated syntax due to changes in Elixir. Use 'config :pleroma, :logger, backends: [...]' instead." + end + end + + describe "config :pleroma, :logger, backends: [{:ex_syslogger, :ex_syslogger}]" do + setup do + clear_config([:logger, :backends], [{:ex_syslogger, :ex_syslogger}]) + end + + test "is handled" do + log = capture_log(fn -> Pleroma.Application.configure_logger() end) + + assert log =~ "Configuration {:ex_syslogger, :ex_syslogger} is incorrect. Use {ExSyslogger, :ex_syslogger} instead!" + assert log =~ "Successfully added logger backend: {ExSyslogger, :ex_syslogger}" + end + end +end From 3ebf4b105629b137f620de7b71167463ad3ca6be Mon Sep 17 00:00:00 2001 From: Phantasm Date: Mon, 12 Jan 2026 19:31:15 +0100 Subject: [PATCH 044/183] Elixir 1.19: Move deprecated logger syntax check to DeprecationWarnings --- lib/pleroma/application.ex | 6 ------ lib/pleroma/config/deprecation_warnings.ex | 17 ++++++++++++++++- test/pleroma/application_test.exs | 15 --------------- .../config/deprecation_warnings_test.exs | 15 +++++++++++++++ 4 files changed, 31 insertions(+), 22 deletions(-) diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index ce5b4f8d6..1e80c6249 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -115,12 +115,6 @@ defmodule Pleroma.Application do end def configure_logger do - if Application.get_env(:logger, :backends) do - Logger.warning( - "'config :logger, backends: [...]' is deprecated syntax due to changes in Elixir. Use 'config :pleroma, :logger, backends: [...]' instead." - ) - end - Config.get([:logger, :backends], []) |> Enum.each(fn backend -> backend = backend_to_logger(backend) diff --git a/lib/pleroma/config/deprecation_warnings.ex b/lib/pleroma/config/deprecation_warnings.ex index 58d164dc7..8dabd728f 100644 --- a/lib/pleroma/config/deprecation_warnings.ex +++ b/lib/pleroma/config/deprecation_warnings.ex @@ -218,7 +218,8 @@ defmodule Pleroma.Config.DeprecationWarnings do check_quarantined_instances_tuples(), check_transparency_exclusions_tuples(), check_simple_policy_tuples(), - check_exiftool_filter() + check_exiftool_filter(), + check_deprecated_logger_config() ] |> Enum.reduce(:ok, fn :ok, :ok -> :ok @@ -415,4 +416,18 @@ defmodule Pleroma.Config.DeprecationWarnings do :ok end end + + @spec check_deprecated_logger_config() :: :ok | :error + def check_deprecated_logger_config() do + if Application.get_env(:logger, :backends) do + Logger.warning( + "'config :logger, backends: [...]' is deprecated syntax due to changes in Elixir. " <> + "Use 'config :pleroma, :logger, backends: [...]' instead." + ) + + :error + else + :ok + end + end end diff --git a/test/pleroma/application_test.exs b/test/pleroma/application_test.exs index bff44f4ff..afe628732 100644 --- a/test/pleroma/application_test.exs +++ b/test/pleroma/application_test.exs @@ -7,21 +7,6 @@ defmodule Pleroma.ApplicationTest do import ExUnit.CaptureLog - # clear_config/2 can only manipulate env under :pleroma. - describe "config :logger, backends: []" do - setup do - Application.put_env(:logger, :backends, [:console, {ExSyslogger, :ex_syslogger}]) - - on_exit(fn -> - Application.delete_env(:logger, :backends) end) - end - - test "warns on deprecated syntax" do - assert capture_log(fn -> Pleroma.Application.configure_logger() end) =~ - "'config :logger, backends: [...]' is deprecated syntax due to changes in Elixir. Use 'config :pleroma, :logger, backends: [...]' instead." - end - end - describe "config :pleroma, :logger, backends: [{:ex_syslogger, :ex_syslogger}]" do setup do clear_config([:logger, :backends], [{:ex_syslogger, :ex_syslogger}]) diff --git a/test/pleroma/config/deprecation_warnings_test.exs b/test/pleroma/config/deprecation_warnings_test.exs index fca2324ff..1491f4ea3 100644 --- a/test/pleroma/config/deprecation_warnings_test.exs +++ b/test/pleroma/config/deprecation_warnings_test.exs @@ -388,4 +388,19 @@ defmodule Pleroma.Config.DeprecationWarningsTest do end) =~ "Your config is using the old namespace for the Shoutbox configuration." end + + describe "check_deprecated_logger_config" do + setup do + Application.put_env(:logger, :backends, [:console, {ExSyslogger, :ex_syslogger}]) + + on_exit(fn -> + Application.delete_env(:logger, :backends) end) + end + + test "warns on deprecated syntax" do + assert capture_log(fn -> Pleroma.Config.DeprecationWarnings.check_deprecated_logger_config() end) =~ + "'config :logger, backends: [...]' is deprecated syntax due to changes in Elixir. Use 'config :pleroma, :logger, backends: [...]' instead." + end + end + end From 39409a784df0a726aee3a6e48769344d83cd81f2 Mon Sep 17 00:00:00 2001 From: Phantasm Date: Mon, 12 Jan 2026 19:35:12 +0100 Subject: [PATCH 045/183] Elixir 1.19: Warn on adding :console to Logger backends Logger.Backend.Console still exists, but it is unused in our configuration. Instead logging to console is done from Logger's "default_handler". --- lib/pleroma/application.ex | 28 +++++++++++++++++++--------- test/pleroma/application_test.exs | 11 +++++++++++ 2 files changed, 30 insertions(+), 9 deletions(-) diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 1e80c6249..0e8723251 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -117,27 +117,37 @@ defmodule Pleroma.Application do def configure_logger do Config.get([:logger, :backends], []) |> Enum.each(fn backend -> - backend = backend_to_logger(backend) + case backend_to_logger(backend) do + {:ok, logger} -> + add_logger(logger) - case LoggerBackends.add(backend) do - {:ok, _} -> - Logger.debug("Successfully added logger backend: #{inspect(backend)}") - - {:error, reason} -> - Logger.error("Failed to add logger backend #{inspect(backend)}: #{inspect(reason)}") + {:error, :console_backend} -> + Logger.warning(":console is no longer considered a backend and is enabled by default") end end) end + defp add_logger(backend) do + case LoggerBackends.add(backend) do + {:ok, _} -> + Logger.debug("Successfully added logger backend: #{inspect(backend)}") + + {:error, reason} -> + Logger.error("Failed to add logger backend #{inspect(backend)}: #{inspect(reason)}") + end + end + + defp backend_to_logger(:console), do: {:error, :console_backend} + defp backend_to_logger({:ex_syslogger = backend, name}) do Logger.warning( "Configuration {:#{backend}, :#{name}} is incorrect. Use {ExSyslogger, :#{name}} instead!" ) - {ExSyslogger, name} + {:ok, {ExSyslogger, name}} end - defp backend_to_logger(backend), do: backend + defp backend_to_logger(backend), do: {:ok, backend} def load_custom_modules do dir = Config.get([:modules, :runtime_dir]) diff --git a/test/pleroma/application_test.exs b/test/pleroma/application_test.exs index afe628732..48f87818f 100644 --- a/test/pleroma/application_test.exs +++ b/test/pleroma/application_test.exs @@ -19,4 +19,15 @@ defmodule Pleroma.ApplicationTest do assert log =~ "Successfully added logger backend: {ExSyslogger, :ex_syslogger}" end end + + describe "config :pleroma, :logger, :backends: [:console]" do + setup do + clear_config([:logger, :backends], [:console]) + end + + test "emits a warning" do + assert capture_log(fn -> Pleroma.Application.configure_logger() end) =~ + ":console is no longer considered a backend and is enabled by default" + end + end end From 6744a86fe3a70834aac0c248f5b1c4d0680278fe Mon Sep 17 00:00:00 2001 From: Phantasm Date: Tue, 13 Jan 2026 00:36:15 +0100 Subject: [PATCH 046/183] Elixir 1.19: Disable logging with colors in test environment Can break some tests capture logs and then match on an expected text. Since color codes end up being in the captured log. --- config/test.exs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/config/test.exs b/config/test.exs index bac5e4df3..5b7f3b1ed 100644 --- a/config/test.exs +++ b/config/test.exs @@ -17,7 +17,11 @@ config :pleroma, Pleroma.Captcha, # Print only warnings and errors during test config :logger, :default_handler, level: :warning -config :logger, :default_formatter, format: "\n[$level] $message\n" +config :logger, :default_formatter, + format: "\n[$level] $message\n", + colors: [ + enabled: false + ] config :pleroma, :auth, oauth_consumer_strategies: [] From 17a563aeb41e4f2ec18788cf5a58a1a928044a6e Mon Sep 17 00:00:00 2001 From: Phantasm Date: Tue, 13 Jan 2026 00:39:48 +0100 Subject: [PATCH 047/183] Elixir 1.19: Inject @compile autoload opt to MRF policies Descriptions of MRF policies are gathered at compile-time and use Pleroma.Docs.Generator.list_behaviour_implementations func, which depends on modules implementing the queried behaviour to be loaded. Otherwise descriptions for MRFs can be missing from the AdminAPI route. Note: @behaviour cannot be injected in the __using__ macro. It causes a loop in the compiler which leads to memory exhaustion. Hence why @behaviour is still defined separately in the MRFs. --- .../web/activity_pub/mrf/activity_expiration_policy.ex | 2 ++ lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex | 6 +++--- lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex | 4 +++- .../web/activity_pub/mrf/anti_mention_spam_policy.ex | 4 +++- lib/pleroma/web/activity_pub/mrf/drop_policy.ex | 4 +++- lib/pleroma/web/activity_pub/mrf/emoji_policy.ex | 4 +++- lib/pleroma/web/activity_pub/mrf/ensure_re_prepended.ex | 6 ++++-- lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex | 3 +++ .../web/activity_pub/mrf/force_bot_unlisted_policy.ex | 7 +++++-- lib/pleroma/web/activity_pub/mrf/force_mention.ex | 3 ++- .../web/activity_pub/mrf/force_mentions_in_content.ex | 4 +++- lib/pleroma/web/activity_pub/mrf/hashtag_policy.ex | 4 +++- lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex | 7 ++++--- lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex | 2 ++ lib/pleroma/web/activity_pub/mrf/keyword_policy.ex | 6 ++++-- .../web/activity_pub/mrf/media_proxy_warming_policy.ex | 2 ++ lib/pleroma/web/activity_pub/mrf/mention_policy.ex | 3 ++- lib/pleroma/web/activity_pub/mrf/no_empty_policy.ex | 2 ++ lib/pleroma/web/activity_pub/mrf/no_op_policy.ex | 2 ++ .../web/activity_pub/mrf/no_placeholder_text_policy.ex | 2 ++ lib/pleroma/web/activity_pub/mrf/normalize_markup.ex | 4 +++- lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex | 5 ++++- lib/pleroma/web/activity_pub/mrf/object_age_policy.ex | 6 ++++-- lib/pleroma/web/activity_pub/mrf/policy.ex | 8 ++++++++ lib/pleroma/web/activity_pub/mrf/quiet_reply.ex | 4 +++- .../web/activity_pub/mrf/quote_to_link_tag_policy.ex | 2 ++ lib/pleroma/web/activity_pub/mrf/reject_non_public.ex | 3 ++- lib/pleroma/web/activity_pub/mrf/remote_report_policy.ex | 2 ++ lib/pleroma/web/activity_pub/mrf/simple_policy.ex | 2 ++ lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex | 6 ++++-- lib/pleroma/web/activity_pub/mrf/subchain_policy.ex | 4 +++- lib/pleroma/web/activity_pub/mrf/tag_policy.ex | 7 +++++-- .../web/activity_pub/mrf/user_allow_list_policy.ex | 6 ++++-- lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex | 3 ++- 34 files changed, 105 insertions(+), 34 deletions(-) diff --git a/lib/pleroma/web/activity_pub/mrf/activity_expiration_policy.ex b/lib/pleroma/web/activity_pub/mrf/activity_expiration_policy.ex index 88f6ca028..41abe17c0 100644 --- a/lib/pleroma/web/activity_pub/mrf/activity_expiration_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/activity_expiration_policy.ex @@ -6,6 +6,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicy do @moduledoc "Adds expiration to all local Create activities" @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy + @impl true def filter(activity) do activity = diff --git a/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex b/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex index 8ea61aec2..990ff31c9 100644 --- a/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex @@ -3,11 +3,11 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.AntiFollowbotPolicy do + @moduledoc "Prevent followbots from following with a bit of heuristic" + @behaviour Pleroma.Web.ActivityPub.MRF.Policy alias Pleroma.User - @moduledoc "Prevent followbots from following with a bit of heuristic" - - @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy # XXX: this should become User.normalize_by_ap_id() or similar, really. defp normalize_by_ap_id(%{"id" => id}), do: User.get_cached_by_ap_id(id) diff --git a/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex b/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex index 2be6d8df4..5c3b66e37 100644 --- a/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex @@ -3,9 +3,11 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.AntiLinkSpamPolicy do + @behaviour Pleroma.Web.ActivityPub.MRF.Policy + alias Pleroma.User - @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy require Logger diff --git a/lib/pleroma/web/activity_pub/mrf/anti_mention_spam_policy.ex b/lib/pleroma/web/activity_pub/mrf/anti_mention_spam_policy.ex index 1d76a307b..c6f5781b3 100644 --- a/lib/pleroma/web/activity_pub/mrf/anti_mention_spam_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/anti_mention_spam_policy.ex @@ -3,11 +3,13 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.AntiMentionSpamPolicy do + @behaviour Pleroma.Web.ActivityPub.MRF.Policy + alias Pleroma.Config alias Pleroma.User require Pleroma.Constants - @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy defp user_has_posted?(%User{} = u), do: u.note_count > 0 diff --git a/lib/pleroma/web/activity_pub/mrf/drop_policy.ex b/lib/pleroma/web/activity_pub/mrf/drop_policy.ex index cf07db7f3..4aa1968f7 100644 --- a/lib/pleroma/web/activity_pub/mrf/drop_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/drop_policy.ex @@ -3,10 +3,12 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.DropPolicy do - require Logger @moduledoc "Drop and log everything received" @behaviour Pleroma.Web.ActivityPub.MRF.Policy + require Logger + use Pleroma.Web.ActivityPub.MRF.Policy + @impl true def filter(activity) do Logger.debug("REJECTING #{inspect(activity)}") diff --git a/lib/pleroma/web/activity_pub/mrf/emoji_policy.ex b/lib/pleroma/web/activity_pub/mrf/emoji_policy.ex index 1de5280d9..cc5fd49bf 100644 --- a/lib/pleroma/web/activity_pub/mrf/emoji_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/emoji_policy.ex @@ -3,6 +3,8 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.EmojiPolicy do + @behaviour Pleroma.Web.ActivityPub.MRF.Policy + require Pleroma.Constants alias Pleroma.Object.Updater @@ -10,7 +12,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.EmojiPolicy do @moduledoc "Reject or force-unlisted emojis with certain URLs or names" - @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy defp config_remove_url do Pleroma.Config.get([:mrf_emoji, :remove_url], []) diff --git a/lib/pleroma/web/activity_pub/mrf/ensure_re_prepended.ex b/lib/pleroma/web/activity_pub/mrf/ensure_re_prepended.ex index f5983c8a7..6e43e746c 100644 --- a/lib/pleroma/web/activity_pub/mrf/ensure_re_prepended.ex +++ b/lib/pleroma/web/activity_pub/mrf/ensure_re_prepended.ex @@ -3,11 +3,13 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.EnsureRePrepended do - alias Pleroma.Object - @moduledoc "Ensure a re: is prepended on replies to a post with a Subject" @behaviour Pleroma.Web.ActivityPub.MRF.Policy + alias Pleroma.Object + + use Pleroma.Web.ActivityPub.MRF.Policy + @reply_prefix Regex.compile!("^re:[[:space:]]*", [:caseless]) def history_awareness, do: :auto diff --git a/lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex b/lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex index 480a03ef6..4612b2cc5 100644 --- a/lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex @@ -4,12 +4,15 @@ defmodule Pleroma.Web.ActivityPub.MRF.FollowBotPolicy do @behaviour Pleroma.Web.ActivityPub.MRF.Policy + alias Pleroma.Config alias Pleroma.User alias Pleroma.Web.CommonAPI require Logger + use Pleroma.Web.ActivityPub.MRF.Policy + @impl true def filter(activity) do with follower_nickname <- Config.get([:mrf_follow_bot, :follower_nickname]), diff --git a/lib/pleroma/web/activity_pub/mrf/force_bot_unlisted_policy.ex b/lib/pleroma/web/activity_pub/mrf/force_bot_unlisted_policy.ex index 3b3251dc3..58d02f433 100644 --- a/lib/pleroma/web/activity_pub/mrf/force_bot_unlisted_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/force_bot_unlisted_policy.ex @@ -3,9 +3,12 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.ForceBotUnlistedPolicy do - alias Pleroma.User - @behaviour Pleroma.Web.ActivityPub.MRF.Policy @moduledoc "Remove bot posts from federated timeline" + @behaviour Pleroma.Web.ActivityPub.MRF.Policy + + alias Pleroma.User + + use Pleroma.Web.ActivityPub.MRF.Policy require Pleroma.Constants diff --git a/lib/pleroma/web/activity_pub/mrf/force_mention.ex b/lib/pleroma/web/activity_pub/mrf/force_mention.ex index 4ea23540d..62aff36bc 100644 --- a/lib/pleroma/web/activity_pub/mrf/force_mention.ex +++ b/lib/pleroma/web/activity_pub/mrf/force_mention.ex @@ -3,13 +3,14 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.ForceMention do + @behaviour Pleroma.Web.ActivityPub.MRF.Policy require Pleroma.Constants alias Pleroma.Config alias Pleroma.Object alias Pleroma.User - @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy defp get_author(url) do with %Object{data: %{"actor" => actor}} <- Object.normalize(url, fetch: false), diff --git a/lib/pleroma/web/activity_pub/mrf/force_mentions_in_content.ex b/lib/pleroma/web/activity_pub/mrf/force_mentions_in_content.ex index caae365e5..e7c3f2d88 100644 --- a/lib/pleroma/web/activity_pub/mrf/force_mentions_in_content.ex +++ b/lib/pleroma/web/activity_pub/mrf/force_mentions_in_content.ex @@ -3,13 +3,15 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.ForceMentionsInContent do + @behaviour Pleroma.Web.ActivityPub.MRF.Policy + require Pleroma.Constants alias Pleroma.Formatter alias Pleroma.Object alias Pleroma.User - @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy @impl true def history_awareness, do: :auto diff --git a/lib/pleroma/web/activity_pub/mrf/hashtag_policy.ex b/lib/pleroma/web/activity_pub/mrf/hashtag_policy.ex index 72f2274ed..a3fae3bae 100644 --- a/lib/pleroma/web/activity_pub/mrf/hashtag_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/hashtag_policy.ex @@ -3,6 +3,8 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.HashtagPolicy do + @behaviour Pleroma.Web.ActivityPub.MRF.Policy + require Pleroma.Constants alias Pleroma.Config @@ -14,7 +16,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.HashtagPolicy do Note: This MRF Policy is always enabled, if you want to disable it you have to set empty lists. """ - @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy @impl true def history_awareness, do: :manual diff --git a/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex b/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex index 3a80d0a69..a84a7b0c8 100644 --- a/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex @@ -3,13 +3,14 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.HellthreadPolicy do + @moduledoc "Block activities with too much mentions (configurable)" + @behaviour Pleroma.Web.ActivityPub.MRF.Policy + alias Pleroma.User require Pleroma.Constants - @moduledoc "Block activities with too much mentions (configurable)" - - @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy defp delist_activity(activity, threshold) when threshold > 0 do follower_collection = User.get_cached_by_ap_id(activity["actor"]).follower_address diff --git a/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex b/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex index 51dcd3918..17879b4bd 100644 --- a/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex @@ -6,6 +6,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.InlineQuotePolicy do @moduledoc "Force a quote line into the message content." @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy + defp build_inline_quote(template, url) do quote_line = String.replace(template, "{url}", "#{url}") diff --git a/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex b/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex index 6ba6fd509..44b79596e 100644 --- a/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex @@ -3,13 +3,15 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do + @moduledoc "Reject or Word-Replace activities with a keyword or regex" + @behaviour Pleroma.Web.ActivityPub.MRF.Policy + require Pleroma.Constants alias Pleroma.Web.ActivityPub.MRF.Utils - @moduledoc "Reject or Word-Replace activities with a keyword or regex" - @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy defp string_matches?(string, pattern) when is_binary(pattern) do String.contains?(string, pattern) diff --git a/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex b/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex index 43c2c0449..07046a666 100644 --- a/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex @@ -6,6 +6,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do @moduledoc "Preloads any attachments in the MediaProxy cache by prefetching them" @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy + alias Pleroma.HTTP alias Pleroma.Web.MediaProxy diff --git a/lib/pleroma/web/activity_pub/mrf/mention_policy.ex b/lib/pleroma/web/activity_pub/mrf/mention_policy.ex index f7bff121f..8e108a0ae 100644 --- a/lib/pleroma/web/activity_pub/mrf/mention_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/mention_policy.ex @@ -4,9 +4,10 @@ defmodule Pleroma.Web.ActivityPub.MRF.MentionPolicy do @moduledoc "Block activities which mention a user" - @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy + @impl true def filter(%{"type" => "Create"} = activity) do reject_actors = Pleroma.Config.get([:mrf_mention, :actors], []) diff --git a/lib/pleroma/web/activity_pub/mrf/no_empty_policy.ex b/lib/pleroma/web/activity_pub/mrf/no_empty_policy.ex index 08dd39878..8283fd504 100644 --- a/lib/pleroma/web/activity_pub/mrf/no_empty_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/no_empty_policy.ex @@ -6,6 +6,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.NoEmptyPolicy do @moduledoc "Filter local activities which have no content" @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy + alias Pleroma.Web.Endpoint @impl true diff --git a/lib/pleroma/web/activity_pub/mrf/no_op_policy.ex b/lib/pleroma/web/activity_pub/mrf/no_op_policy.ex index 64a5872bc..3adbb598a 100644 --- a/lib/pleroma/web/activity_pub/mrf/no_op_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/no_op_policy.ex @@ -6,6 +6,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.NoOpPolicy do @moduledoc "Does nothing (lets the messages go through unmodified)" @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy + @impl true def filter(activity) do {:ok, activity} diff --git a/lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex b/lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex index c6f239a5e..2a10c7e97 100644 --- a/lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex @@ -6,6 +6,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.NoPlaceholderTextPolicy do @moduledoc "Ensure no content placeholder is present (such as the dot from mastodon)" @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy + @impl true def history_awareness, do: :auto diff --git a/lib/pleroma/web/activity_pub/mrf/normalize_markup.ex b/lib/pleroma/web/activity_pub/mrf/normalize_markup.ex index 91855ef84..de0f5b619 100644 --- a/lib/pleroma/web/activity_pub/mrf/normalize_markup.ex +++ b/lib/pleroma/web/activity_pub/mrf/normalize_markup.ex @@ -4,9 +4,11 @@ defmodule Pleroma.Web.ActivityPub.MRF.NormalizeMarkup do @moduledoc "Scrub configured hypertext markup" + @behaviour Pleroma.Web.ActivityPub.MRF.Policy + alias Pleroma.HTML - @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy @impl true def history_awareness, do: :auto diff --git a/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex b/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex index 52aaf05aa..7e3ec3de5 100644 --- a/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex @@ -38,6 +38,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.NsfwApiPolicy do - `unlist`: Unlist all detected NSFW content? Default: `false` - `reject`: Reject all detected NSFW content (takes precedence)? Default: `false` """ + @behaviour Pleroma.Web.ActivityPub.MRF.Policy + alias Pleroma.Config alias Pleroma.Constants alias Pleroma.HTTP @@ -46,7 +48,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.NsfwApiPolicy do require Logger require Pleroma.Constants - @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy + @policy :mrf_nsfw_api def build_request_url(url) do diff --git a/lib/pleroma/web/activity_pub/mrf/object_age_policy.ex b/lib/pleroma/web/activity_pub/mrf/object_age_policy.ex index 34905fc21..00f4cf84b 100644 --- a/lib/pleroma/web/activity_pub/mrf/object_age_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/object_age_policy.ex @@ -3,13 +3,15 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy do + @moduledoc "Filter activities depending on their age" + @behaviour Pleroma.Web.ActivityPub.MRF.Policy + alias Pleroma.Config alias Pleroma.User require Pleroma.Constants - @moduledoc "Filter activities depending on their age" - @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy defp check_date(%{"object" => %{"published" => published}} = activity) do with %DateTime{} = now <- DateTime.utc_now(), diff --git a/lib/pleroma/web/activity_pub/mrf/policy.ex b/lib/pleroma/web/activity_pub/mrf/policy.ex index 08bcac08a..b9a693c9e 100644 --- a/lib/pleroma/web/activity_pub/mrf/policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/policy.ex @@ -15,4 +15,12 @@ defmodule Pleroma.Web.ActivityPub.MRF.Policy do } @callback history_awareness() :: :auto | :manual @optional_callbacks config_description: 0, history_awareness: 0, id_filter: 1 + + defmacro __using__(_) do + quote do + # Otherwise Pleroma.Docs.Generator.list_behaviour_implementations fails + # since the Elixir compiler doesn't automatically load the module since v1.19. + @compile {:autoload, true} + end + end end diff --git a/lib/pleroma/web/activity_pub/mrf/quiet_reply.ex b/lib/pleroma/web/activity_pub/mrf/quiet_reply.ex index b3eb0b390..b47e9edb9 100644 --- a/lib/pleroma/web/activity_pub/mrf/quiet_reply.ex +++ b/lib/pleroma/web/activity_pub/mrf/quiet_reply.ex @@ -6,11 +6,13 @@ defmodule Pleroma.Web.ActivityPub.MRF.QuietReply do @moduledoc """ QuietReply alters the scope of activities from local users when replying by enforcing them to be "Unlisted" or "Quiet Public". This delivers the activity to all the expected recipients and instances, but it will not be published in the Federated / The Whole Known Network timelines. It will still be published to the Home timelines of the user's followers and visible to anyone who opens the thread. """ + @behaviour Pleroma.Web.ActivityPub.MRF.Policy + require Pleroma.Constants alias Pleroma.User - @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy @impl true def history_awareness, do: :auto diff --git a/lib/pleroma/web/activity_pub/mrf/quote_to_link_tag_policy.ex b/lib/pleroma/web/activity_pub/mrf/quote_to_link_tag_policy.ex index 2a17b6761..e1ad17585 100644 --- a/lib/pleroma/web/activity_pub/mrf/quote_to_link_tag_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/quote_to_link_tag_policy.ex @@ -6,6 +6,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.QuoteToLinkTagPolicy do @moduledoc "Force a Link tag for posts quoting another post. (may break outgoing federation of quote posts with older Pleroma versions)" @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy + alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes require Pleroma.Constants diff --git a/lib/pleroma/web/activity_pub/mrf/reject_non_public.ex b/lib/pleroma/web/activity_pub/mrf/reject_non_public.ex index 9d4a7a405..7d113dbf6 100644 --- a/lib/pleroma/web/activity_pub/mrf/reject_non_public.ex +++ b/lib/pleroma/web/activity_pub/mrf/reject_non_public.ex @@ -4,11 +4,12 @@ defmodule Pleroma.Web.ActivityPub.MRF.RejectNonPublic do @moduledoc "Rejects non-public (followers-only, direct) activities" + @behaviour Pleroma.Web.ActivityPub.MRF.Policy alias Pleroma.Config alias Pleroma.User - @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy require Pleroma.Constants diff --git a/lib/pleroma/web/activity_pub/mrf/remote_report_policy.ex b/lib/pleroma/web/activity_pub/mrf/remote_report_policy.ex index 8422832dd..f9f8d2298 100644 --- a/lib/pleroma/web/activity_pub/mrf/remote_report_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/remote_report_policy.ex @@ -2,6 +2,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.RemoteReportPolicy do @moduledoc "Drop remote reports if they don't contain enough information." @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy + alias Pleroma.Config @impl true diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex index a97e8db7b..7045fe90f 100644 --- a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex @@ -6,6 +6,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do @moduledoc "Filter activities depending on their origin instance" @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy + alias Pleroma.Config alias Pleroma.FollowingRelationship alias Pleroma.User diff --git a/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex b/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex index 54f0e6bc1..f1e99aead 100644 --- a/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex @@ -3,12 +3,14 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.StealEmojiPolicy do + @moduledoc "Detect new emojis by their shortcode and steals them" + @behaviour Pleroma.Web.ActivityPub.MRF.Policy + require Logger alias Pleroma.Config - @moduledoc "Detect new emojis by their shortcode and steals them" - @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy defp accept_host?(host), do: host in Config.get([:mrf_steal_emoji, :hosts], []) diff --git a/lib/pleroma/web/activity_pub/mrf/subchain_policy.ex b/lib/pleroma/web/activity_pub/mrf/subchain_policy.ex index 97acca7e8..1fe7b00e9 100644 --- a/lib/pleroma/web/activity_pub/mrf/subchain_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/subchain_policy.ex @@ -3,12 +3,14 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.SubchainPolicy do + @behaviour Pleroma.Web.ActivityPub.MRF.Policy + alias Pleroma.Config alias Pleroma.Web.ActivityPub.MRF require Logger - @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy defp lookup_subchain(actor) do with matches <- Config.get([:mrf_subchain, :match_actor]), diff --git a/lib/pleroma/web/activity_pub/mrf/tag_policy.ex b/lib/pleroma/web/activity_pub/mrf/tag_policy.ex index c236a5a99..df5594808 100644 --- a/lib/pleroma/web/activity_pub/mrf/tag_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/tag_policy.ex @@ -3,8 +3,6 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.TagPolicy do - alias Pleroma.User - @behaviour Pleroma.Web.ActivityPub.MRF.Policy @moduledoc """ Apply policies based on user tags @@ -18,6 +16,11 @@ defmodule Pleroma.Web.ActivityPub.MRF.TagPolicy do - `mrf_tag:disable-remote-subscription`: Reject non-local follow requests - `mrf_tag:disable-any-subscription`: Reject any follow requests """ + @behaviour Pleroma.Web.ActivityPub.MRF.Policy + + alias Pleroma.User + + use Pleroma.Web.ActivityPub.MRF.Policy require Pleroma.Constants diff --git a/lib/pleroma/web/activity_pub/mrf/user_allow_list_policy.ex b/lib/pleroma/web/activity_pub/mrf/user_allow_list_policy.ex index 10cc0e09d..a04ca90bd 100644 --- a/lib/pleroma/web/activity_pub/mrf/user_allow_list_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/user_allow_list_policy.ex @@ -3,11 +3,13 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.UserAllowListPolicy do - alias Pleroma.Config - @moduledoc "Accept-list of users from specified instances" @behaviour Pleroma.Web.ActivityPub.MRF.Policy + alias Pleroma.Config + + use Pleroma.Web.ActivityPub.MRF.Policy + defp filter_by_list(activity, []), do: {:ok, activity} defp filter_by_list(%{"actor" => actor} = activity, allow_list) do diff --git a/lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex b/lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex index 5671e4cf3..ecba3a2c6 100644 --- a/lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex @@ -4,9 +4,10 @@ defmodule Pleroma.Web.ActivityPub.MRF.VocabularyPolicy do @moduledoc "Filter activities which belong to certain activity vocabularies" - @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy + @impl true def filter(%{"type" => "Undo", "object" => object} = activity) do with {:ok, _} <- filter(object) do From 2a23eacee899c69f55d8a3cb6ed8291b56c7187b Mon Sep 17 00:00:00 2001 From: Phantasm Date: Tue, 13 Jan 2026 14:58:32 +0100 Subject: [PATCH 048/183] Elixir 1.19: Only match once on structs Second match is not needed and a simple Map update is recommended by the compiler --- lib/pleroma/marker.ex | 2 +- lib/pleroma/web/api_spec/render_error.ex | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/marker.ex b/lib/pleroma/marker.ex index 4f645240e..fab24d183 100644 --- a/lib/pleroma/marker.ex +++ b/lib/pleroma/marker.ex @@ -78,7 +78,7 @@ defmodule Pleroma.Marker do defp get_marker(user, timeline) do case Repo.find_resource(get_query(user, timeline)) do - {:ok, %__MODULE__{} = marker} -> %__MODULE__{marker | user: user} + {:ok, %__MODULE__{} = marker} -> %{marker | user: user} _ -> %__MODULE__{timeline: timeline, user_id: user.id} end end diff --git a/lib/pleroma/web/api_spec/render_error.ex b/lib/pleroma/web/api_spec/render_error.ex index acf510774..2ba76f250 100644 --- a/lib/pleroma/web/api_spec/render_error.ex +++ b/lib/pleroma/web/api_spec/render_error.ex @@ -18,10 +18,10 @@ defmodule Pleroma.Web.ApiSpec.RenderError do errors = Enum.map(errors, fn %OpenApiSpex.Cast.Error{name: nil, reason: :invalid_enum} = err -> - %OpenApiSpex.Cast.Error{err | name: err.value} + %{err | name: err.value} %OpenApiSpex.Cast.Error{name: nil} = err -> - %OpenApiSpex.Cast.Error{err | name: List.last(err.path)} + %{err | name: List.last(err.path)} err -> err From c455e0225581e3ebe43e9ef51a7ee393d8e87bdd Mon Sep 17 00:00:00 2001 From: Phantasm Date: Tue, 13 Jan 2026 15:21:41 +0100 Subject: [PATCH 049/183] Elixir 1.19: Add non-test Elixir files in test/ to mix.exs filter This is a new filter introduced in 1.19. --- mix.exs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/mix.exs b/mix.exs index dc9fa31f5..60064e846 100644 --- a/mix.exs +++ b/mix.exs @@ -15,6 +15,13 @@ defmodule Pleroma.Mixfile do aliases: aliases(), deps: deps(), test_coverage: [tool: :covertool, summary: true], + test_ignore_filters: [ + "test/credo/check/consistency/file_location.ex", + "test/fixtures/config/temp.exported_from_db.secret.exs", + "test/fixtures/config/temp.secret.exs", + "test/fixtures/modules/good_mrf.ex", + "test/fixtures/modules/runtime_module.ex" + ], # Docs name: "Pleroma", homepage_url: "https://pleroma.social/", From a4c929c527353f99cd11aeb9c08a44f212632f12 Mon Sep 17 00:00:00 2001 From: Phantasm Date: Tue, 13 Jan 2026 20:44:26 +0100 Subject: [PATCH 050/183] Lint --- lib/pleroma/config/deprecation_warnings.ex | 4 ++-- lib/pleroma/web/activity_pub/mrf/keyword_policy.ex | 1 - test/pleroma/application_test.exs | 6 ++++-- test/pleroma/config/deprecation_warnings_test.exs | 10 ++++++---- 4 files changed, 12 insertions(+), 9 deletions(-) diff --git a/lib/pleroma/config/deprecation_warnings.ex b/lib/pleroma/config/deprecation_warnings.ex index 8dabd728f..8f56a3771 100644 --- a/lib/pleroma/config/deprecation_warnings.ex +++ b/lib/pleroma/config/deprecation_warnings.ex @@ -418,11 +418,11 @@ defmodule Pleroma.Config.DeprecationWarnings do end @spec check_deprecated_logger_config() :: :ok | :error - def check_deprecated_logger_config() do + def check_deprecated_logger_config do if Application.get_env(:logger, :backends) do Logger.warning( "'config :logger, backends: [...]' is deprecated syntax due to changes in Elixir. " <> - "Use 'config :pleroma, :logger, backends: [...]' instead." + "Use 'config :pleroma, :logger, backends: [...]' instead." ) :error diff --git a/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex b/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex index 44b79596e..33f32e781 100644 --- a/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex @@ -10,7 +10,6 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do alias Pleroma.Web.ActivityPub.MRF.Utils - use Pleroma.Web.ActivityPub.MRF.Policy defp string_matches?(string, pattern) when is_binary(pattern) do diff --git a/test/pleroma/application_test.exs b/test/pleroma/application_test.exs index 48f87818f..b4a29fc9b 100644 --- a/test/pleroma/application_test.exs +++ b/test/pleroma/application_test.exs @@ -15,7 +15,9 @@ defmodule Pleroma.ApplicationTest do test "is handled" do log = capture_log(fn -> Pleroma.Application.configure_logger() end) - assert log =~ "Configuration {:ex_syslogger, :ex_syslogger} is incorrect. Use {ExSyslogger, :ex_syslogger} instead!" + assert log =~ + "Configuration {:ex_syslogger, :ex_syslogger} is incorrect. Use {ExSyslogger, :ex_syslogger} instead!" + assert log =~ "Successfully added logger backend: {ExSyslogger, :ex_syslogger}" end end @@ -27,7 +29,7 @@ defmodule Pleroma.ApplicationTest do test "emits a warning" do assert capture_log(fn -> Pleroma.Application.configure_logger() end) =~ - ":console is no longer considered a backend and is enabled by default" + ":console is no longer considered a backend and is enabled by default" end end end diff --git a/test/pleroma/config/deprecation_warnings_test.exs b/test/pleroma/config/deprecation_warnings_test.exs index 1491f4ea3..5383e2bce 100644 --- a/test/pleroma/config/deprecation_warnings_test.exs +++ b/test/pleroma/config/deprecation_warnings_test.exs @@ -394,13 +394,15 @@ defmodule Pleroma.Config.DeprecationWarningsTest do Application.put_env(:logger, :backends, [:console, {ExSyslogger, :ex_syslogger}]) on_exit(fn -> - Application.delete_env(:logger, :backends) end) + Application.delete_env(:logger, :backends) + end) end test "warns on deprecated syntax" do - assert capture_log(fn -> Pleroma.Config.DeprecationWarnings.check_deprecated_logger_config() end) =~ - "'config :logger, backends: [...]' is deprecated syntax due to changes in Elixir. Use 'config :pleroma, :logger, backends: [...]' instead." + assert capture_log(fn -> + Pleroma.Config.DeprecationWarnings.check_deprecated_logger_config() + end) =~ + "'config :logger, backends: [...]' is deprecated syntax due to changes in Elixir. Use 'config :pleroma, :logger, backends: [...]' instead." end end - end From a4137243b62795b6299248ccf11f80ccda059d56 Mon Sep 17 00:00:00 2001 From: Phantasm Date: Tue, 13 Jan 2026 21:07:06 +0100 Subject: [PATCH 051/183] Elixir 1.19: Reword Logger backends deprecation warning --- lib/pleroma/config/deprecation_warnings.ex | 15 +++++++++++---- test/pleroma/config/deprecation_warnings_test.exs | 12 +++++++++++- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/config/deprecation_warnings.ex b/lib/pleroma/config/deprecation_warnings.ex index 8f56a3771..8b8628862 100644 --- a/lib/pleroma/config/deprecation_warnings.ex +++ b/lib/pleroma/config/deprecation_warnings.ex @@ -420,10 +420,17 @@ defmodule Pleroma.Config.DeprecationWarnings do @spec check_deprecated_logger_config() :: :ok | :error def check_deprecated_logger_config do if Application.get_env(:logger, :backends) do - Logger.warning( - "'config :logger, backends: [...]' is deprecated syntax due to changes in Elixir. " <> - "Use 'config :pleroma, :logger, backends: [...]' instead." - ) + Logger.warning(""" + !!!DEPRECATION WARNING!!! + Your configuration is using deprecated syntax for configuring backends of Elixir's logger. + `config :logger, backends: [...]` is deprecated syntax due to changes in Elixir. + Please update your configuration at your earliest convenience to use: + `config :pleroma, :logger, backends: [...]` + Pleroma's default configuration will be updated in the next release which WILL override your settings. + + Note: :console is no longer considered a backend and is used by default, you can disable it using: + `config :logger, :default_handler: false` + """) :error else diff --git a/test/pleroma/config/deprecation_warnings_test.exs b/test/pleroma/config/deprecation_warnings_test.exs index 5383e2bce..a9a71be36 100644 --- a/test/pleroma/config/deprecation_warnings_test.exs +++ b/test/pleroma/config/deprecation_warnings_test.exs @@ -402,7 +402,17 @@ defmodule Pleroma.Config.DeprecationWarningsTest do assert capture_log(fn -> Pleroma.Config.DeprecationWarnings.check_deprecated_logger_config() end) =~ - "'config :logger, backends: [...]' is deprecated syntax due to changes in Elixir. Use 'config :pleroma, :logger, backends: [...]' instead." + """ + !!!DEPRECATION WARNING!!! + Your configuration is using deprecated syntax for configuring backends of Elixir's logger. + `config :logger, backends: [...]` is deprecated syntax due to changes in Elixir. + Please update your configuration at your earliest convenience to use: + `config :pleroma, :logger, backends: [...]` + Pleroma's default configuration will be updated in the next release which WILL override your settings. + + Note: :console is no longer considered a backend and is used by default, you can disable it using: + `config :logger, :default_handler: false` + """ end end end From 6ffd25682d4a0e6cebe9b0c8789c76bb9e8a312d Mon Sep 17 00:00:00 2001 From: Phantasm Date: Tue, 13 Jan 2026 21:18:08 +0100 Subject: [PATCH 052/183] Elixir 1.19: Add deprecation warning for :logger, :console configs --- lib/pleroma/config/deprecation_warnings.ex | 30 +++++++++++--- .../config/deprecation_warnings_test.exs | 41 +++++++++++++------ 2 files changed, 54 insertions(+), 17 deletions(-) diff --git a/lib/pleroma/config/deprecation_warnings.ex b/lib/pleroma/config/deprecation_warnings.ex index 8b8628862..f38fb7f2f 100644 --- a/lib/pleroma/config/deprecation_warnings.ex +++ b/lib/pleroma/config/deprecation_warnings.ex @@ -419,22 +419,42 @@ defmodule Pleroma.Config.DeprecationWarnings do @spec check_deprecated_logger_config() :: :ok | :error def check_deprecated_logger_config do - if Application.get_env(:logger, :backends) do + backend = if Application.get_env(:logger, :backends) do Logger.warning(""" !!!DEPRECATION WARNING!!! Your configuration is using deprecated syntax for configuring backends of Elixir's logger. `config :logger, backends: [...]` is deprecated syntax due to changes in Elixir. Please update your configuration at your earliest convenience to use: - `config :pleroma, :logger, backends: [...]` + `config :logger, backends: [...]` Pleroma's default configuration will be updated in the next release which WILL override your settings. - Note: :console is no longer considered a backend and is used by default, you can disable it using: + Note: `:console` is no longer considered a backend and is used by default, you can disable it using: `config :logger, :default_handler: false` """) - :error + true else - :ok + false end + + console = if Application.get_env(:logger, :console) do + Logger.warning(""" + !!!DEPRECATION WARNING!!! + Your configuration is using deprecated syntax for configuring logging to console. + `config :logger, :console` is deprecated syntax due to changes in Elixir. + Please update your configuration at your earliest convenience to use + `config :logger, default_handler` and `config :logger, :default_formatter`. + Pleroma's default configuration will be updated in the next release which WILL override your settings. + + Note: `:default_handler` is used only for the `level` setting. All other configurations go under + `:default_formatter`. For more info visit: https://hexdocs.pm/logger/Logger.html#module-backends-and-backwards-compatibility + """) + + true + else + false + end + + if backend or console, do: :error, else: :ok end end diff --git a/test/pleroma/config/deprecation_warnings_test.exs b/test/pleroma/config/deprecation_warnings_test.exs index a9a71be36..f5b9d76d3 100644 --- a/test/pleroma/config/deprecation_warnings_test.exs +++ b/test/pleroma/config/deprecation_warnings_test.exs @@ -392,27 +392,44 @@ defmodule Pleroma.Config.DeprecationWarningsTest do describe "check_deprecated_logger_config" do setup do Application.put_env(:logger, :backends, [:console, {ExSyslogger, :ex_syslogger}]) + Application.put_env(:logger, :console, level: :warning) on_exit(fn -> Application.delete_env(:logger, :backends) + Application.delete_env(:logger, :console) end) end test "warns on deprecated syntax" do - assert capture_log(fn -> + log = capture_log(fn -> Pleroma.Config.DeprecationWarnings.check_deprecated_logger_config() - end) =~ - """ - !!!DEPRECATION WARNING!!! - Your configuration is using deprecated syntax for configuring backends of Elixir's logger. - `config :logger, backends: [...]` is deprecated syntax due to changes in Elixir. - Please update your configuration at your earliest convenience to use: - `config :pleroma, :logger, backends: [...]` - Pleroma's default configuration will be updated in the next release which WILL override your settings. + end) - Note: :console is no longer considered a backend and is used by default, you can disable it using: - `config :logger, :default_handler: false` - """ + assert log =~ + """ + !!!DEPRECATION WARNING!!! + Your configuration is using deprecated syntax for configuring backends of Elixir's logger. + `config :logger, backends: [...]` is deprecated syntax due to changes in Elixir. + Please update your configuration at your earliest convenience to use: + `config :logger, backends: [...]` + Pleroma's default configuration will be updated in the next release which WILL override your settings. + + Note: `:console` is no longer considered a backend and is used by default, you can disable it using: + `config :logger, :default_handler: false` + """ + + assert log =~ + """ + !!!DEPRECATION WARNING!!! + Your configuration is using deprecated syntax for configuring logging to console. + `config :logger, :console` is deprecated syntax due to changes in Elixir. + Please update your configuration at your earliest convenience to use + `config :logger, default_handler` and `config :logger, :default_formatter`. + Pleroma's default configuration will be updated in the next release which WILL override your settings. + + Note: `:default_handler` is used only for the `level` setting. All other configurations go under + `:default_formatter`. For more info visit: https://hexdocs.pm/logger/Logger.html#module-backends-and-backwards-compatibility + """ end end end From a5bd4e0d9230fe9aaad4722681ed12aec0cb5301 Mon Sep 17 00:00:00 2001 From: Phantasm Date: Tue, 13 Jan 2026 21:31:07 +0100 Subject: [PATCH 053/183] Partially revert "Elixir 1.19: Refactor Logger configuration to what Elixir 1.15+ expects" This partially reverts commit fc35204cc1d718ba7815cc5e1b08746202b4f7dc. --- config/config.exs | 9 ++++----- config/dev.exs | 2 +- config/prod.exs | 3 ++- config/test.exs | 10 +++------- docs/configuration/cheatsheet.md | 2 +- 5 files changed, 11 insertions(+), 15 deletions(-) diff --git a/config/config.exs b/config/config.exs index 08311146e..5bf2c5c2e 100644 --- a/config/config.exs +++ b/config/config.exs @@ -132,12 +132,11 @@ config :pleroma, Pleroma.Web.Endpoint, "SameSite=Lax" ] -# config :logger, :console has been changed in 1.15+ and split into -# :default_handler and :default_formatter. It started emitting warnings in 1.19. -# See https://hexdocs.pm/logger/1.19.0/Logger.html#module-backends-and-backwards-compatibility -config :logger, :default_handler, level: :debug +# Configures Elixir's Logger +config :logger, backends: [:console] -config :logger, :default_formatter, +config :logger, :console, + level: :debug, format: "\n$time $metadata[$level] $message\n", metadata: [:actor, :path, :type, :user] diff --git a/config/dev.exs b/config/dev.exs index e86bdb7fd..14cf4a6dc 100644 --- a/config/dev.exs +++ b/config/dev.exs @@ -36,7 +36,7 @@ config :pleroma, Pleroma.Emails.Mailer, adapter: Swoosh.Adapters.Local # different ports. # Do not include timestamps in development logs -config :logger, :default_formatter, format: "$metadata[$level] $message\n" +config :logger, Logger.Backends.Console, format: "$metadata[$level] $message\n" # Set a higher stacktrace during development. Avoid configuring such # in production as building large stacktraces may be expensive. diff --git a/config/prod.exs b/config/prod.exs index 1dfbbe937..2d252bf02 100644 --- a/config/prod.exs +++ b/config/prod.exs @@ -20,7 +20,8 @@ config :pleroma, Pleroma.Web.Endpoint, config :phoenix, serve_endpoints: true # Do not print debug messages in production -config :logger, :default_handler, level: :info +config :logger, Logger.Backends.Console, level: :info +config :logger, :console, level: :info config :logger, :ex_syslogger, level: :info # ## SSL Support diff --git a/config/test.exs b/config/test.exs index 5b7f3b1ed..9652b7d4b 100644 --- a/config/test.exs +++ b/config/test.exs @@ -15,13 +15,9 @@ config :pleroma, Pleroma.Captcha, method: Pleroma.Captcha.Mock # Print only warnings and errors during test -config :logger, :default_handler, level: :warning - -config :logger, :default_formatter, - format: "\n[$level] $message\n", - colors: [ - enabled: false - ] +config :logger, :console, + level: :warning, + format: "\n[$level] $message\n" config :pleroma, :auth, oauth_consumer_strategies: [] diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index 01d81f6ca..b0a7087a7 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -886,7 +886,7 @@ See: [logger’s documentation](https://hexdocs.pm/logger/Logger.html) and [ex_s An example of logging info to local syslog, but debug to console: ```elixir config :pleroma, :logger, - backends: [ {ExSyslogger, :ex_syslogger}] + backends: [{ExSyslogger, :ex_syslogger}] config :logger, :ex_syslogger, level: :info, From aed0eb33331568b38c14b83d753c671b8336cb38 Mon Sep 17 00:00:00 2001 From: Phantasm Date: Tue, 13 Jan 2026 23:28:57 +0100 Subject: [PATCH 054/183] Elixir 1.19: Warn on old config of :console Logger and conflicts --- lib/pleroma/config/deprecation_warnings.ex | 122 +++++++++++++----- .../config/deprecation_warnings_test.exs | 47 +++---- 2 files changed, 115 insertions(+), 54 deletions(-) diff --git a/lib/pleroma/config/deprecation_warnings.ex b/lib/pleroma/config/deprecation_warnings.ex index f38fb7f2f..ee3965b64 100644 --- a/lib/pleroma/config/deprecation_warnings.ex +++ b/lib/pleroma/config/deprecation_warnings.ex @@ -419,42 +419,102 @@ defmodule Pleroma.Config.DeprecationWarnings do @spec check_deprecated_logger_config() :: :ok | :error def check_deprecated_logger_config do - backend = if Application.get_env(:logger, :backends) do - Logger.warning(""" - !!!DEPRECATION WARNING!!! - Your configuration is using deprecated syntax for configuring backends of Elixir's logger. - `config :logger, backends: [...]` is deprecated syntax due to changes in Elixir. - Please update your configuration at your earliest convenience to use: - `config :logger, backends: [...]` - Pleroma's default configuration will be updated in the next release which WILL override your settings. + mix_env = Pleroma.Config.get(:env) + config_file_name = "config/#{mix_env}.secret.exs" + config_file = File.read(config_file_name) - Note: `:console` is no longer considered a backend and is used by default, you can disable it using: - `config :logger, :default_handler: false` - """) + config_file = + case config_file do + {:ok, contents} -> + String.split(contents, "\n", trim: true) + |> Enum.reject(fn string -> String.starts_with?(string, "#") end) + |> Enum.join("\n") - true - else - false - end + _ -> + false + end - console = if Application.get_env(:logger, :console) do - Logger.warning(""" - !!!DEPRECATION WARNING!!! - Your configuration is using deprecated syntax for configuring logging to console. - `config :logger, :console` is deprecated syntax due to changes in Elixir. - Please update your configuration at your earliest convenience to use - `config :logger, default_handler` and `config :logger, :default_formatter`. - Pleroma's default configuration will be updated in the next release which WILL override your settings. + declared_deprecated_backend = + if is_binary(config_file) do + String.contains?(config_file, "config :logger, backends") or + String.contains?(config_file, "config :logger,\n backends") + else + nil + end - Note: `:default_handler` is used only for the `level` setting. All other configurations go under - `:default_formatter`. For more info visit: https://hexdocs.pm/logger/Logger.html#module-backends-and-backwards-compatibility - """) + declared_deprecated_console = + if is_binary(config_file) do + String.contains?(config_file, "config :logger, :console") or + String.contains?(config_file, "config :logger,\n console") + else + nil + end - true - else - false - end + # We can't use Application.get_env/3, because by default default_formatter is populated + declared_new_console = + if is_binary(config_file) do + String.contains?(config_file, "config :logger, :default_handler") or + String.contains?(config_file, "config :logger,\n :default_handler") or + String.contains?(config_file, "config :logger, :default_formatter") or + String.contains?(config_file, "config :logger,\n :default_formatter") + else + nil + end - if backend or console, do: :error, else: :ok + use_new_backend_config = Pleroma.Config.get([:logger, :backends]) + + backend = + if declared_deprecated_backend do + Logger.warning(""" + !!!DEPRECATION WARNING!!! + Your configuration is using deprecated syntax for configuring backends of Elixir's logger. + `config :logger, backends: [...]` is deprecated syntax due to changes in Elixir. + Please update your configuration at your earliest convenience to use: + `config :logger, backends: [...]` + Pleroma's default configuration will be updated in the next release which WILL override your settings. + + Note: `:console` is no longer considered a backend and is used by default, you can disable it using: + `config :logger, :default_handler: false` + """) + + true + else + false + end + + console = + if declared_deprecated_console do + Logger.warning(""" + !!!DEPRECATION WARNING!!! + Your configuration is using deprecated syntax for configuring logging to console. + `config :logger, :console` is deprecated syntax due to changes in Elixir. + Please update your configuration at your earliest convenience to use + `config :logger, default_handler` and `config :logger, :default_formatter`. + Pleroma's default configuration will be updated in the next release which WILL override your settings. + + Note: `:default_handler` is used only for the `level` setting. All other configurations go under + `:default_formatter`. For more info visit: https://hexdocs.pm/logger/Logger.html#module-backends-and-backwards-compatibility + """) + + true + else + false + end + + conflict = + if (!is_nil(use_new_backend_config) and declared_deprecated_backend) or + (declared_new_console and declared_deprecated_console) do + Logger.warning(""" + !!!CONFIGURATION CONFLICT!!! + Pleroma has detected that both the deprecated way of configuring Logger and the new way are defined in your configuration. + Delete the deprecated configuration syntax. + """) + + true + else + false + end + + if backend or console or conflict, do: :error, else: :ok end end diff --git a/test/pleroma/config/deprecation_warnings_test.exs b/test/pleroma/config/deprecation_warnings_test.exs index f5b9d76d3..cf2c6b4d2 100644 --- a/test/pleroma/config/deprecation_warnings_test.exs +++ b/test/pleroma/config/deprecation_warnings_test.exs @@ -401,35 +401,36 @@ defmodule Pleroma.Config.DeprecationWarningsTest do end test "warns on deprecated syntax" do - log = capture_log(fn -> - Pleroma.Config.DeprecationWarnings.check_deprecated_logger_config() - end) + log = + capture_log(fn -> + Pleroma.Config.DeprecationWarnings.check_deprecated_logger_config() + end) assert log =~ - """ - !!!DEPRECATION WARNING!!! - Your configuration is using deprecated syntax for configuring backends of Elixir's logger. - `config :logger, backends: [...]` is deprecated syntax due to changes in Elixir. - Please update your configuration at your earliest convenience to use: - `config :logger, backends: [...]` - Pleroma's default configuration will be updated in the next release which WILL override your settings. + """ + !!!DEPRECATION WARNING!!! + Your configuration is using deprecated syntax for configuring backends of Elixir's logger. + `config :logger, backends: [...]` is deprecated syntax due to changes in Elixir. + Please update your configuration at your earliest convenience to use: + `config :logger, backends: [...]` + Pleroma's default configuration will be updated in the next release which WILL override your settings. - Note: `:console` is no longer considered a backend and is used by default, you can disable it using: - `config :logger, :default_handler: false` - """ + Note: `:console` is no longer considered a backend and is used by default, you can disable it using: + `config :logger, :default_handler: false` + """ assert log =~ - """ - !!!DEPRECATION WARNING!!! - Your configuration is using deprecated syntax for configuring logging to console. - `config :logger, :console` is deprecated syntax due to changes in Elixir. - Please update your configuration at your earliest convenience to use - `config :logger, default_handler` and `config :logger, :default_formatter`. - Pleroma's default configuration will be updated in the next release which WILL override your settings. + """ + !!!DEPRECATION WARNING!!! + Your configuration is using deprecated syntax for configuring logging to console. + `config :logger, :console` is deprecated syntax due to changes in Elixir. + Please update your configuration at your earliest convenience to use + `config :logger, default_handler` and `config :logger, :default_formatter`. + Pleroma's default configuration will be updated in the next release which WILL override your settings. - Note: `:default_handler` is used only for the `level` setting. All other configurations go under - `:default_formatter`. For more info visit: https://hexdocs.pm/logger/Logger.html#module-backends-and-backwards-compatibility - """ + Note: `:default_handler` is used only for the `level` setting. All other configurations go under + `:default_formatter`. For more info visit: https://hexdocs.pm/logger/Logger.html#module-backends-and-backwards-compatibility + """ end end end From bd0c5aa748d226d109aeec7462ac9e6ba8d12f75 Mon Sep 17 00:00:00 2001 From: Phantasm Date: Wed, 14 Jan 2026 11:09:57 +0100 Subject: [PATCH 055/183] Elixir 1.19: Move default configs to the new Logger configuration, again Trying to do a two-release migration proved to be very hacky and ugly. Instead this is preparation for moving defaults first and then merging the old configuration left in admin's config file at runtime while still emitting a warning. This hopefully will be a lot nicer and less complicated. --- config/config.exs | 5 ++--- config/dev.exs | 2 +- config/prod.exs | 3 +-- config/test.exs | 5 ++--- 4 files changed, 6 insertions(+), 9 deletions(-) diff --git a/config/config.exs b/config/config.exs index 5bf2c5c2e..069c0c7c0 100644 --- a/config/config.exs +++ b/config/config.exs @@ -133,10 +133,9 @@ config :pleroma, Pleroma.Web.Endpoint, ] # Configures Elixir's Logger -config :logger, backends: [:console] +config :logger, :default_handler, level: :debug -config :logger, :console, - level: :debug, +config :logger, :default_formatter, format: "\n$time $metadata[$level] $message\n", metadata: [:actor, :path, :type, :user] diff --git a/config/dev.exs b/config/dev.exs index 14cf4a6dc..e86bdb7fd 100644 --- a/config/dev.exs +++ b/config/dev.exs @@ -36,7 +36,7 @@ config :pleroma, Pleroma.Emails.Mailer, adapter: Swoosh.Adapters.Local # different ports. # Do not include timestamps in development logs -config :logger, Logger.Backends.Console, format: "$metadata[$level] $message\n" +config :logger, :default_formatter, format: "$metadata[$level] $message\n" # Set a higher stacktrace during development. Avoid configuring such # in production as building large stacktraces may be expensive. diff --git a/config/prod.exs b/config/prod.exs index 2d252bf02..1dfbbe937 100644 --- a/config/prod.exs +++ b/config/prod.exs @@ -20,8 +20,7 @@ config :pleroma, Pleroma.Web.Endpoint, config :phoenix, serve_endpoints: true # Do not print debug messages in production -config :logger, Logger.Backends.Console, level: :info -config :logger, :console, level: :info +config :logger, :default_handler, level: :info config :logger, :ex_syslogger, level: :info # ## SSL Support diff --git a/config/test.exs b/config/test.exs index 9652b7d4b..083759327 100644 --- a/config/test.exs +++ b/config/test.exs @@ -15,9 +15,8 @@ config :pleroma, Pleroma.Captcha, method: Pleroma.Captcha.Mock # Print only warnings and errors during test -config :logger, :console, - level: :warning, - format: "\n[$level] $message\n" +config :logger, :default_handler, level: :warning +config :logger, :default_formatter, format: "\n[$level] $message\n" config :pleroma, :auth, oauth_consumer_strategies: [] From 9450fba83d10fca74e013708bd12406520ef9788 Mon Sep 17 00:00:00 2001 From: Phantasm Date: Wed, 14 Jan 2026 13:45:20 +0100 Subject: [PATCH 056/183] Elixir 1.19: Disable colors for logging in test env Color ANSI codes end up in the capture_log output and break existing tests. --- config/test.exs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/config/test.exs b/config/test.exs index 083759327..ed5283a26 100644 --- a/config/test.exs +++ b/config/test.exs @@ -16,7 +16,10 @@ config :pleroma, Pleroma.Captcha, # Print only warnings and errors during test config :logger, :default_handler, level: :warning -config :logger, :default_formatter, format: "\n[$level] $message\n" + +config :logger, :default_formatter, + format: "\n[$level] $message\n", + colors: [enabled: false] config :pleroma, :auth, oauth_consumer_strategies: [] From 0f262bb2f3238bf8d999d22e3a4a45bcbc7fc301 Mon Sep 17 00:00:00 2001 From: Phantasm Date: Wed, 14 Jan 2026 13:49:31 +0100 Subject: [PATCH 057/183] Elixir 1.19: Replace ugly old logger config checks with config merging Side note: Going up in log levels doesn't work properly. - debug->info works - info->debug does not work --- lib/pleroma/application.ex | 3 + lib/pleroma/config/deprecation_warnings.ex | 88 ++++++------------- .../config/deprecation_warnings_test.exs | 20 ++++- 3 files changed, 51 insertions(+), 60 deletions(-) diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 0e8723251..78073b430 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -132,6 +132,9 @@ defmodule Pleroma.Application do {:ok, _} -> Logger.debug("Successfully added logger backend: #{inspect(backend)}") + {:error, :already_present} -> + :ok + {:error, reason} -> Logger.error("Failed to add logger backend #{inspect(backend)}: #{inspect(reason)}") end diff --git a/lib/pleroma/config/deprecation_warnings.ex b/lib/pleroma/config/deprecation_warnings.ex index ee3965b64..4b6281e01 100644 --- a/lib/pleroma/config/deprecation_warnings.ex +++ b/lib/pleroma/config/deprecation_warnings.ex @@ -20,6 +20,9 @@ defmodule Pleroma.Config.DeprecationWarnings do "\n* `config :pleroma, :instance, mrf_transparency_exclusions` is now `config :pleroma, :mrf, transparency_exclusions`"} ] + @logger_config_knobs [:level, :translator_inspect_opts] + @logger_formatter_config_knows [:colors, :format, :metadata, :truncate, :utc_log] + def check_exiftool_filter do filters = Config.get([Pleroma.Upload]) |> Keyword.get(:filters, []) @@ -417,54 +420,33 @@ defmodule Pleroma.Config.DeprecationWarnings do end end + defp merge_deprecated_logger_config(config) do + handler_config = Enum.filter(config, fn {k, _} -> k in @logger_config_knobs end) + formatter_config = Enum.filter(config, fn {k, _} -> k in @logger_formatter_config_knows end) + formatter = Logger.default_formatter(formatter_config) + + Logger.debug(""" + Reconfiguring console Logger with deprecated configuration syntax. + Handler configuration: + #{inspect(handler_config)} + + Formatter: + #{inspect(formatter)} + """) + + Logger.configure(handler_config) + :logger.update_handler_config(:default, :formatter, formatter) + end + @spec check_deprecated_logger_config() :: :ok | :error def check_deprecated_logger_config do - mix_env = Pleroma.Config.get(:env) - config_file_name = "config/#{mix_env}.secret.exs" - config_file = File.read(config_file_name) - - config_file = - case config_file do - {:ok, contents} -> - String.split(contents, "\n", trim: true) - |> Enum.reject(fn string -> String.starts_with?(string, "#") end) - |> Enum.join("\n") - - _ -> - false - end - - declared_deprecated_backend = - if is_binary(config_file) do - String.contains?(config_file, "config :logger, backends") or - String.contains?(config_file, "config :logger,\n backends") - else - nil - end - - declared_deprecated_console = - if is_binary(config_file) do - String.contains?(config_file, "config :logger, :console") or - String.contains?(config_file, "config :logger,\n console") - else - nil - end - - # We can't use Application.get_env/3, because by default default_formatter is populated - declared_new_console = - if is_binary(config_file) do - String.contains?(config_file, "config :logger, :default_handler") or - String.contains?(config_file, "config :logger,\n :default_handler") or - String.contains?(config_file, "config :logger, :default_formatter") or - String.contains?(config_file, "config :logger,\n :default_formatter") - else - nil - end - - use_new_backend_config = Pleroma.Config.get([:logger, :backends]) + backends_config = Application.get_env(:logger, :backends) + console_config = Application.get_env(:logger, :console) + # Note: No need to merge the old backends config since it still works. + # And new configuration will just add new Logger backends. backend = - if declared_deprecated_backend do + if backends_config do Logger.warning(""" !!!DEPRECATION WARNING!!! Your configuration is using deprecated syntax for configuring backends of Elixir's logger. @@ -483,7 +465,7 @@ defmodule Pleroma.Config.DeprecationWarnings do end console = - if declared_deprecated_console do + if console_config do Logger.warning(""" !!!DEPRECATION WARNING!!! Your configuration is using deprecated syntax for configuring logging to console. @@ -496,25 +478,13 @@ defmodule Pleroma.Config.DeprecationWarnings do `:default_formatter`. For more info visit: https://hexdocs.pm/logger/Logger.html#module-backends-and-backwards-compatibility """) - true - else - false - end - - conflict = - if (!is_nil(use_new_backend_config) and declared_deprecated_backend) or - (declared_new_console and declared_deprecated_console) do - Logger.warning(""" - !!!CONFIGURATION CONFLICT!!! - Pleroma has detected that both the deprecated way of configuring Logger and the new way are defined in your configuration. - Delete the deprecated configuration syntax. - """) + merge_deprecated_logger_config(console_config) true else false end - if backend or console or conflict, do: :error, else: :ok + if backend or console, do: :error, else: :ok end end diff --git a/test/pleroma/config/deprecation_warnings_test.exs b/test/pleroma/config/deprecation_warnings_test.exs index cf2c6b4d2..787f9d2fd 100644 --- a/test/pleroma/config/deprecation_warnings_test.exs +++ b/test/pleroma/config/deprecation_warnings_test.exs @@ -392,11 +392,12 @@ defmodule Pleroma.Config.DeprecationWarningsTest do describe "check_deprecated_logger_config" do setup do Application.put_env(:logger, :backends, [:console, {ExSyslogger, :ex_syslogger}]) - Application.put_env(:logger, :console, level: :warning) + Application.put_env(:logger, :console, level: :debug) on_exit(fn -> Application.delete_env(:logger, :backends) Application.delete_env(:logger, :console) + :logger.update_handler_config(:default, :formatter, Logger.default_formatter()) end) end @@ -432,5 +433,22 @@ defmodule Pleroma.Config.DeprecationWarningsTest do `:default_formatter`. For more info visit: https://hexdocs.pm/logger/Logger.html#module-backends-and-backwards-compatibility """ end + + test "reconfigures logger" do + log = + capture_log(fn -> + Pleroma.Config.DeprecationWarnings.check_deprecated_logger_config() + end) + + # Formatter has inconsitent color formatting order + assert log =~ + """ + Reconfiguring console Logger with deprecated configuration syntax. + Handler configuration: + [level: :debug] + + Formatter: + """ + end end end From 02fc770e3638de20592fdaab115a041cda05f952 Mon Sep 17 00:00:00 2001 From: Phantasm Date: Wed, 14 Jan 2026 15:02:51 +0100 Subject: [PATCH 058/183] Elixir 1.19: Improve DeprecationWarningsTest Logger reconfiguration test --- .../config/deprecation_warnings_test.exs | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/test/pleroma/config/deprecation_warnings_test.exs b/test/pleroma/config/deprecation_warnings_test.exs index 787f9d2fd..a6ccf063c 100644 --- a/test/pleroma/config/deprecation_warnings_test.exs +++ b/test/pleroma/config/deprecation_warnings_test.exs @@ -11,6 +11,8 @@ defmodule Pleroma.Config.DeprecationWarningsTest do alias Pleroma.Config alias Pleroma.Config.DeprecationWarnings + require Logger + describe "filter exiftool" do test "gives warning when still used" do clear_config( @@ -392,11 +394,12 @@ defmodule Pleroma.Config.DeprecationWarningsTest do describe "check_deprecated_logger_config" do setup do Application.put_env(:logger, :backends, [:console, {ExSyslogger, :ex_syslogger}]) - Application.put_env(:logger, :console, level: :debug) + Application.put_env(:logger, :console, level: :info) on_exit(fn -> Application.delete_env(:logger, :backends) Application.delete_env(:logger, :console) + Logger.configure(level: :debug) :logger.update_handler_config(:default, :formatter, Logger.default_formatter()) end) end @@ -435,20 +438,18 @@ defmodule Pleroma.Config.DeprecationWarningsTest do end test "reconfigures logger" do - log = + empty_log = capture_log(fn -> - Pleroma.Config.DeprecationWarnings.check_deprecated_logger_config() + Logger.debug("This should not be logged") end) - # Formatter has inconsitent color formatting order - assert log =~ - """ - Reconfiguring console Logger with deprecated configuration syntax. - Handler configuration: - [level: :debug] + logged = + capture_log(fn -> + Logger.info("This should be logged") + end) - Formatter: - """ + assert empty_log =~ "" + assert logged =~ "This should be logged" end end end From 829a60ce9365ddc69655cc8e88f625dfa17c5e96 Mon Sep 17 00:00:00 2001 From: Phantasm Date: Thu, 15 Jan 2026 16:03:44 +0100 Subject: [PATCH 059/183] Elixir 1.19 MRFTest: Replace matchable_regexes with regexes_match! func --- test/pleroma/web/activity_pub/mrf_test.exs | 26 +++++++++++----------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/test/pleroma/web/activity_pub/mrf_test.exs b/test/pleroma/web/activity_pub/mrf_test.exs index 6656a01b8..401d4ebb3 100644 --- a/test/pleroma/web/activity_pub/mrf_test.exs +++ b/test/pleroma/web/activity_pub/mrf_test.exs @@ -10,22 +10,25 @@ defmodule Pleroma.Web.ActivityPub.MRFTest do alias Pleroma.Web.ActivityPub.MRF + defp regexes_match!([],[]), do: true + + defp regexes_match!([authority | authority_rest], [checked | checked_rest]) do + authority.source == checked.source and regexes_match!(authority_rest, checked_rest) + end + + defp regexes_match!(_, _), do: false + test "subdomains_regex/1" do regexes = MRF.subdomains_regex(["unsafe.tld", "*.unsafe.tld"]) - matchable_regexes = Enum.map(regexes, fn r -> r.source end) - assert matchable_regexes == [ - ~r/^unsafe.tld$/i.source, - ~r/^(.*\.)*unsafe.tld$/i.source - ] + assert regexes_match!(regexes, [~r/^unsafe.tld$/i, ~r/^(.*\.)*unsafe.tld$/i]) end describe "subdomain_match/2" do test "common domains" do regexes = MRF.subdomains_regex(["unsafe.tld", "unsafe2.tld"]) - matchable_regexes = Enum.map(regexes, fn r -> r.source end) - assert matchable_regexes == [~r/^unsafe.tld$/i.source, ~r/^unsafe2.tld$/i.source] + assert regexes_match!(regexes, [~r/^unsafe.tld$/i, ~r/^unsafe2.tld$/i]) assert MRF.subdomain_match?(regexes, "unsafe.tld") assert MRF.subdomain_match?(regexes, "unsafe2.tld") @@ -35,9 +38,8 @@ defmodule Pleroma.Web.ActivityPub.MRFTest do test "wildcard domains with one subdomain" do regexes = MRF.subdomains_regex(["*.unsafe.tld"]) - matchable_regexes = Enum.map(regexes, fn r -> r.source end) - assert matchable_regexes == [~r/^(.*\.)*unsafe.tld$/i.source] + assert regexes_match!(regexes, [~r/^(.*\.)*unsafe.tld$/i]) assert MRF.subdomain_match?(regexes, "unsafe.tld") assert MRF.subdomain_match?(regexes, "sub.unsafe.tld") @@ -47,9 +49,8 @@ defmodule Pleroma.Web.ActivityPub.MRFTest do test "wildcard domains with two subdomains" do regexes = MRF.subdomains_regex(["*.unsafe.tld"]) - matchable_regexes = Enum.map(regexes, fn r -> r.source end) - assert matchable_regexes == [~r/^(.*\.)*unsafe.tld$/i.source] + assert regexes_match!(regexes, [~r/^(.*\.)*unsafe.tld$/i]) assert MRF.subdomain_match?(regexes, "unsafe.tld") assert MRF.subdomain_match?(regexes, "sub.sub.unsafe.tld") @@ -59,9 +60,8 @@ defmodule Pleroma.Web.ActivityPub.MRFTest do test "matches are case-insensitive" do regexes = MRF.subdomains_regex(["UnSafe.TLD", "UnSAFE2.Tld"]) - matchable_regexes = Enum.map(regexes, fn r -> r.source end) - assert matchable_regexes == [~r/^UnSafe.TLD$/i.source, ~r/^UnSAFE2.Tld$/i.source] + assert regexes_match!(regexes, [~r/^UnSafe.TLD$/i, ~r/^UnSAFE2.Tld$/i]) assert MRF.subdomain_match?(regexes, "UNSAFE.TLD") assert MRF.subdomain_match?(regexes, "UNSAFE2.TLD") From ad7ab7741b1d426554765401c92857500f432c6c Mon Sep 17 00:00:00 2001 From: Phantasm Date: Thu, 15 Jan 2026 16:36:59 +0100 Subject: [PATCH 060/183] Elixir 1.19: Update dependencies in docs Elixir 1.15 support only OTP>=24 --- docs/installation/generic_dependencies.include | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/installation/generic_dependencies.include b/docs/installation/generic_dependencies.include index 588a6c019..382f07a6c 100644 --- a/docs/installation/generic_dependencies.include +++ b/docs/installation/generic_dependencies.include @@ -1,8 +1,8 @@ ## Required dependencies * PostgreSQL >=11.0 -* Elixir >=1.15.0 <1.19 -* Erlang OTP >=23.0.0 (supported: <28) +* Elixir >=1.15.0, <1.20 +* Erlang OTP >=24.0.0 (supported: <29) * git * file / libmagic * gcc or clang From b75f8f7c9addbd73f88ac468473e23aef86b55fa Mon Sep 17 00:00:00 2001 From: Phantasm Date: Thu, 15 Jan 2026 16:56:01 +0100 Subject: [PATCH 061/183] Elixir 1.19: Added changelogs --- changelog.d/elixir-1-19.fix | 1 + changelog.d/logger-backends.change | 1 + 2 files changed, 2 insertions(+) create mode 100644 changelog.d/elixir-1-19.fix create mode 100644 changelog.d/logger-backends.change diff --git a/changelog.d/elixir-1-19.fix b/changelog.d/elixir-1-19.fix new file mode 100644 index 000000000..53079b0f7 --- /dev/null +++ b/changelog.d/elixir-1-19.fix @@ -0,0 +1 @@ +Fixed Elixir 1.19 warnings diff --git a/changelog.d/logger-backends.change b/changelog.d/logger-backends.change new file mode 100644 index 000000000..5f044c2c7 --- /dev/null +++ b/changelog.d/logger-backends.change @@ -0,0 +1 @@ +Changed how Logger is configured due to changes in Elixir and added deprecation warnings for the old configuration format From 0ac1b65ba95b9bc3382494a84432bb8ecf088771 Mon Sep 17 00:00:00 2001 From: Phantasm Date: Wed, 21 Jan 2026 18:39:15 +0100 Subject: [PATCH 062/183] Elixir 1.19: Fix improper deprecation warning wording --- lib/pleroma/config/deprecation_warnings.ex | 4 +--- test/pleroma/config/deprecation_warnings_test.exs | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/config/deprecation_warnings.ex b/lib/pleroma/config/deprecation_warnings.ex index 4b6281e01..4f68c141c 100644 --- a/lib/pleroma/config/deprecation_warnings.ex +++ b/lib/pleroma/config/deprecation_warnings.ex @@ -452,8 +452,7 @@ defmodule Pleroma.Config.DeprecationWarnings do Your configuration is using deprecated syntax for configuring backends of Elixir's logger. `config :logger, backends: [...]` is deprecated syntax due to changes in Elixir. Please update your configuration at your earliest convenience to use: - `config :logger, backends: [...]` - Pleroma's default configuration will be updated in the next release which WILL override your settings. + `config :pleroma, :logger, backends: [...]` Note: `:console` is no longer considered a backend and is used by default, you can disable it using: `config :logger, :default_handler: false` @@ -472,7 +471,6 @@ defmodule Pleroma.Config.DeprecationWarnings do `config :logger, :console` is deprecated syntax due to changes in Elixir. Please update your configuration at your earliest convenience to use `config :logger, default_handler` and `config :logger, :default_formatter`. - Pleroma's default configuration will be updated in the next release which WILL override your settings. Note: `:default_handler` is used only for the `level` setting. All other configurations go under `:default_formatter`. For more info visit: https://hexdocs.pm/logger/Logger.html#module-backends-and-backwards-compatibility diff --git a/test/pleroma/config/deprecation_warnings_test.exs b/test/pleroma/config/deprecation_warnings_test.exs index a6ccf063c..221609da0 100644 --- a/test/pleroma/config/deprecation_warnings_test.exs +++ b/test/pleroma/config/deprecation_warnings_test.exs @@ -416,8 +416,7 @@ defmodule Pleroma.Config.DeprecationWarningsTest do Your configuration is using deprecated syntax for configuring backends of Elixir's logger. `config :logger, backends: [...]` is deprecated syntax due to changes in Elixir. Please update your configuration at your earliest convenience to use: - `config :logger, backends: [...]` - Pleroma's default configuration will be updated in the next release which WILL override your settings. + `config :pleroma, :logger, backends: [...]` Note: `:console` is no longer considered a backend and is used by default, you can disable it using: `config :logger, :default_handler: false` @@ -430,7 +429,6 @@ defmodule Pleroma.Config.DeprecationWarningsTest do `config :logger, :console` is deprecated syntax due to changes in Elixir. Please update your configuration at your earliest convenience to use `config :logger, default_handler` and `config :logger, :default_formatter`. - Pleroma's default configuration will be updated in the next release which WILL override your settings. Note: `:default_handler` is used only for the `level` setting. All other configurations go under `:default_formatter`. For more info visit: https://hexdocs.pm/logger/Logger.html#module-backends-and-backwards-compatibility From 1743a837dd31e7756a7e3293a170234b2d94fc57 Mon Sep 17 00:00:00 2001 From: Phantasm Date: Wed, 21 Jan 2026 20:02:26 +0100 Subject: [PATCH 063/183] Elixir 1.19: Set merged logger config correctly, remove log level test --- lib/pleroma/config/deprecation_warnings.ex | 29 +++++++++--------- .../config/deprecation_warnings_test.exs | 30 +++++-------------- 2 files changed, 23 insertions(+), 36 deletions(-) diff --git a/lib/pleroma/config/deprecation_warnings.ex b/lib/pleroma/config/deprecation_warnings.ex index 4f68c141c..e77785483 100644 --- a/lib/pleroma/config/deprecation_warnings.ex +++ b/lib/pleroma/config/deprecation_warnings.ex @@ -20,8 +20,7 @@ defmodule Pleroma.Config.DeprecationWarnings do "\n* `config :pleroma, :instance, mrf_transparency_exclusions` is now `config :pleroma, :mrf, transparency_exclusions`"} ] - @logger_config_knobs [:level, :translator_inspect_opts] - @logger_formatter_config_knows [:colors, :format, :metadata, :truncate, :utc_log] + @logger_formatter_config_knobs [:colors, :format, :metadata, :truncate, :utc_log] def check_exiftool_filter do filters = Config.get([Pleroma.Upload]) |> Keyword.get(:filters, []) @@ -421,21 +420,23 @@ defmodule Pleroma.Config.DeprecationWarnings do end defp merge_deprecated_logger_config(config) do - handler_config = Enum.filter(config, fn {k, _} -> k in @logger_config_knobs end) - formatter_config = Enum.filter(config, fn {k, _} -> k in @logger_formatter_config_knows end) + formatter_config = Enum.filter(config, fn {k, _} -> k in @logger_formatter_config_knobs end) formatter = Logger.default_formatter(formatter_config) + log_level = Keyword.get(config, :level, nil) - Logger.debug(""" - Reconfiguring console Logger with deprecated configuration syntax. - Handler configuration: - #{inspect(handler_config)} + if Config.get(:env) != :test do + Logger.debug(""" + Reconfiguring console Logger with deprecated configuration syntax. + Handler configuration: + log_level: #{inspect(log_level)} - Formatter: - #{inspect(formatter)} - """) + Formatter: + #{inspect(formatter)} + """) - Logger.configure(handler_config) - :logger.update_handler_config(:default, :formatter, formatter) + if log_level, do: :logger.update_handler_config(:default, :level, log_level) + :logger.update_handler_config(:default, :formatter, formatter) + end end @spec check_deprecated_logger_config() :: :ok | :error @@ -443,7 +444,7 @@ defmodule Pleroma.Config.DeprecationWarnings do backends_config = Application.get_env(:logger, :backends) console_config = Application.get_env(:logger, :console) - # Note: No need to merge the old backends config since it still works. + # NOTE: No need to merge the old backends config since it still works. # And new configuration will just add new Logger backends. backend = if backends_config do diff --git a/test/pleroma/config/deprecation_warnings_test.exs b/test/pleroma/config/deprecation_warnings_test.exs index 221609da0..475a88c4d 100644 --- a/test/pleroma/config/deprecation_warnings_test.exs +++ b/test/pleroma/config/deprecation_warnings_test.exs @@ -3,7 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Config.DeprecationWarningsTest do - use ExUnit.Case + use ExUnit.Case, async: false use Pleroma.Tests.Helpers import ExUnit.CaptureLog @@ -393,14 +393,15 @@ defmodule Pleroma.Config.DeprecationWarningsTest do describe "check_deprecated_logger_config" do setup do - Application.put_env(:logger, :backends, [:console, {ExSyslogger, :ex_syslogger}]) - Application.put_env(:logger, :console, level: :info) + initial_console = Application.get_env(:logger, :console, nil) + initial_backends = Application.get_env(:logger, :backends, nil) + + Application.put_env(:logger, :console, level: :all) + Application.put_env(:logger, :backends, [:console]) on_exit(fn -> - Application.delete_env(:logger, :backends) - Application.delete_env(:logger, :console) - Logger.configure(level: :debug) - :logger.update_handler_config(:default, :formatter, Logger.default_formatter()) + if initial_console, do: Application.put_env(:logger, :console, initial_console), else: Application.delete_env(:logger, :console) + if initial_backends, do: Application.put_env(:logger, :backends, initial_backends), else: Application.delete_env(:logger, :backends) end) end @@ -434,20 +435,5 @@ defmodule Pleroma.Config.DeprecationWarningsTest do `:default_formatter`. For more info visit: https://hexdocs.pm/logger/Logger.html#module-backends-and-backwards-compatibility """ end - - test "reconfigures logger" do - empty_log = - capture_log(fn -> - Logger.debug("This should not be logged") - end) - - logged = - capture_log(fn -> - Logger.info("This should be logged") - end) - - assert empty_log =~ "" - assert logged =~ "This should be logged" - end end end From f44774d387c88ce6289ff691a4aa5662c0966964 Mon Sep 17 00:00:00 2001 From: Phantasm Date: Wed, 21 Jan 2026 21:13:23 +0100 Subject: [PATCH 064/183] lint --- test/pleroma/config/deprecation_warnings_test.exs | 13 +++++++++++-- test/pleroma/web/activity_pub/mrf_test.exs | 2 +- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/test/pleroma/config/deprecation_warnings_test.exs b/test/pleroma/config/deprecation_warnings_test.exs index 475a88c4d..72dd63100 100644 --- a/test/pleroma/config/deprecation_warnings_test.exs +++ b/test/pleroma/config/deprecation_warnings_test.exs @@ -400,8 +400,17 @@ defmodule Pleroma.Config.DeprecationWarningsTest do Application.put_env(:logger, :backends, [:console]) on_exit(fn -> - if initial_console, do: Application.put_env(:logger, :console, initial_console), else: Application.delete_env(:logger, :console) - if initial_backends, do: Application.put_env(:logger, :backends, initial_backends), else: Application.delete_env(:logger, :backends) + if initial_console do + Application.put_env(:logger, :console, initial_console) + else + Application.delete_env(:logger, :console) + end + + if initial_backends do + Application.put_env(:logger, :backends, initial_backends) + else + Application.delete_env(:logger, :backends) + end end) end diff --git a/test/pleroma/web/activity_pub/mrf_test.exs b/test/pleroma/web/activity_pub/mrf_test.exs index 401d4ebb3..2d0f3b317 100644 --- a/test/pleroma/web/activity_pub/mrf_test.exs +++ b/test/pleroma/web/activity_pub/mrf_test.exs @@ -10,7 +10,7 @@ defmodule Pleroma.Web.ActivityPub.MRFTest do alias Pleroma.Web.ActivityPub.MRF - defp regexes_match!([],[]), do: true + defp regexes_match!([], []), do: true defp regexes_match!([authority | authority_rest], [checked | checked_rest]) do authority.source == checked.source and regexes_match!(authority_rest, checked_rest) From a12f9777f857002318a6d6da17473bb6a52276b7 Mon Sep 17 00:00:00 2001 From: Phantasm Date: Thu, 12 Feb 2026 23:46:08 +0100 Subject: [PATCH 065/183] Elixir 1.19: Bump Woodpecker test pipeline to 1.19 --- .woodpecker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.woodpecker.yml b/.woodpecker.yml index 4ceb1cab5..f6b06dc61 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -4,7 +4,7 @@ when: steps: test: - image: elixir:1.15-alpine + image: elixir:1.19-alpine environment: MIX_ENV: test DB_HOST: postgres From 01b139c6331977f5d622866127be9b4291cd4732 Mon Sep 17 00:00:00 2001 From: Phantasm Date: Sun, 22 Feb 2026 21:50:28 +0100 Subject: [PATCH 066/183] Elixir 1.19: Add note to deprecation warnings test The Logger does not get reconfigured when running in the test env, the added and on_exit removed settings below should have no effect besides triggering the deprecation warnings. To make breakage even less possible, async is turned off for these tests. TransferTask test does almost the same with the Application env. --- test/pleroma/config/deprecation_warnings_test.exs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/pleroma/config/deprecation_warnings_test.exs b/test/pleroma/config/deprecation_warnings_test.exs index 72dd63100..b65b0e630 100644 --- a/test/pleroma/config/deprecation_warnings_test.exs +++ b/test/pleroma/config/deprecation_warnings_test.exs @@ -396,6 +396,9 @@ defmodule Pleroma.Config.DeprecationWarningsTest do initial_console = Application.get_env(:logger, :console, nil) initial_backends = Application.get_env(:logger, :backends, nil) + # NOTE: The Logger does not get reconfigured when running in the test env, + # the added and on_exit removed settings below should have no effect besides + # triggering the deprecation warnings Application.put_env(:logger, :console, level: :all) Application.put_env(:logger, :backends, [:console]) From 08ba0d46d77cbb19f218b464a1cbeb700227e6a8 Mon Sep 17 00:00:00 2001 From: Phantasm Date: Sun, 22 Feb 2026 22:27:48 +0100 Subject: [PATCH 067/183] ActivityDraft: Add missing __MODULE__ matches and drop unneeded ones --- lib/pleroma/web/common_api/activity_draft.ex | 48 ++++++++++---------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/lib/pleroma/web/common_api/activity_draft.ex b/lib/pleroma/web/common_api/activity_draft.ex index 6c84ef656..16489663a 100644 --- a/lib/pleroma/web/common_api/activity_draft.ex +++ b/lib/pleroma/web/common_api/activity_draft.ex @@ -99,34 +99,34 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do |> Map.put("cc", draft.cc) |> Map.put("actor", draft.user.ap_id) - %__MODULE__{draft | object: object} + %{draft | object: object} end defp put_params(%__MODULE__{} = draft, params) do params = Map.put_new(params, :in_reply_to_status_id, params[:in_reply_to_id]) - %__MODULE__{draft | params: params} + %{draft | params: params} end defp status(%__MODULE__{params: %{status: status}} = draft) do - %__MODULE__{draft | status: String.trim(status)} + %{draft | status: String.trim(status)} end defp summary(%__MODULE__{params: params} = draft) do - %__MODULE__{draft | summary: Map.get(params, :spoiler_text, "")} + %{draft | summary: Map.get(params, :spoiler_text, "")} end defp full_payload(%__MODULE__{status: status, summary: summary} = draft) do full_payload = String.trim(status <> summary) case Utils.validate_character_limit(full_payload, draft.attachments) do - :ok -> %__MODULE__{draft | full_payload: full_payload} + :ok -> %{draft | full_payload: full_payload} {:error, message} -> add_error(draft, message) end end defp attachments(%__MODULE__{params: params} = draft) do attachments = Utils.attachments_from_ids(params, draft.user) - draft = %__MODULE__{draft | attachments: attachments} + draft = %{draft | attachments: attachments} case Utils.validate_attachments_count(attachments) do :ok -> draft @@ -149,7 +149,7 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do with %Activity{} = activity <- Activity.get_by_id(id), true <- Visibility.visible_for_user?(activity, draft.user), {_, type} when type in ["Create", "Announce"] <- {:type, activity.data["type"]} do - %__MODULE__{draft | in_reply_to: activity} + %{draft | in_reply_to: activity} else nil -> add_error(draft, dgettext("errors", "Cannot reply to a deleted status")) @@ -170,7 +170,7 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do defp in_reply_to( %__MODULE__{params: %{in_reply_to_status_id: %Activity{} = in_reply_to}} = draft ) do - %__MODULE__{draft | in_reply_to: in_reply_to} + %{draft | in_reply_to: in_reply_to} end defp in_reply_to(draft), do: draft @@ -179,14 +179,14 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do when not_empty_string(id) do case Activity.get_by_id_with_object(id) do %Activity{} = activity -> - %__MODULE__{draft | quote_post: activity} + %{draft | quote_post: activity} _ -> draft end end - defp quote_post(%{params: %{quote_id: id}} = draft) when not_empty_string(id) do + defp quote_post(%__MODULE__{params: %{quote_id: id}} = draft) when not_empty_string(id) do quote_post(%{draft | params: Map.put(draft.params, :quoted_status_id, id)}) end @@ -194,7 +194,7 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do defp in_reply_to_conversation(%__MODULE__{} = draft) do in_reply_to_conversation = Participation.get(draft.params[:in_reply_to_conversation_id]) - %__MODULE__{draft | in_reply_to_conversation: in_reply_to_conversation} + %{draft | in_reply_to_conversation: in_reply_to_conversation} end defp visibility(%__MODULE__{params: params} = draft) do @@ -203,7 +203,7 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do add_error(draft, dgettext("errors", "The message visibility must be direct")) {visibility, _} -> - %__MODULE__{draft | visibility: visibility} + %{draft | visibility: visibility} end end @@ -219,7 +219,7 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do false end - defp quoting_visibility(%{quote_post: %Activity{}} = draft) do + defp quoting_visibility(%__MODULE__{quote_post: %Activity{}} = draft) do with %Object{} = object <- Object.normalize(draft.quote_post, fetch: false), true <- can_quote?(draft, object, Visibility.get_visibility(object)) do draft @@ -232,7 +232,7 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do defp expires_at(%__MODULE__{} = draft) do case CommonAPI.check_expiry_date(draft.params[:expires_in]) do - {:ok, expires_at} -> %__MODULE__{draft | expires_at: expires_at} + {:ok, expires_at} -> %{draft | expires_at: expires_at} {:error, message} -> add_error(draft, message) end end @@ -240,7 +240,7 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do defp poll(%__MODULE__{} = draft) do case Utils.make_poll_data(draft.params) do {:ok, {poll, poll_emoji}} -> - %__MODULE__{draft | extra: poll, emoji: Map.merge(draft.emoji, poll_emoji)} + %{draft | extra: poll, emoji: Map.merge(draft.emoji, poll_emoji)} {:error, message} -> add_error(draft, message) @@ -258,22 +258,22 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do |> Kernel.++(mentioned_ap_ids) |> Utils.get_addressed_users(draft.params[:to]) - %__MODULE__{draft | content_html: content_html, mentions: mentions, tags: tags} + %{draft | content_html: content_html, mentions: mentions, tags: tags} end defp to_and_cc(%__MODULE__{} = draft) do {to, cc} = Utils.get_to_and_cc(draft) - %__MODULE__{draft | to: to, cc: cc} + %{draft | to: to, cc: cc} end defp context(%__MODULE__{} = draft) do context = Utils.make_context(draft.in_reply_to, draft.in_reply_to_conversation) - %__MODULE__{draft | context: context} + %{draft | context: context} end defp sensitive(%__MODULE__{} = draft) do sensitive = draft.params[:sensitive] - %__MODULE__{draft | sensitive: sensitive} + %{draft | sensitive: sensitive} end defp language(%__MODULE__{} = draft) do @@ -285,7 +285,7 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do _ -> LanguageDetector.detect(draft.content_html <> " " <> draft.summary) end - %__MODULE__{draft | language: language} + %{draft | language: language} end defp object(%__MODULE__{} = draft) do @@ -329,12 +329,12 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do |> Map.put("generator", draft.params[:generator]) |> Map.put("language", draft.language) - %__MODULE__{draft | object: object} + %{draft | object: object} end defp preview?(%__MODULE__{} = draft) do preview? = Pleroma.Web.Utils.Params.truthy_param?(draft.params[:preview]) - %__MODULE__{draft | preview?: preview?} + %{draft | preview?: preview?} end defp changes(%__MODULE__{} = draft) do @@ -357,14 +357,14 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do } |> Utils.maybe_add_list_data(draft.user, draft.visibility) - %__MODULE__{draft | changes: changes} + %{draft | changes: changes} end defp with_valid(%{valid?: true} = draft, func), do: func.(draft) defp with_valid(draft, _func), do: draft defp add_error(%__MODULE__{} = draft, message) do - %__MODULE__{draft | valid?: false, errors: [message | draft.errors]} + %{draft | valid?: false, errors: [message | draft.errors]} end defp validate(%{valid?: true} = draft), do: {:ok, draft} From 8ac967518b1729aa2530bb2e3003a31bc40cabfb Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 25 Mar 2026 15:11:45 -0700 Subject: [PATCH 068/183] Switch to the newly relocated Pleroma.Search.object_to_search_data/1 --- lib/pleroma/search/paradedb.ex | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/search/paradedb.ex b/lib/pleroma/search/paradedb.ex index a6649a894..20be5b6f9 100644 --- a/lib/pleroma/search/paradedb.ex +++ b/lib/pleroma/search/paradedb.ex @@ -18,10 +18,9 @@ defmodule Pleroma.Search.ParadeDB do maybe_restrict_local: 2 ] - import Pleroma.Search.Meilisearch, only: [object_to_search_data: 1] - alias Pleroma.Activity alias Pleroma.Config.Getting, as: Config + alias Pleroma.Search @max_limit 40 @default_table "pleroma_search_documents" @@ -81,7 +80,7 @@ defmodule Pleroma.Search.ParadeDB do @impl true def add_to_index(activity) do - maybe_search_data = object_to_search_data(activity.object) + maybe_search_data = Search.object_to_search_data(activity.object) if activity.data["type"] == "Create" and maybe_search_data do dumped_activity_id = dump_activity_id(activity.id) From 1e3456a1604ebe5818e319f47bbeb79828c3d6e5 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 25 Mar 2026 15:14:35 -0700 Subject: [PATCH 069/183] Ensure the Ecto Repo for ParadeDB is started for the data indexing Mix task --- lib/mix/pleroma.ex | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/lib/mix/pleroma.ex b/lib/mix/pleroma.ex index e7869919c..372bf3399 100644 --- a/lib/mix/pleroma.ex +++ b/lib/mix/pleroma.ex @@ -62,7 +62,8 @@ defmodule Mix.Pleroma do {Majic.Pool, [name: Pleroma.MajicPool, pool_size: Pleroma.Config.get([:majic_pool, :size], 2)]} ] ++ - http_children(adapter) + http_children(adapter) ++ + search_children() cachex_children = Enum.map(@cachex_children, &Pleroma.Application.build_cachex(&1, [])) @@ -141,4 +142,14 @@ defmodule Mix.Pleroma do end defp http_children(_), do: [] + + defp search_children do + case Config.get([Pleroma.Search, :module]) do + Pleroma.Search.ParadeDB -> + [Pleroma.Search.ParadeDB.Repo] + + _ -> + [] + end + end end From 3776f13f20172fa7c88127ba28583fba27cb1e1c Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 1 Apr 2026 19:10:29 +0000 Subject: [PATCH 070/183] Fix warning --- lib/mix/pleroma.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/mix/pleroma.ex b/lib/mix/pleroma.ex index 372bf3399..aebad6984 100644 --- a/lib/mix/pleroma.ex +++ b/lib/mix/pleroma.ex @@ -144,7 +144,7 @@ defmodule Mix.Pleroma do defp http_children(_), do: [] defp search_children do - case Config.get([Pleroma.Search, :module]) do + case Pleroma.Config.get([Pleroma.Search, :module]) do Pleroma.Search.ParadeDB -> [Pleroma.Search.ParadeDB.Repo] From ae763ff6cf97fdb78edc5bdad13aab917cea37ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?nicole=20miko=C5=82ajczyk?= Date: Mon, 4 May 2026 01:41:12 +0200 Subject: [PATCH 071/183] api/v2/admin/accounts: support 'permissions' param MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: nicole mikołajczyk --- .../api_spec/operations/mastodon_admin/account_operation.ex | 2 +- .../web/mastodon_api/admin/controllers/account_controller.ex | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/api_spec/operations/mastodon_admin/account_operation.ex b/lib/pleroma/web/api_spec/operations/mastodon_admin/account_operation.ex index e9143364c..ad79d43d7 100644 --- a/lib/pleroma/web/api_spec/operations/mastodon_admin/account_operation.ex +++ b/lib/pleroma/web/api_spec/operations/mastodon_admin/account_operation.ex @@ -126,7 +126,7 @@ defmodule Pleroma.Web.ApiSpec.MastodonAdmin.AccountOperation do :permissions, :query, :string, - "Filter for accounts with staff permissions (users that can manage reports). (not implemented yet)" + "Filter for accounts with staff permissions (users that can manage reports)." ), Operation.parameter( :role_ids, diff --git a/lib/pleroma/web/mastodon_api/admin/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/admin/controllers/account_controller.ex index 833bf1db5..e96566774 100644 --- a/lib/pleroma/web/mastodon_api/admin/controllers/account_controller.ex +++ b/lib/pleroma/web/mastodon_api/admin/controllers/account_controller.ex @@ -183,6 +183,7 @@ defmodule Pleroma.Web.MastodonAPI.Admin.AccountController do %{} |> maybe_filter_origin(params) |> maybe_filter_status(params) + |> maybe_filter_staff(params) |> maybe_filter_nickname(params) |> maybe_filter_name(params) |> maybe_filter_domain(params) @@ -240,6 +241,9 @@ defmodule Pleroma.Web.MastodonAPI.Admin.AccountController do defp maybe_filter_email(criteria, %{email: email} = _params), do: Map.put(criteria, :email, email) + defp maybe_filter_staff(criteria, %{permissions: "staff"} = _params), + do: Map.put(criteria, :staff, true) + defp maybe_filter_staff(criteria, %{staff: staff} = _params), do: Map.put(criteria, :staff, staff) From 11281d740a136c349b927587c1d895ea590f76af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?nicole=20miko=C5=82ajczyk?= Date: Mon, 4 May 2026 13:48:38 +0200 Subject: [PATCH 072/183] make GET /api/v1/admin/reports match Mastodon admin API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: nicole mikołajczyk --- .../mastodon_admin/report_operation.ex | 1 + .../admin/controllers/report_controller.ex | 39 ++++++++++++++++++- .../controllers/report_controller_test.exs | 12 +++++- 3 files changed, 49 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/api_spec/operations/mastodon_admin/report_operation.ex b/lib/pleroma/web/api_spec/operations/mastodon_admin/report_operation.ex index 3360cb843..f59db9966 100644 --- a/lib/pleroma/web/api_spec/operations/mastodon_admin/report_operation.ex +++ b/lib/pleroma/web/api_spec/operations/mastodon_admin/report_operation.ex @@ -28,6 +28,7 @@ defmodule Pleroma.Web.ApiSpec.MastodonAdmin.ReportOperation do parameters: [ Operation.parameter(:resolved, :query, :boolean, "Filter for resolved reports"), + Operation.parameter(:unresolved, :query, :boolean, "Filter for open reports"), Operation.parameter(:account_id, :query, :string, "Filter by author account id"), Operation.parameter( :target_account_id, diff --git a/lib/pleroma/web/mastodon_api/admin/controllers/report_controller.ex b/lib/pleroma/web/mastodon_api/admin/controllers/report_controller.ex index 2d89d13bc..5dbf19aba 100644 --- a/lib/pleroma/web/mastodon_api/admin/controllers/report_controller.ex +++ b/lib/pleroma/web/mastodon_api/admin/controllers/report_controller.ex @@ -89,11 +89,46 @@ defmodule Pleroma.Web.MastodonAPI.Admin.ReportController do end end + def assign_to_self(%{assigns: %{user: admin}} = conn, %{id: id}) do + with {:ok, activity} <- CommonAPI.assign_report_to_account(id, admin.id), + report <- Activity.get_by_id_with_user_actor(activity.id) do + ModerationLog.insert_log(%{ + action: "report_assigned", + actor: admin, + subject: activity, + subject_actor: report.user_actor, + assigned_account: admin.nickname + }) + + render(conn, "show.json", Report.extract_report_info(report)) + else + {:error, error} -> + json_response(conn, :bad_request, %{error: error}) + end + end + + def unassign(%{assigns: %{user: admin}} = conn, %{id: id}) do + with {:ok, activity} <- CommonAPI.assign_report_to_account(id, nil), + report <- Activity.get_by_id_with_user_actor(activity.id) do + ModerationLog.insert_log(%{ + action: "report_unassigned", + actor: admin, + subject: activity, + subject_actor: report.user_actor + }) + + render(conn, "show.json", Report.extract_report_info(report)) + else + {:error, error} -> + json_response(conn, :bad_request, %{error: error}) + end + end + defp restrict_state(opts, %{resolved: true}), do: Map.put(opts, :state, "resolved") - defp restrict_state(opts, %{resolved: false}), do: Map.put(opts, :state, "open") + defp restrict_state(opts, %{resolved: _}), do: Map.put(opts, :state, "resolved") - defp restrict_state(opts, _params), do: opts + defp restrict_state(opts, _params), do: Map.put(opts, :state, "open") defp restrict_actor(opts, %{account_id: actor}) do with %User{ap_id: ap_id} <- User.get_by_id(actor) do diff --git a/test/pleroma/web/mastodon_api/admin/controllers/report_controller_test.exs b/test/pleroma/web/mastodon_api/admin/controllers/report_controller_test.exs index a0cc5f459..702dcf1db 100644 --- a/test/pleroma/web/mastodon_api/admin/controllers/report_controller_test.exs +++ b/test/pleroma/web/mastodon_api/admin/controllers/report_controller_test.exs @@ -50,7 +50,17 @@ defmodule Pleroma.Web.MastodonAPI.Admin.ReportControllerTest do assert [%{"id" => ^report_id1}] = conn - |> get("/api/v1/admin/reports?resolved=false") + |> get("/api/v1/admin/reports?unresolved=true") + |> json_response_and_validate_schema(200) + + assert [%{"id" => ^report_id1}] = + conn + |> get("/api/v1/admin/reports") + |> json_response_and_validate_schema(200) + + assert [%{"id" => ^report_id2}, %{"id" => ^report_id1}] = + conn + |> get("/api/v1/admin/reports?resolved=true&unresolved=true") |> json_response_and_validate_schema(200) assert [%{"id" => ^report_id2}] = From 784a319f2eb9b98e6e034c38b462dd1e11390359 Mon Sep 17 00:00:00 2001 From: Phantasm Date: Thu, 14 May 2026 00:22:24 +0200 Subject: [PATCH 073/183] Woodpecker CI: Bump unit testing pipeline to Elixir 1.19 --- ...t-testing-elixir-1.18.yaml => unit-testing-elixir-1.19.yaml} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename .woodpecker/{unit-testing-elixir-1.18.yaml => unit-testing-elixir-1.19.yaml} (97%) diff --git a/.woodpecker/unit-testing-elixir-1.18.yaml b/.woodpecker/unit-testing-elixir-1.19.yaml similarity index 97% rename from .woodpecker/unit-testing-elixir-1.18.yaml rename to .woodpecker/unit-testing-elixir-1.19.yaml index 9ad9eebc9..462e2ee5d 100644 --- a/.woodpecker/unit-testing-elixir-1.18.yaml +++ b/.woodpecker/unit-testing-elixir-1.19.yaml @@ -19,7 +19,7 @@ variables: steps: unit-testing-elixir-1.18: - image: elixir:1.18-otp-27-alpine + image: elixir:1.19-otp-28-alpine entrypoint: *script_file_entrypoint environment: MIX_ENV: test From 1e65c7a69e3cbd84009563a8d1974f772188a4b7 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Fri, 22 May 2026 20:36:52 +0400 Subject: [PATCH 074/183] Add grouped notifications API --- changelog.d/grouped-notifications-api.add | 1 + lib/pleroma/notification.ex | 84 +++++++++ .../operations/notification_operation.ex | 159 ++++++++++++++++ .../controllers/notification_controller.ex | 146 ++++++++++++++- lib/pleroma/web/mastodon_api/mastodon_api.ex | 177 ++++++++++++++++-- .../mastodon_api/views/notification_view.ex | 120 +++++++++++- lib/pleroma/web/router.ex | 10 + .../notification_controller_test.exs | 169 +++++++++++++++++ .../views/notification_view_test.exs | 8 +- 9 files changed, 850 insertions(+), 24 deletions(-) create mode 100644 changelog.d/grouped-notifications-api.add diff --git a/changelog.d/grouped-notifications-api.add b/changelog.d/grouped-notifications-api.add new file mode 100644 index 000000000..82ce8b178 --- /dev/null +++ b/changelog.d/grouped-notifications-api.add @@ -0,0 +1 @@ +Add Mastodon-compatible grouped notifications API endpoints under `/api/v2/notifications` diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index b8693c3a8..ec10854e3 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -62,6 +62,90 @@ defmodule Pleroma.Notification do |> Repo.aggregate(:count, :id) end + @groupable_notification_types ~w{favourite follow reblog} + @group_bucket_seconds 12 * 60 * 60 + + def groupable_notification_types, do: @groupable_notification_types + + def group_notifications(notifications, grouped_types \\ nil) do + grouped_types = normalize_grouped_types(grouped_types) + + {group_keys, grouped_notifications} = + Enum.reduce(notifications, {[], %{}}, fn notification, + {group_keys, grouped_notifications} -> + group_key = group_key(notification, grouped_types) + + if Map.has_key?(grouped_notifications, group_key) do + {group_keys, Map.update!(grouped_notifications, group_key, &[notification | &1])} + else + {[group_key | group_keys], Map.put(grouped_notifications, group_key, [notification])} + end + end) + + group_keys + |> Enum.reverse() + |> Enum.map(fn group_key -> + grouped_notifications + |> Map.fetch!(group_key) + |> Enum.reverse() + end) + end + + def group_key(notification, grouped_types \\ nil) + + def group_key(%Notification{} = notification, nil) do + group_key(notification, @groupable_notification_types) + end + + def group_key(%Notification{type: type} = notification, grouped_types) do + grouped_types = normalize_grouped_types(grouped_types) + + if type in @groupable_notification_types and type in grouped_types do + case group_target_id(notification) do + nil -> ungrouped_group_key(notification) + target_id -> "#{type}-#{target_id}-#{group_time_bucket(notification)}" + end + else + ungrouped_group_key(notification) + end + end + + def normalize_grouped_types(nil), do: @groupable_notification_types + def normalize_grouped_types(types) when is_list(types), do: types + def normalize_grouped_types(type), do: [type] + + defp ungrouped_group_key(%Notification{id: id}), do: "ungrouped-#{id}" + + defp group_time_bucket(%Notification{inserted_at: inserted_at}) do + inserted_at + |> NaiveDateTime.to_erl() + |> :calendar.datetime_to_gregorian_seconds() + |> div(@group_bucket_seconds) + end + + defp group_target_id(%Notification{type: type, activity: activity}) + when type in ["favourite", "reblog"] do + with object_id when is_binary(object_id) <- object_id_for(activity), + %Activity{id: id} <- Activity.get_create_by_object_ap_id(object_id) do + to_string(id) + else + _ -> nil + end + end + + defp group_target_id(%Notification{type: "follow", activity: %{data: %{"object" => ap_id}}}) do + case User.get_cached_by_ap_id(ap_id) do + %User{id: id} -> to_string(id) + _ -> nil + end + end + + defp group_target_id(_), do: nil + + defp object_id_for(%{data: %{"object" => %{"id" => id}}}) when is_binary(id), do: id + defp object_id_for(%{data: %{"object" => id}}) when is_binary(id), do: id + defp object_id_for(_), do: nil + @notification_types ~w{ favourite follow diff --git a/lib/pleroma/web/api_spec/operations/notification_operation.ex b/lib/pleroma/web/api_spec/operations/notification_operation.ex index 94d1f6b82..ba996c13b 100644 --- a/lib/pleroma/web/api_spec/operations/notification_operation.ex +++ b/lib/pleroma/web/api_spec/operations/notification_operation.ex @@ -91,6 +91,79 @@ defmodule Pleroma.Web.ApiSpec.NotificationOperation do } end + def grouped_index_operation do + %Operation{ + tags: ["Notifications"], + summary: "Retrieve grouped notifications", + description: "Notifications concerning the user, grouped by supported notification types.", + operationId: "NotificationController.grouped_index", + security: [%{"oAuth" => ["read:notifications"]}], + parameters: grouped_notification_params() ++ pagination_params(), + responses: %{ + 200 => Operation.response("Grouped notifications", "application/json", grouped_result()), + 404 => Operation.response("Error", "application/json", ApiError) + } + } + end + + def show_group_operation do + %Operation{ + tags: ["Notifications"], + summary: "Retrieve a notification group", + operationId: "NotificationController.show_group", + security: [%{"oAuth" => ["read:notifications"]}], + parameters: [group_key_param()], + responses: %{ + 200 => Operation.response("Grouped notifications", "application/json", grouped_result()), + 404 => Operation.response("Error", "application/json", ApiError) + } + } + end + + def group_accounts_operation do + %Operation{ + tags: ["Notifications"], + summary: "Retrieve notification group accounts", + operationId: "NotificationController.group_accounts", + security: [%{"oAuth" => ["write:notifications"]}], + parameters: [group_key_param()], + responses: %{ + 200 => + Operation.response("Accounts", "application/json", %Schema{ + type: :array, + items: Account + }) + } + } + end + + def unread_count_operation do + %Operation{ + tags: ["Notifications"], + summary: "Retrieve unread grouped notification count", + operationId: "NotificationController.unread_count", + security: [%{"oAuth" => ["read:notifications"]}], + parameters: + grouped_notification_params() ++ + [ + Operation.parameter( + :limit, + :query, + %Schema{type: :integer}, + "Maximum number of notifications to count" + ) + ], + responses: %{ + 200 => + Operation.response("Unread notification group count", "application/json", %Schema{ + type: :object, + properties: %{count: %Schema{type: :integer}} + }), + 404 => Operation.response("Error", "application/json", ApiError) + } + } + end + def clear_operation do %Operation{ tags: ["Notifications"], @@ -114,6 +187,18 @@ defmodule Pleroma.Web.ApiSpec.NotificationOperation do } end + def dismiss_group_operation do + %Operation{ + tags: ["Notifications"], + summary: "Dismiss a notification group", + description: "Clear all notifications in a notification group from the server.", + operationId: "NotificationController.dismiss_group", + parameters: [group_key_param()], + security: [%{"oAuth" => ["write:notifications"]}], + responses: %{200 => empty_object_response()} + } + end + def dismiss_via_body_operation do %Operation{ tags: ["Notifications"], @@ -194,6 +279,73 @@ defmodule Pleroma.Web.ApiSpec.NotificationOperation do } end + defp grouped_result do + %Schema{ + title: "GroupedNotificationsResults", + type: :object, + properties: %{ + accounts: %Schema{type: :array, items: Account}, + statuses: %Schema{type: :array, items: Status}, + notification_groups: %Schema{type: :array, items: notification_group()} + }, + required: [:accounts, :statuses, :notification_groups] + } + end + + defp notification_group do + %Schema{ + title: "NotificationGroup", + type: :object, + properties: %{ + group_key: %Schema{type: :string}, + notifications_count: %Schema{type: :integer}, + type: notification_type(), + most_recent_notification_id: %Schema{type: :string}, + page_min_id: %Schema{type: :string, nullable: true}, + page_max_id: %Schema{type: :string, nullable: true}, + latest_page_notification_at: %Schema{type: :string, format: :"date-time", nullable: true}, + sample_account_ids: %Schema{type: :array, items: %Schema{type: :string}}, + status_id: %Schema{type: :string, nullable: true} + }, + required: [ + :group_key, + :notifications_count, + :type, + :most_recent_notification_id, + :sample_account_ids + ] + } + end + + defp grouped_notification_params do + [ + Operation.parameter( + :exclude_types, + :query, + %Schema{type: :array, items: notification_type()}, + "Array of types to exclude" + ), + Operation.parameter( + :account_id, + :query, + %Schema{type: :string}, + "Return only notifications received from this account" + ), + Operation.parameter( + :types, + :query, + %Schema{type: :array, items: notification_type()}, + "Include the notifications for activities with the given types" + ), + Operation.parameter( + :grouped_types, + :query, + %Schema{type: :array, items: notification_type()}, + "Notification types that may be grouped" + ) + ] + end + defp notification_type do %Schema{ type: :string, @@ -239,4 +391,11 @@ defmodule Pleroma.Web.ApiSpec.NotificationOperation do required: true ) end + + defp group_key_param do + Operation.parameter(:group_key, :path, :string, "Notification group key", + example: "favourite-123", + required: true + ) + end end diff --git a/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex b/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex index afd83b785..5cd8576c1 100644 --- a/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex @@ -8,10 +8,12 @@ defmodule Pleroma.Web.MastodonAPI.NotificationController do import Pleroma.Web.ControllerHelper, only: [add_link_headers: 2] alias Pleroma.Notification + alias Pleroma.User + alias Pleroma.Web.MastodonAPI.AccountView alias Pleroma.Web.MastodonAPI.MastodonAPI alias Pleroma.Web.Plugs.OAuthScopesPlug - @oauth_read_actions [:show, :index] + @oauth_read_actions [:show, :index, :grouped_index, :show_group, :unread_count] plug(Pleroma.Web.ApiSpec.CastAndValidate, replace_params: false) @@ -39,7 +41,7 @@ defmodule Pleroma.Web.MastodonAPI.NotificationController do # GET /api/v1/notifications def index(%{private: %{open_api_spex: %{params: %{account_id: account_id} = params}}} = conn, _) do - case Pleroma.User.get_cached_by_id(account_id) do + case User.get_cached_by_id(account_id) do %{ap_id: account_ap_id} -> params = params @@ -59,10 +61,110 @@ defmodule Pleroma.Web.MastodonAPI.NotificationController do do_get_notifications(conn, params) end + # GET /api/v2/notifications + def grouped_index( + %{private: %{open_api_spex: %{params: %{account_id: account_id} = params}}} = conn, + _ + ) do + case User.get_cached_by_id(account_id) do + %{ap_id: account_ap_id} -> + params = + params + |> Map.delete(:account_id) + |> Map.put(:account_ap_id, account_ap_id) + + do_get_grouped_notifications(conn, params) + + _ -> + conn + |> put_status(:not_found) + |> json(%{"error" => "Account is not found"}) + end + end + + def grouped_index(%{private: %{open_api_spex: %{params: params}}} = conn, _) do + do_get_grouped_notifications(conn, params) + end + + # GET /api/v2/notifications/:group_key + def show_group( + %{assigns: %{user: user}, private: %{open_api_spex: %{params: %{group_key: group_key}}}} = + conn, + _ + ) do + notifications = MastodonAPI.get_notification_group(user, group_key, %{}) + + if Enum.empty?(notifications) do + conn + |> put_status(:not_found) + |> json(%{"error" => "Notification group is not found"}) + else + grouped_types = if String.starts_with?(group_key, "ungrouped-"), do: [], else: nil + + render(conn, "grouped_index.json", + notification_groups: [notifications], + for: user, + grouped_types: grouped_types + ) + end + end + + # GET /api/v2/notifications/:group_key/accounts + def group_accounts( + %{assigns: %{user: user}, private: %{open_api_spex: %{params: %{group_key: group_key}}}} = + conn, + _ + ) do + users = + user + |> MastodonAPI.get_notification_group(group_key, %{}) + |> notification_actors() + + json(conn, AccountView.render("index.json", %{users: users, for: user})) + end + + # GET /api/v2/notifications/unread_count + def unread_count( + %{private: %{open_api_spex: %{params: %{account_id: account_id} = params}}} = conn, + _ + ) do + case User.get_cached_by_id(account_id) do + %{ap_id: account_ap_id} -> + params = + params + |> Map.delete(:account_id) + |> Map.put(:account_ap_id, account_ap_id) + + do_get_unread_group_count(conn, params) + + _ -> + conn + |> put_status(:not_found) + |> json(%{"error" => "Account is not found"}) + end + end + + def unread_count(%{private: %{open_api_spex: %{params: params}}} = conn, _) do + do_get_unread_group_count(conn, params) + end + + # POST /api/v2/notifications/:group_key/dismiss + def dismiss_group( + %{assigns: %{user: user}, private: %{open_api_spex: %{params: %{group_key: group_key}}}} = + conn, + _ + ) do + ids = + user + |> MastodonAPI.get_notification_group(group_key, %{}) + |> Enum.map(& &1.id) + + Notification.destroy_multiple(user, ids) + json(conn, %{}) + end + defp do_get_notifications(%{assigns: %{user: user}} = conn, params) do - params = - Map.new(params, fn {k, v} -> {to_string(k), v} end) - |> Map.put_new("types", Map.get(params, :include_types, @default_notification_types)) + params = normalize_notification_params(params) notifications = MastodonAPI.get_notifications(user, params) @@ -74,6 +176,40 @@ defmodule Pleroma.Web.MastodonAPI.NotificationController do ) end + defp do_get_grouped_notifications(%{assigns: %{user: user}} = conn, params) do + params = normalize_notification_params(params) + + {notification_groups, page_notifications, notification_group_counts} = + MastodonAPI.get_grouped_notification_page(user, params) + + conn + |> add_link_headers(page_notifications) + |> render("grouped_index.json", + notification_groups: notification_groups, + notification_group_counts: notification_group_counts, + for: user, + grouped_types: params["grouped_types"] + ) + end + + defp do_get_unread_group_count(%{assigns: %{user: user}} = conn, params) do + params = normalize_notification_params(params) + json(conn, %{count: MastodonAPI.unread_notification_group_count(user, params)}) + end + + defp normalize_notification_params(params) do + params + |> Map.new(fn {k, v} -> {to_string(k), v} end) + |> Map.put_new("types", Map.get(params, :include_types, @default_notification_types)) + end + + defp notification_actors(notifications) do + notifications + |> Enum.map(&User.get_cached_by_ap_id(&1.activity.data["actor"])) + |> Enum.filter(& &1) + |> Enum.uniq_by(& &1.id) + end + # GET /api/v1/notifications/:id def show(%{assigns: %{user: user}, private: %{open_api_spex: %{params: %{id: id}}}} = conn, _) do with {:ok, notification} <- Notification.get(user, id) do diff --git a/lib/pleroma/web/mastodon_api/mastodon_api.ex b/lib/pleroma/web/mastodon_api/mastodon_api.ex index c9e045d23..57cbfd6bf 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api.ex @@ -8,6 +8,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPI do alias Pleroma.Notification alias Pleroma.Pagination + alias Pleroma.Repo alias Pleroma.ScheduledActivity alias Pleroma.User alias Pleroma.Web.CommonAPI @@ -61,29 +62,175 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPI do end def get_notifications(user, params \\ %{}) do - options = - cast_params(params) |> Map.update(:include_types, [], fn include_types -> include_types end) + user + |> notifications_query(params) + |> Pagination.fetch_paginated(params) + end - options = - if ("pleroma:report" not in options.include_types and - User.privileged?(user, :reports_manage_reports)) or - User.privileged?(user, :reports_manage_reports) do - options - else - options - |> Map.update(:exclude_types, ["pleroma:report"], fn current_exclude_types -> - current_exclude_types ++ ["pleroma:report"] - end) - end + def get_grouped_notification_page(user, params \\ %{}) do + grouped_types = Map.get(params, "grouped_types", Map.get(params, :grouped_types)) + {query, order} = group_pagination_query(notifications_query(user, params), params) + + notifications = + query + |> order_by([n], [{^order, n.id}]) + |> Repo.all() + + notification_group_counts = + Enum.frequencies_by(notifications, &Notification.group_key(&1, grouped_types)) + + page_notifications = + notifications + |> take_grouped_page_notifications(grouped_types, grouped_limit(params)) + + page_notifications = + if order == :asc, do: Enum.reverse(page_notifications), else: page_notifications + + { + Notification.group_notifications(page_notifications, grouped_types), + page_notifications, + notification_group_counts + } + end + + def get_grouped_notification_groups(user, params \\ %{}) do + {groups, _notifications, _notification_group_counts} = + get_grouped_notification_page(user, params) + + groups + end + + def get_notification_group(user, group_key, params \\ %{}) + + def get_notification_group(user, "ungrouped-" <> notification_id, _params) do + case Notification.get(user, notification_id) do + {:ok, notification} -> [notification] + _ -> [] + end + end + + def get_notification_group(user, group_key, params) do + grouped_types = Map.get(params, "grouped_types", Map.get(params, :grouped_types)) + + user + |> notifications_query(params) + |> order_by([n], desc: n.id) + |> Repo.all() + |> Enum.filter(&(Notification.group_key(&1, grouped_types) == group_key)) + end + + defp group_pagination_query(query, params) do + cond do + min_id = Map.get(params, "min_id", Map.get(params, :min_id)) -> + query = where(query, [n], n.id > ^min_id) + + query = + case Map.get(params, "max_id", Map.get(params, :max_id)) do + nil -> query + max_id -> where(query, [n], n.id < ^max_id) + end + + {query, :asc} + + since_id = Map.get(params, "since_id", Map.get(params, :since_id)) -> + {where(query, [n], n.id > ^since_id), :desc} + + max_id = Map.get(params, "max_id", Map.get(params, :max_id)) -> + {where(query, [n], n.id < ^max_id), :desc} + + true -> + {query, :desc} + end + end + + defp grouped_limit(params) do + params + |> Map.get("limit", Map.get(params, :limit, 40)) + |> parse_limit(40) + |> min(80) + end + + defp take_grouped_page_notifications(notifications, grouped_types, group_limit) do + {notifications, _group_keys} = + Enum.reduce_while(notifications, {[], MapSet.new()}, fn notification, + {notifications, group_keys} -> + group_key = Notification.group_key(notification, grouped_types) + + cond do + MapSet.member?(group_keys, group_key) -> + {:cont, {[notification | notifications], group_keys}} + + MapSet.size(group_keys) < group_limit -> + {:cont, {[notification | notifications], MapSet.put(group_keys, group_key)}} + + true -> + {:halt, {notifications, group_keys}} + end + end) + + Enum.reverse(notifications) + end + + def unread_notification_group_count(user, params \\ %{}) do + grouped_types = Map.get(params, "grouped_types", Map.get(params, :grouped_types)) + limit = unread_count_limit(params) + + user + |> notifications_query(params) + |> where([n], n.seen == false) + |> order_by([n], desc: n.id) + |> limit(^limit) + |> Repo.all() + |> Notification.group_notifications(grouped_types) + |> length() + end + + defp notifications_query(user, params) do + options = notification_options(user, params) user |> Notification.for_user_query(options) |> restrict(:types, options) |> restrict(:exclude_types, options) |> restrict(:account_ap_id, options) - |> Pagination.fetch_paginated(params) end + defp notification_options(user, params) do + options = + params + |> cast_params() + |> Map.update(:include_types, [], fn include_types -> include_types end) + + if ("pleroma:report" not in options.include_types and + User.privileged?(user, :reports_manage_reports)) or + User.privileged?(user, :reports_manage_reports) do + options + else + options + |> Map.update(:exclude_types, ["pleroma:report"], fn current_exclude_types -> + current_exclude_types ++ ["pleroma:report"] + end) + end + end + + defp unread_count_limit(params) do + params + |> Map.get("limit", Map.get(params, :limit, 100)) + |> parse_limit(100) + |> min(1000) + end + + defp parse_limit(limit, _default) when is_integer(limit) and limit > 0, do: limit + + defp parse_limit(limit, default) when is_binary(limit) do + case Integer.parse(limit) do + {limit, _} when limit > 0 -> limit + _ -> default + end + end + + defp parse_limit(_, default), do: default + def get_scheduled_activities(user, params \\ %{}) do user |> ScheduledActivity.for_user_query() @@ -95,6 +242,8 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPI do exclude_types: {:array, :string}, types: {:array, :string}, exclude_visibilities: {:array, :string}, + grouped_types: {:array, :string}, + limit: :integer, reblogs: :boolean, with_muted: :boolean, account_ap_id: :string, diff --git a/lib/pleroma/web/mastodon_api/views/notification_view.ex b/lib/pleroma/web/mastodon_api/views/notification_view.ex index 27532c42a..466073853 100644 --- a/lib/pleroma/web/mastodon_api/views/notification_view.ex +++ b/lib/pleroma/web/mastodon_api/views/notification_view.ex @@ -72,6 +72,49 @@ defmodule Pleroma.Web.MastodonAPI.NotificationView do safe_render_many(notifications, NotificationView, "show.json", opts) end + def render("grouped_index.json", %{notifications: notifications} = opts) do + grouped_types = Notification.normalize_grouped_types(opts[:grouped_types]) + + opts + |> Map.delete(:notifications) + |> Map.put( + :notification_groups, + Notification.group_notifications(notifications, grouped_types) + ) + |> then(&render("grouped_index.json", &1)) + end + + def render( + "grouped_index.json", + %{notification_groups: notification_groups, for: reading_user} = opts + ) do + grouped_types = Notification.normalize_grouped_types(opts[:grouped_types]) + notification_group_counts = Map.get(opts, :notification_group_counts, %{}) + + statuses = + notification_groups + |> Enum.map(&List.first/1) + |> Enum.map(&render("show.json", %{notification: &1, for: reading_user})) + |> Enum.map(& &1[:status]) + |> Enum.filter(& &1) + |> Enum.uniq_by(& &1[:id]) + + actors = + notification_groups + |> List.flatten() + |> notification_actors() + + %{ + accounts: AccountView.render("index.json", %{users: actors, for: reading_user}), + statuses: statuses, + notification_groups: + Enum.map( + notification_groups, + &render_group(&1, reading_user, grouped_types, notification_group_counts) + ) + } + end + def render( "show.json", %{ @@ -95,7 +138,7 @@ defmodule Pleroma.Web.MastodonAPI.NotificationView do response = %{ id: to_string(notification.id), - group_key: "ungrouped-" <> to_string(notification.id), + group_key: Notification.group_key(notification), type: notification.type, created_at: CommonAPI.Utils.to_masto_date(notification.inserted_at), account: account, @@ -137,6 +180,81 @@ defmodule Pleroma.Web.MastodonAPI.NotificationView do Map.put(response, :report, report_render) end + defp render_group( + [%Notification{} = notification | _] = notifications, + _reading_user, + grouped_types, + notification_group_counts + ) do + latest_notification = List.first(notifications) + oldest_notification = List.last(notifications) + status_activity = status_activity_for_group(notifications, grouped_types) + group_key = Notification.group_key(notification, grouped_types) + + response = %{ + group_key: group_key, + notifications_count: Map.get(notification_group_counts, group_key, length(notifications)), + type: notification.type, + most_recent_notification_id: to_string(latest_notification.id), + page_min_id: to_string(oldest_notification.id), + page_max_id: to_string(latest_notification.id), + latest_page_notification_at: CommonAPI.Utils.to_masto_date(latest_notification.inserted_at), + sample_account_ids: + notifications + |> notification_actors() + |> Enum.map(&to_string(&1.id)) + } + + if status_activity do + Map.put(response, :status_id, to_string(status_activity.id)) + else + response + end + end + + defp status_activity_for_group([%Notification{} = notification | _], grouped_types) do + status_activity_for(notification, grouped_types) + end + + defp status_activity_for(%Notification{type: type, activity: activity}, _grouped_types) + when type in ["mention", "status", "poll"] do + activity + end + + defp status_activity_for(%Notification{type: type} = notification, grouped_types) + when type in ["favourite", "reblog"] do + group_key = Notification.group_key(notification, grouped_types) + + case String.split(group_key, "-", parts: 3) do + [^type, activity_id, _bucket] -> + case Activity.create_by_id_with_object(activity_id) do + %Activity{} = activity -> activity + _ -> parent_status_activity(notification.activity) + end + + _ -> + parent_status_activity(notification.activity) + end + end + + defp status_activity_for(%Notification{type: type, activity: activity}, _grouped_types) + when type in ["update", "pleroma:emoji_reaction"] do + parent_status_activity(activity) + end + + defp status_activity_for(_, _grouped_types), do: nil + + defp parent_status_activity(activity) do + Activity.get_create_by_object_ap_id(object_id_for(activity)) + end + + defp notification_actors(notifications) do + notifications + |> Enum.map(&User.get_cached_by_ap_id(&1.activity.data["actor"])) + |> Enum.filter(& &1) + |> Enum.uniq_by(& &1.id) + end + defp put_emoji(response, activity) do response |> Map.put(:emoji, activity.data["content"]) diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 813c5c482..5657bcad7 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -844,6 +844,16 @@ defmodule Pleroma.Web.Router do get("/directory", DirectoryController, :index) end + scope "/api/v2", Pleroma.Web.MastodonAPI do + pipe_through(:authenticated_api) + + get("/notifications", NotificationController, :grouped_index) + get("/notifications/unread_count", NotificationController, :unread_count) + get("/notifications/:group_key/accounts", NotificationController, :group_accounts) + get("/notifications/:group_key", NotificationController, :show_group) + post("/notifications/:group_key/dismiss", NotificationController, :dismiss_group) + end + scope "/api/v2", Pleroma.Web.MastodonAPI do pipe_through(:api) diff --git a/test/pleroma/web/mastodon_api/controllers/notification_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/notification_controller_test.exs index dd4b7389e..1d3a9e966 100644 --- a/test/pleroma/web/mastodon_api/controllers/notification_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/notification_controller_test.exs @@ -577,6 +577,175 @@ defmodule Pleroma.Web.MastodonAPI.NotificationControllerTest do assert [%{"id" => ^notification4_id}, %{"id" => ^notification3_id}] = result end + describe "GET /api/v2/notifications" do + test "groups favourite notifications for the same status" do + %{user: user, conn: conn} = oauth_access(["read:notifications"]) + other_user1 = insert(:user) + other_user2 = insert(:user) + + {:ok, status} = CommonAPI.post(user, %{status: "hello"}) + {:ok, _} = CommonAPI.favorite(status.id, other_user1) + {:ok, _} = CommonAPI.favorite(status.id, other_user2) + + notification_ids = + Notification + |> Repo.all() + |> Enum.map(&to_string(&1.id)) + + result = + conn + |> get("/api/v2/notifications") + |> json_response_and_validate_schema(200) + + assert [%{"id" => account_id1}, %{"id" => account_id2}] = result["accounts"] + assert account_id1 in [other_user1.id, other_user2.id] + assert account_id2 in [other_user1.id, other_user2.id] + assert [%{"id" => status_id}] = result["statuses"] + assert status_id == status.id + + assert [group] = result["notification_groups"] + assert group["type"] == "favourite" + assert group["notifications_count"] == 2 + assert group["status_id"] == status.id + assert group["most_recent_notification_id"] in notification_ids + assert group["page_min_id"] in notification_ids + assert group["page_max_id"] in notification_ids + assert group["latest_page_notification_at"] + assert Enum.sort(group["sample_account_ids"]) == Enum.sort([other_user1.id, other_user2.id]) + group_key = group["group_key"] + + assert [%{"group_key" => ^group_key}, %{"group_key" => ^group_key}] = + conn + |> get("/api/v1/notifications") + |> json_response_and_validate_schema(200) + + assert %{"notification_groups" => [shown_group]} = + conn + |> get("/api/v2/notifications/#{group_key}") + |> json_response_and_validate_schema(200) + + assert shown_group["group_key"] == group_key + assert shown_group["notifications_count"] == 2 + end + + test "round-trips ungrouped group keys when grouped_types excludes a type" do + %{user: user, conn: conn} = oauth_access(["read:notifications"]) + other_user = insert(:user) + + {:ok, status} = CommonAPI.post(user, %{status: "hello"}) + {:ok, _} = CommonAPI.favorite(status.id, other_user) + + %{"notification_groups" => [%{"group_key" => "ungrouped-" <> _ = group_key}]} = + conn + |> get("/api/v2/notifications?grouped_types[]=reblog") + |> json_response_and_validate_schema(200) + + assert %{ + "notification_groups" => [%{"group_key" => ^group_key, "notifications_count" => 1}] + } = + conn + |> get("/api/v2/notifications/#{group_key}") + |> json_response_and_validate_schema(200) + end + + test "paginates notification groups instead of raw notifications" do + %{user: user, conn: conn} = oauth_access(["read:notifications"]) + other_user1 = insert(:user) + other_user2 = insert(:user) + other_user3 = insert(:user) + + {:ok, older_status} = CommonAPI.post(user, %{status: "older"}) + {:ok, _} = CommonAPI.favorite(older_status.id, other_user3) + + {:ok, newer_status} = CommonAPI.post(user, %{status: "newer"}) + {:ok, _} = CommonAPI.favorite(newer_status.id, other_user1) + {:ok, _} = CommonAPI.favorite(newer_status.id, other_user2) + older_status_id = older_status.id + newer_status_id = newer_status.id + + %{"notification_groups" => groups} = + conn + |> get("/api/v2/notifications?limit=2") + |> json_response_and_validate_schema(200) + + assert [ + %{"status_id" => ^newer_status_id, "notifications_count" => 2}, + %{"status_id" => ^older_status_id, "notifications_count" => 1} + ] = groups + end + + test "returns total notification count for a partially represented group" do + %{user: user, conn: conn} = oauth_access(["read:notifications"]) + other_user1 = insert(:user) + other_user2 = insert(:user) + other_user3 = insert(:user) + + {:ok, grouped_status} = CommonAPI.post(user, %{status: "grouped"}) + {:ok, _} = CommonAPI.favorite(grouped_status.id, other_user1) + + {:ok, other_status} = CommonAPI.post(user, %{status: "other"}) + {:ok, _} = CommonAPI.favorite(other_status.id, other_user3) + {:ok, _} = CommonAPI.favorite(grouped_status.id, other_user2) + grouped_status_id = grouped_status.id + + assert %{ + "notification_groups" => [ + %{"status_id" => ^grouped_status_id, "notifications_count" => 2} + ] + } = + conn + |> get("/api/v2/notifications?limit=1") + |> json_response_and_validate_schema(200) + end + + test "counts unread notification groups" do + %{user: user, conn: conn} = oauth_access(["read:notifications"]) + other_user1 = insert(:user) + other_user2 = insert(:user) + mentioner = insert(:user) + + {:ok, status} = CommonAPI.post(user, %{status: "hello"}) + {:ok, _} = CommonAPI.favorite(status.id, other_user1) + {:ok, _} = CommonAPI.favorite(status.id, other_user2) + {:ok, _} = CommonAPI.post(mentioner, %{status: "hi @#{user.nickname}"}) + + assert %{"count" => 2} = + conn + |> get("/api/v2/notifications/unread_count") + |> json_response_and_validate_schema(200) + end + + test "lists accounts from and dismisses a notification group" do + %{user: user, conn: conn} = oauth_access(["read:notifications", "write:notifications"]) + other_user1 = insert(:user) + other_user2 = insert(:user) + + {:ok, status} = CommonAPI.post(user, %{status: "hello"}) + {:ok, _} = CommonAPI.favorite(status.id, other_user1) + {:ok, _} = CommonAPI.favorite(status.id, other_user2) + + %{"notification_groups" => [%{"group_key" => group_key}]} = + conn + |> get("/api/v2/notifications") + |> json_response_and_validate_schema(200) + + account_ids = + conn + |> get("/api/v2/notifications/#{group_key}/accounts") + |> json_response_and_validate_schema(200) + |> Enum.map(& &1["id"]) + + assert Enum.sort(account_ids) == Enum.sort([other_user1.id, other_user2.id]) + + assert %{} = + conn + |> post("/api/v2/notifications/#{group_key}/dismiss") + |> json_response_and_validate_schema(200) + + assert [] = Notification.for_user(user) + end + end + test "doesn't see notifications after muting user with notifications" do %{user: user, conn: conn} = oauth_access(["read:notifications"]) user2 = insert(:user) diff --git a/test/pleroma/web/mastodon_api/views/notification_view_test.exs b/test/pleroma/web/mastodon_api/views/notification_view_test.exs index c8287bb79..aa11ad682 100644 --- a/test/pleroma/web/mastodon_api/views/notification_view_test.exs +++ b/test/pleroma/web/mastodon_api/views/notification_view_test.exs @@ -101,7 +101,7 @@ defmodule Pleroma.Web.MastodonAPI.NotificationViewTest do expected = %{ id: to_string(notification.id), - group_key: "ungrouped-#{to_string(notification.id)}", + group_key: Notification.group_key(notification), pleroma: %{is_seen: false, is_muted: false}, type: "favourite", account: AccountView.render("show.json", %{user: another_user, for: user}), @@ -122,7 +122,7 @@ defmodule Pleroma.Web.MastodonAPI.NotificationViewTest do expected = %{ id: to_string(notification.id), - group_key: "ungrouped-#{to_string(notification.id)}", + group_key: Notification.group_key(notification), pleroma: %{is_seen: false, is_muted: false}, type: "reblog", account: AccountView.render("show.json", %{user: another_user, for: user}), @@ -141,7 +141,7 @@ defmodule Pleroma.Web.MastodonAPI.NotificationViewTest do expected = %{ id: to_string(notification.id), - group_key: "ungrouped-#{to_string(notification.id)}", + group_key: Notification.group_key(notification), pleroma: %{is_seen: false, is_muted: false}, type: "follow", account: AccountView.render("show.json", %{user: follower, for: followed}), @@ -333,7 +333,7 @@ defmodule Pleroma.Web.MastodonAPI.NotificationViewTest do expected = %{ id: to_string(notification.id), - group_key: "ungrouped-#{to_string(notification.id)}", + group_key: Notification.group_key(notification), pleroma: %{is_seen: true, is_muted: true}, type: "favourite", account: AccountView.render("show.json", %{user: another_user, for: user}), From add3844c6d8f6e540d139f55e44047d946d9559a Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sat, 23 May 2026 11:38:33 +0400 Subject: [PATCH 075/183] Persist notification group keys --- changelog.d/grouped-notifications-api.add | 2 +- lib/pleroma/notification.ex | 39 ++- .../operations/notification_operation.ex | 5 +- lib/pleroma/web/controller_helper.ex | 12 +- .../controllers/notification_controller.ex | 37 +-- lib/pleroma/web/mastodon_api/mastodon_api.ex | 307 ++++++++++++++---- .../mastodon_api/views/notification_view.ex | 44 ++- ...3070000_add_group_key_to_notifications.exs | 15 + .../notification_controller_test.exs | 157 ++++++++- .../views/notification_view_test.exs | 3 + 10 files changed, 505 insertions(+), 116 deletions(-) create mode 100644 priv/repo/migrations/20260523070000_add_group_key_to_notifications.exs diff --git a/changelog.d/grouped-notifications-api.add b/changelog.d/grouped-notifications-api.add index 82ce8b178..8c2cc5397 100644 --- a/changelog.d/grouped-notifications-api.add +++ b/changelog.d/grouped-notifications-api.add @@ -1 +1 @@ -Add Mastodon-compatible grouped notifications API endpoints under `/api/v2/notifications` +Add Mastodon-compatible grouped notifications API endpoints under `/api/v2/notifications` for newly-created notifications diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index ec10854e3..afd750823 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -35,6 +35,7 @@ defmodule Pleroma.Notification do # remember to add a migration to add it to the `notifications_type` enum # as well. field(:type, :string) + field(:group_key, :string) belongs_to(:user, User, type: FlakeId.Ecto.CompatType) belongs_to(:activity, Activity, type: FlakeId.Ecto.CompatType) @@ -49,7 +50,7 @@ defmodule Pleroma.Notification do |> type_from_activity() notification - |> changeset(%{type: type}) + |> changeset(%{type: type, group_key: grouped_notification_key(type, activity)}) |> Repo.update() end end @@ -97,14 +98,11 @@ defmodule Pleroma.Notification do group_key(notification, @groupable_notification_types) end - def group_key(%Notification{type: type} = notification, grouped_types) do + def group_key(%Notification{type: type, group_key: group_key} = notification, grouped_types) do grouped_types = normalize_grouped_types(grouped_types) - if type in @groupable_notification_types and type in grouped_types do - case group_target_id(notification) do - nil -> ungrouped_group_key(notification) - target_id -> "#{type}-#{target_id}-#{group_time_bucket(notification)}" - end + if type in @groupable_notification_types and type in grouped_types and is_binary(group_key) do + group_key else ungrouped_group_key(notification) end @@ -116,15 +114,27 @@ defmodule Pleroma.Notification do defp ungrouped_group_key(%Notification{id: id}), do: "ungrouped-#{id}" - defp group_time_bucket(%Notification{inserted_at: inserted_at}) do + defp grouped_notification_key(type, activity) when type in @groupable_notification_types do + with target_id when is_binary(target_id) <- group_target_id(type, activity) do + # Mastodon uses Redis to reuse a recent bucket for rolling 12h windows. Pleroma keeps keys + # deterministic and self-contained; clients must treat group_key as opaque either way. + "#{type}-#{target_id}-#{group_time_bucket(activity)}" + end + end + + defp grouped_notification_key(_type, _activity), do: nil + + defp group_time_bucket(%Activity{inserted_at: inserted_at}) when not is_nil(inserted_at) do inserted_at |> NaiveDateTime.to_erl() |> :calendar.datetime_to_gregorian_seconds() |> div(@group_bucket_seconds) end - defp group_target_id(%Notification{type: type, activity: activity}) - when type in ["favourite", "reblog"] do + defp group_time_bucket(_), + do: group_time_bucket(%Activity{inserted_at: NaiveDateTime.utc_now()}) + + defp group_target_id(type, activity) when type in ["favourite", "reblog"] do with object_id when is_binary(object_id) <- object_id_for(activity), %Activity{id: id} <- Activity.get_create_by_object_ap_id(object_id) do to_string(id) @@ -133,14 +143,14 @@ defmodule Pleroma.Notification do end end - defp group_target_id(%Notification{type: "follow", activity: %{data: %{"object" => ap_id}}}) do + defp group_target_id("follow", %{data: %{"object" => ap_id}}) do case User.get_cached_by_ap_id(ap_id) do %User{id: id} -> to_string(id) _ -> nil end end - defp group_target_id(_), do: nil + defp group_target_id(_, _), do: nil defp object_id_for(%{data: %{"object" => %{"id" => id}}}) when is_binary(id), do: id defp object_id_for(%{data: %{"object" => id}}) when is_binary(id), do: id @@ -163,7 +173,7 @@ defmodule Pleroma.Notification do def changeset(%Notification{} = notification, attrs) do notification - |> cast(attrs, [:seen, :type]) + |> cast(attrs, [:seen, :type, :group_key]) |> validate_inclusion(:type, @notification_types) end @@ -542,7 +552,8 @@ defmodule Pleroma.Notification do user_id: user.id, activity: activity, seen: mark_as_read?(activity, user), - type: type + type: type, + group_key: grouped_notification_key(type, activity) }) |> Marker.multi_set_last_read_id(user, "notifications") |> Repo.transaction() diff --git a/lib/pleroma/web/api_spec/operations/notification_operation.ex b/lib/pleroma/web/api_spec/operations/notification_operation.ex index ba996c13b..6c8f7d5e9 100644 --- a/lib/pleroma/web/api_spec/operations/notification_operation.ex +++ b/lib/pleroma/web/api_spec/operations/notification_operation.ex @@ -3,7 +3,6 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ApiSpec.NotificationOperation do - alias OpenApiSpex.Operation alias OpenApiSpex.Operation alias OpenApiSpex.Schema alias Pleroma.Web.ApiSpec.Schemas.Account @@ -125,7 +124,7 @@ defmodule Pleroma.Web.ApiSpec.NotificationOperation do tags: ["Notifications"], summary: "Retrieve notification group accounts", operationId: "NotificationController.group_accounts", - security: [%{"oAuth" => ["write:notifications"]}], + security: [%{"oAuth" => ["read:notifications"]}], parameters: [group_key_param()], responses: %{ 200 => @@ -269,7 +268,7 @@ defmodule Pleroma.Web.ApiSpec.NotificationOperation do }, example: %{ "id" => "34975861", - "group-key" => "ungrouped-34975861", + "group_key" => "ungrouped-34975861", "type" => "mention", "created_at" => "2019-11-23T07:49:02.064Z", "account" => Account.schema().example, diff --git a/lib/pleroma/web/controller_helper.ex b/lib/pleroma/web/controller_helper.ex index b15b0ea81..eecd1774f 100644 --- a/lib/pleroma/web/controller_helper.ex +++ b/lib/pleroma/web/controller_helper.ex @@ -55,16 +55,24 @@ defmodule Pleroma.Web.ControllerHelper do # TODO: Only fetch the params from open_api_spex when everything is converted @id_keys Pagination.page_keys() -- ["limit", "order"] + @id_key_atoms Enum.map(@id_keys, &String.to_atom/1) defp build_pagination_fields(conn, min_id, max_id, extra_params, order) do + path_param_keys = + conn.path_params + |> Map.keys() + |> Enum.flat_map(&[&1, String.to_existing_atom(&1)]) + params = if Map.has_key?(conn.private, :open_api_spex) do get_in(conn, [Access.key(:private), Access.key(:open_api_spex), Access.key(:params)]) else conn.params end - |> Map.drop(Map.keys(conn.path_params) |> Enum.map(&String.to_existing_atom/1)) + |> Map.drop(path_param_keys) |> Map.merge(extra_params) - |> Map.drop(@id_keys) + # OpenApiSpex casts params to atoms while uncast conn params are string-keyed. Drop both so + # generated links replace pagination cursors instead of preserving stale request cursors. + |> Map.drop(@id_keys ++ @id_key_atoms) {{next_id, nid}, {prev_id, pid}} = if order == :desc, diff --git a/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex b/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex index 5cd8576c1..2cc31db0c 100644 --- a/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex @@ -13,7 +13,9 @@ defmodule Pleroma.Web.MastodonAPI.NotificationController do alias Pleroma.Web.MastodonAPI.MastodonAPI alias Pleroma.Web.Plugs.OAuthScopesPlug - @oauth_read_actions [:show, :index, :grouped_index, :show_group, :unread_count] + # Mastodon's docs currently list write:notifications for group accounts, but the endpoint is + # read-only and Mastodon's implementation accepts read:notifications. Prefer least privilege. + @oauth_read_actions [:show, :index, :grouped_index, :show_group, :group_accounts, :unread_count] plug(Pleroma.Web.ApiSpec.CastAndValidate, replace_params: false) @@ -92,7 +94,8 @@ defmodule Pleroma.Web.MastodonAPI.NotificationController do conn, _ ) do - notifications = MastodonAPI.get_notification_group(user, group_key, %{}) + {notifications, notification_group_counts, notification_group_bounds} = + MastodonAPI.get_notification_group_result(user, group_key, %{}) if Enum.empty?(notifications) do conn @@ -103,8 +106,11 @@ defmodule Pleroma.Web.MastodonAPI.NotificationController do render(conn, "grouped_index.json", notification_groups: [notifications], + notification_group_counts: notification_group_counts, + notification_group_bounds: notification_group_bounds, for: user, - grouped_types: grouped_types + grouped_types: grouped_types, + include_page_metadata: false ) end end @@ -115,10 +121,9 @@ defmodule Pleroma.Web.MastodonAPI.NotificationController do conn, _ ) do - users = - user - |> MastodonAPI.get_notification_group(group_key, %{}) - |> notification_actors() + # Mastodon paginates this endpoint in code, but the public docs say it returns accounts of all + # notifications in the group and do not document cursor params here. Follow the documented API. + users = MastodonAPI.get_notification_group_accounts(user, group_key) json(conn, AccountView.render("index.json", %{users: users, for: user})) end @@ -154,12 +159,7 @@ defmodule Pleroma.Web.MastodonAPI.NotificationController do conn, _ ) do - ids = - user - |> MastodonAPI.get_notification_group(group_key, %{}) - |> Enum.map(& &1.id) - - Notification.destroy_multiple(user, ids) + MastodonAPI.dismiss_notification_group(user, group_key) json(conn, %{}) end @@ -179,7 +179,8 @@ defmodule Pleroma.Web.MastodonAPI.NotificationController do defp do_get_grouped_notifications(%{assigns: %{user: user}} = conn, params) do params = normalize_notification_params(params) - {notification_groups, page_notifications, notification_group_counts} = + {notification_groups, page_notifications, notification_group_counts, + notification_group_bounds} = MastodonAPI.get_grouped_notification_page(user, params) conn @@ -187,6 +188,7 @@ defmodule Pleroma.Web.MastodonAPI.NotificationController do |> render("grouped_index.json", notification_groups: notification_groups, notification_group_counts: notification_group_counts, + notification_group_bounds: notification_group_bounds, for: user, grouped_types: params["grouped_types"] ) @@ -203,13 +205,6 @@ defmodule Pleroma.Web.MastodonAPI.NotificationController do |> Map.put_new("types", Map.get(params, :include_types, @default_notification_types)) end - defp notification_actors(notifications) do - notifications - |> Enum.map(&User.get_cached_by_ap_id(&1.activity.data["actor"])) - |> Enum.filter(& &1) - |> Enum.uniq_by(& &1.id) - end - # GET /api/v1/notifications/:id def show(%{assigns: %{user: user}, private: %{open_api_spex: %{params: %{id: id}}}} = conn, _) do with {:ok, notification} <- Notification.get(user, id) do diff --git a/lib/pleroma/web/mastodon_api/mastodon_api.ex b/lib/pleroma/web/mastodon_api/mastodon_api.ex index 57cbfd6bf..cb11a3f4a 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api.ex @@ -6,6 +6,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPI do import Ecto.Query import Ecto.Changeset + alias Pleroma.Marker alias Pleroma.Notification alias Pleroma.Pagination alias Pleroma.Repo @@ -13,6 +14,8 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPI do alias Pleroma.User alias Pleroma.Web.CommonAPI + @notification_group_sample_limit 8 + @spec follow(User.t(), User.t(), map) :: {:ok, User.t()} | {:error, String.t()} def follow(follower, followed, params \\ %{}) do result = @@ -68,38 +71,55 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPI do end def get_grouped_notification_page(user, params \\ %{}) do - grouped_types = Map.get(params, "grouped_types", Map.get(params, :grouped_types)) - {query, order} = group_pagination_query(notifications_query(user, params), params) + grouped_types = + params + |> Map.get("grouped_types", Map.get(params, :grouped_types)) + |> Notification.normalize_grouped_types() - notifications = + query = notifications_query(user, params) + {order, cursor_filters} = group_pagination(params) + + group_rows = query - |> order_by([n], [{^order, n.id}]) - |> Repo.all() + |> notification_group_rows(grouped_types, grouped_limit(params), order, cursor_filters) - notification_group_counts = - Enum.frequencies_by(notifications, &Notification.group_key(&1, grouped_types)) + group_rows = if order == :asc, do: Enum.reverse(group_rows), else: group_rows - page_notifications = - notifications - |> take_grouped_page_notifications(grouped_types, grouped_limit(params)) + page_notifications = representative_notifications(query, group_rows) - page_notifications = - if order == :asc, do: Enum.reverse(page_notifications), else: page_notifications + notification_groups = + Enum.map(group_rows, ¬ification_group_sample(user, params, &1.group_key)) { - Notification.group_notifications(page_notifications, grouped_types), + notification_groups, page_notifications, - notification_group_counts + notification_group_counts(group_rows), + notification_group_bounds(group_rows) } end def get_grouped_notification_groups(user, params \\ %{}) do - {groups, _notifications, _notification_group_counts} = + {groups, _notifications, _notification_group_counts, _notification_group_bounds} = get_grouped_notification_page(user, params) groups end + def get_notification_group_result(user, group_key, params \\ %{}) do + notifications = notification_group_sample(user, params, group_key) + metadata = notification_group_metadata(user, group_key, params) + + if Enum.empty?(notifications) or is_nil(metadata) do + {[], %{}, %{}} + else + { + notifications, + %{group_key => metadata.notifications_count}, + %{group_key => Map.drop(metadata, [:notifications_count])} + } + end + end + def get_notification_group(user, group_key, params \\ %{}) def get_notification_group(user, "ungrouped-" <> notification_id, _params) do @@ -110,36 +130,58 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPI do end def get_notification_group(user, group_key, params) do - grouped_types = Map.get(params, "grouped_types", Map.get(params, :grouped_types)) - - user - |> notifications_query(params) - |> order_by([n], desc: n.id) - |> Repo.all() - |> Enum.filter(&(Notification.group_key(&1, grouped_types) == group_key)) + notification_group_sample(user, params, group_key) end - defp group_pagination_query(query, params) do + def get_notification_group_accounts(user, "ungrouped-" <> notification_id) do + with {:ok, notification} <- Notification.get(user, notification_id), + %User{} = actor <- User.get_cached_by_ap_id(notification.activity.data["actor"]) do + [actor] + else + _ -> [] + end + end + + def get_notification_group_accounts(user, group_key) do + user + |> notification_group_query(group_key, %{}) + |> exclude(:preload) + |> distinct(true) + |> select([user_actor: user_actor], user_actor) + |> Repo.all() + end + + def dismiss_notification_group(user, "ungrouped-" <> notification_id) do + Notification.destroy_multiple(user, [notification_id]) + end + + def dismiss_notification_group(%User{id: user_id}, group_key) do + Notification + |> where([n], n.user_id == ^user_id and n.group_key == ^group_key) + |> Repo.delete_all() + end + + defp group_pagination(params) do cond do min_id = Map.get(params, "min_id", Map.get(params, :min_id)) -> - query = where(query, [n], n.id > ^min_id) + cursor_filters = [{:gt, min_id}] - query = + cursor_filters = case Map.get(params, "max_id", Map.get(params, :max_id)) do - nil -> query - max_id -> where(query, [n], n.id < ^max_id) + nil -> cursor_filters + max_id -> [{:lt, max_id} | cursor_filters] end - {query, :asc} + {:asc, cursor_filters} since_id = Map.get(params, "since_id", Map.get(params, :since_id)) -> - {where(query, [n], n.id > ^since_id), :desc} + {:desc, [{:gt, since_id}]} max_id = Map.get(params, "max_id", Map.get(params, :max_id)) -> - {where(query, [n], n.id < ^max_id), :desc} + {:desc, [{:lt, max_id}]} true -> - {query, :desc} + {:desc, []} end end @@ -150,41 +192,190 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPI do |> min(80) end - defp take_grouped_page_notifications(notifications, grouped_types, group_limit) do - {notifications, _group_keys} = - Enum.reduce_while(notifications, {[], MapSet.new()}, fn notification, - {notifications, group_keys} -> - group_key = Notification.group_key(notification, grouped_types) - - cond do - MapSet.member?(group_keys, group_key) -> - {:cont, {[notification | notifications], group_keys}} - - MapSet.size(group_keys) < group_limit -> - {:cont, {[notification | notifications], MapSet.put(group_keys, group_key)}} - - true -> - {:halt, {notifications, group_keys}} - end - end) - - Enum.reverse(notifications) - end - def unread_notification_group_count(user, params \\ %{}) do - grouped_types = Map.get(params, "grouped_types", Map.get(params, :grouped_types)) + grouped_types = + params + |> Map.get("grouped_types", Map.get(params, :grouped_types)) + |> Notification.normalize_grouped_types() + limit = unread_count_limit(params) user |> notifications_query(params) - |> where([n], n.seen == false) - |> order_by([n], desc: n.id) - |> limit(^limit) - |> Repo.all() - |> Notification.group_notifications(grouped_types) + # The grouped API docs define unread by the notifications marker, not by Pleroma's per-row + # seen flag used by the v1 unread count. Keep this marker-based for Mastodon clients. + |> restrict_after_marker(notification_marker_last_read_id(user)) + |> notification_group_rows(grouped_types, limit, :desc, []) |> length() end + defp notification_group_rows(query, grouped_types, group_limit, :desc, cursor_filters) do + query + |> notification_group_keyed_query(grouped_types) + |> group_by([n], n.group_key) + |> apply_group_cursor_filters(cursor_filters) + |> select([n], %{ + group_key: n.group_key, + representative_id: max(n.id), + notifications_count: count(n.id), + page_min_id: min(n.id), + page_max_id: max(n.id), + latest_page_notification_at: max(n.inserted_at) + }) + |> order_by([n], desc: max(n.id)) + |> limit(^group_limit) + |> Repo.all() + end + + defp notification_group_rows(query, grouped_types, group_limit, :asc, cursor_filters) do + query + |> notification_group_keyed_query(grouped_types) + |> group_by([n], n.group_key) + |> apply_group_cursor_filters(cursor_filters) + |> select([n], %{ + group_key: n.group_key, + representative_id: max(n.id), + notifications_count: count(n.id), + page_min_id: min(n.id), + page_max_id: max(n.id), + latest_page_notification_at: max(n.inserted_at) + }) + |> order_by([n], asc: max(n.id)) + |> limit(^group_limit) + |> Repo.all() + end + + defp apply_group_cursor_filters(query, []), do: query + + defp apply_group_cursor_filters(query, [{:gt, id} | rest]) do + query + |> having([n], max(n.id) > ^id) + |> apply_group_cursor_filters(rest) + end + + defp apply_group_cursor_filters(query, [{:lt, id} | rest]) do + query + |> having([n], max(n.id) < ^id) + |> apply_group_cursor_filters(rest) + end + + defp notification_group_keyed_query(query, grouped_types) do + query + |> exclude(:preload) + |> select([n], %{ + id: n.id, + inserted_at: n.inserted_at, + group_key: + fragment( + "CASE WHEN ? IS NOT NULL AND ?::text = ANY(?) THEN ? ELSE 'ungrouped-' || ?::text END", + n.group_key, + n.type, + type(^grouped_types, {:array, :string}), + n.group_key, + n.id + ) + }) + |> subquery() + end + + defp representative_notifications(_query, []), do: [] + + defp representative_notifications(query, group_rows) do + representative_ids = Enum.map(group_rows, & &1.representative_id) + + notifications_by_id = + query + |> where([n], n.id in ^representative_ids) + |> Repo.all() + |> Map.new(&{to_string(&1.id), &1}) + + group_rows + |> Enum.map(&Map.get(notifications_by_id, to_string(&1.representative_id))) + |> Enum.filter(& &1) + end + + defp notification_group_counts(group_rows) do + Map.new(group_rows, &{&1.group_key, &1.notifications_count}) + end + + defp notification_group_bounds(group_rows) do + Map.new(group_rows, fn row -> + {row.group_key, + %{ + page_min_id: row.page_min_id, + page_max_id: row.page_max_id, + latest_page_notification_at: row.latest_page_notification_at + }} + end) + end + + defp notification_group_sample(user, _params, "ungrouped-" <> notification_id) do + case Notification.get(user, notification_id) do + {:ok, notification} -> [notification] + _ -> [] + end + end + + defp notification_group_sample(user, params, group_key) do + user + |> notification_group_query(group_key, params) + |> order_by([n], desc: n.id) + |> limit(^@notification_group_sample_limit) + |> Repo.all() + end + + defp notification_group_metadata(user, "ungrouped-" <> notification_id, _params) do + case Notification.get(user, notification_id) do + {:ok, notification} -> + %{ + notifications_count: 1, + page_min_id: notification.id, + page_max_id: notification.id, + latest_page_notification_at: notification.inserted_at + } + + _ -> + nil + end + end + + defp notification_group_metadata(user, group_key, params) do + user + |> notification_group_query(group_key, params) + |> exclude(:preload) + |> select([n], %{ + notifications_count: count(n.id), + page_min_id: min(n.id), + page_max_id: max(n.id), + latest_page_notification_at: max(n.inserted_at) + }) + |> Repo.one() + |> case do + %{notifications_count: 0} -> nil + metadata -> metadata + end + end + + defp notification_group_query(user, group_key, params) do + user + |> notifications_query(params) + |> where([n], n.group_key == ^group_key) + end + + defp notification_marker_last_read_id(user) do + Marker + |> where([m], m.user_id == ^user.id and m.timeline == "notifications") + |> select([m], m.last_read_id) + |> Repo.one() + end + + defp restrict_after_marker(query, last_read_id) + when is_binary(last_read_id) and last_read_id != "" do + where(query, [n], n.id > ^last_read_id) + end + + defp restrict_after_marker(query, _last_read_id), do: query + defp notifications_query(user, params) do options = notification_options(user, params) diff --git a/lib/pleroma/web/mastodon_api/views/notification_view.ex b/lib/pleroma/web/mastodon_api/views/notification_view.ex index 466073853..33d4b2924 100644 --- a/lib/pleroma/web/mastodon_api/views/notification_view.ex +++ b/lib/pleroma/web/mastodon_api/views/notification_view.ex @@ -90,6 +90,8 @@ defmodule Pleroma.Web.MastodonAPI.NotificationView do ) do grouped_types = Notification.normalize_grouped_types(opts[:grouped_types]) notification_group_counts = Map.get(opts, :notification_group_counts, %{}) + notification_group_bounds = Map.get(opts, :notification_group_bounds, %{}) + include_page_metadata = Map.get(opts, :include_page_metadata, true) statuses = notification_groups @@ -110,7 +112,14 @@ defmodule Pleroma.Web.MastodonAPI.NotificationView do notification_groups: Enum.map( notification_groups, - &render_group(&1, reading_user, grouped_types, notification_group_counts) + &render_group( + &1, + reading_user, + grouped_types, + notification_group_counts, + notification_group_bounds, + include_page_metadata + ) ) } end @@ -184,27 +193,50 @@ defmodule Pleroma.Web.MastodonAPI.NotificationView do [%Notification{} = notification | _] = notifications, _reading_user, grouped_types, - notification_group_counts + notification_group_counts, + notification_group_bounds, + include_page_metadata ) do latest_notification = List.first(notifications) oldest_notification = List.last(notifications) status_activity = status_activity_for_group(notifications, grouped_types) group_key = Notification.group_key(notification, grouped_types) + bounds = Map.get(notification_group_bounds, group_key, %{}) response = %{ group_key: group_key, notifications_count: Map.get(notification_group_counts, group_key, length(notifications)), type: notification.type, - most_recent_notification_id: to_string(latest_notification.id), - page_min_id: to_string(oldest_notification.id), - page_max_id: to_string(latest_notification.id), - latest_page_notification_at: CommonAPI.Utils.to_masto_date(latest_notification.inserted_at), + most_recent_notification_id: + bounds + |> Map.get(:page_max_id, latest_notification.id) + |> to_string(), sample_account_ids: notifications |> notification_actors() |> Enum.map(&to_string(&1.id)) } + response = + if include_page_metadata do + Map.merge(response, %{ + page_min_id: + bounds + |> Map.get(:page_min_id, oldest_notification.id) + |> to_string(), + page_max_id: + bounds + |> Map.get(:page_max_id, latest_notification.id) + |> to_string(), + latest_page_notification_at: + bounds + |> Map.get(:latest_page_notification_at, latest_notification.inserted_at) + |> CommonAPI.Utils.to_masto_date() + }) + else + response + end + if status_activity do Map.put(response, :status_id, to_string(status_activity.id)) else diff --git a/priv/repo/migrations/20260523070000_add_group_key_to_notifications.exs b/priv/repo/migrations/20260523070000_add_group_key_to_notifications.exs new file mode 100644 index 000000000..7c7d529e0 --- /dev/null +++ b/priv/repo/migrations/20260523070000_add_group_key_to_notifications.exs @@ -0,0 +1,15 @@ +defmodule Pleroma.Repo.Migrations.AddGroupKeyToNotifications do + use Ecto.Migration + + def change do + alter table(:notifications) do + add(:group_key, :string) + end + + create_if_not_exists( + index(:notifications, [:user_id, :group_key, "id desc nulls last"], + where: "group_key IS NOT NULL" + ) + ) + end +end diff --git a/test/pleroma/web/mastodon_api/controllers/notification_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/notification_controller_test.exs index 1d3a9e966..e9c521d6b 100644 --- a/test/pleroma/web/mastodon_api/controllers/notification_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/notification_controller_test.exs @@ -587,10 +587,10 @@ defmodule Pleroma.Web.MastodonAPI.NotificationControllerTest do {:ok, _} = CommonAPI.favorite(status.id, other_user1) {:ok, _} = CommonAPI.favorite(status.id, other_user2) - notification_ids = - Notification - |> Repo.all() - |> Enum.map(&to_string(&1.id)) + notifications = Repo.all(Notification) + notification_ids = Enum.map(notifications, &to_string(&1.id)) + assert [persisted_group_key] = notifications |> Enum.map(& &1.group_key) |> Enum.uniq() + assert is_binary(persisted_group_key) result = conn @@ -613,6 +613,7 @@ defmodule Pleroma.Web.MastodonAPI.NotificationControllerTest do assert group["latest_page_notification_at"] assert Enum.sort(group["sample_account_ids"]) == Enum.sort([other_user1.id, other_user2.id]) group_key = group["group_key"] + assert group_key == persisted_group_key assert [%{"group_key" => ^group_key}, %{"group_key" => ^group_key}] = conn @@ -626,6 +627,9 @@ defmodule Pleroma.Web.MastodonAPI.NotificationControllerTest do assert shown_group["group_key"] == group_key assert shown_group["notifications_count"] == 2 + refute Map.has_key?(shown_group, "page_min_id") + refute Map.has_key?(shown_group, "page_max_id") + refute Map.has_key?(shown_group, "latest_page_notification_at") end test "round-trips ungrouped group keys when grouped_types excludes a type" do @@ -648,6 +652,48 @@ defmodule Pleroma.Web.MastodonAPI.NotificationControllerTest do |> json_response_and_validate_schema(200) end + test "keeps legacy notifications without persisted group keys ungrouped" do + %{user: user, conn: conn} = oauth_access(["read:notifications"]) + other_user1 = insert(:user) + other_user2 = insert(:user) + + {:ok, status} = CommonAPI.post(user, %{status: "hello"}) + {:ok, favorite1} = CommonAPI.favorite(status.id, other_user1) + {:ok, favorite2} = CommonAPI.favorite(status.id, other_user2) + + notification_ids = + [favorite1, favorite2] + |> Enum.map(&get_notification_id_by_activity/1) + + Repo.update_all(Notification, set: [group_key: nil]) + + expected_group_keys = Enum.map(notification_ids, &"ungrouped-#{&1}") + + assert [%{"group_key" => group_key1}, %{"group_key" => group_key2}] = + conn + |> get("/api/v1/notifications") + |> json_response_and_validate_schema(200) + + assert Enum.sort([group_key1, group_key2]) == Enum.sort(expected_group_keys) + + %{"notification_groups" => groups} = + conn + |> get("/api/v2/notifications") + |> json_response_and_validate_schema(200) + + assert Enum.sort(Enum.map(groups, & &1["group_key"])) == Enum.sort(expected_group_keys) + assert Enum.all?(groups, &(&1["notifications_count"] == 1)) + + group_key = List.first(expected_group_keys) + + assert %{ + "notification_groups" => [%{"group_key" => ^group_key, "notifications_count" => 1}] + } = + conn + |> get("/api/v2/notifications/#{group_key}") + |> json_response_and_validate_schema(200) + end + test "paginates notification groups instead of raw notifications" do %{user: user, conn: conn} = oauth_access(["read:notifications"]) other_user1 = insert(:user) @@ -674,6 +720,44 @@ defmodule Pleroma.Web.MastodonAPI.NotificationControllerTest do ] = groups end + test "uses group-level cursors when paginating notification groups" do + %{user: user, conn: conn} = oauth_access(["read:notifications"]) + other_user1 = insert(:user) + other_user2 = insert(:user) + other_user3 = insert(:user) + + {:ok, older_status} = CommonAPI.post(user, %{status: "older"}) + {:ok, _} = CommonAPI.favorite(older_status.id, other_user3) + + {:ok, newer_status} = CommonAPI.post(user, %{status: "newer"}) + {:ok, _} = CommonAPI.favorite(newer_status.id, other_user1) + {:ok, _} = CommonAPI.favorite(newer_status.id, other_user2) + older_status_id = older_status.id + newer_status_id = newer_status.id + + assert %{ + "notification_groups" => [ + %{ + "status_id" => ^newer_status_id, + "notifications_count" => 2, + "most_recent_notification_id" => cursor + } + ] + } = + conn + |> get("/api/v2/notifications?limit=1") + |> json_response_and_validate_schema(200) + + assert %{ + "notification_groups" => [ + %{"status_id" => ^older_status_id, "notifications_count" => 1} + ] + } = + conn + |> get("/api/v2/notifications?limit=1&max_id=#{cursor}") + |> json_response_and_validate_schema(200) + end + test "returns total notification count for a partially represented group" do %{user: user, conn: conn} = oauth_access(["read:notifications"]) other_user1 = insert(:user) @@ -715,27 +799,78 @@ defmodule Pleroma.Web.MastodonAPI.NotificationControllerTest do |> json_response_and_validate_schema(200) end + test "counts unread notification groups newer than the notification marker" do + %{user: user, conn: conn} = oauth_access(["read:notifications"]) + other_user = insert(:user) + mentioner = insert(:user) + + {:ok, status} = CommonAPI.post(user, %{status: "older"}) + {:ok, favorite_activity} = CommonAPI.favorite(status.id, other_user) + marker_id = get_notification_id_by_activity(favorite_activity) + + {:ok, _} = CommonAPI.post(mentioner, %{status: "newer @#{user.nickname}"}) + + {:ok, _} = + Pleroma.Marker.upsert(user, %{"notifications" => %{"last_read_id" => marker_id}}) + + assert %{"count" => 1} = + conn + |> get("/api/v2/notifications/unread_count") + |> json_response_and_validate_schema(200) + end + + test "does not preserve stale cursor params in grouped notification link headers" do + %{user: user, conn: conn} = oauth_access(["read:notifications"]) + other_user = insert(:user) + + {:ok, activity1} = CommonAPI.post(other_user, %{status: "one @#{user.nickname}"}) + {:ok, _activity2} = CommonAPI.post(other_user, %{status: "two @#{user.nickname}"}) + {:ok, activity3} = CommonAPI.post(other_user, %{status: "three @#{user.nickname}"}) + + notification1_id = get_notification_id_by_activity(activity1) + notification3_id = get_notification_id_by_activity(activity3) + + conn = get(conn, "/api/v2/notifications?since_id=#{notification1_id}&limit=1") + + assert [link_header] = get_resp_header(conn, "link") + assert link_header =~ ~r/max_id=#{notification3_id}/ + refute link_header =~ "since_id=" + end + test "lists accounts from and dismisses a notification group" do %{user: user, conn: conn} = oauth_access(["read:notifications", "write:notifications"]) - other_user1 = insert(:user) - other_user2 = insert(:user) + other_users = insert_list(9, :user) {:ok, status} = CommonAPI.post(user, %{status: "hello"}) - {:ok, _} = CommonAPI.favorite(status.id, other_user1) - {:ok, _} = CommonAPI.favorite(status.id, other_user2) - %{"notification_groups" => [%{"group_key" => group_key}]} = + Enum.each(other_users, fn other_user -> + {:ok, _} = CommonAPI.favorite(status.id, other_user) + end) + + %{ + "notification_groups" => [ + %{ + "group_key" => group_key, + "notifications_count" => 9, + "sample_account_ids" => sample_account_ids + } + ] + } = conn |> get("/api/v2/notifications") |> json_response_and_validate_schema(200) + assert length(sample_account_ids) == 8 + + %{conn: read_conn} = oauth_access(["read:notifications"], user: user) + account_ids = - conn + read_conn |> get("/api/v2/notifications/#{group_key}/accounts") |> json_response_and_validate_schema(200) |> Enum.map(& &1["id"]) - assert Enum.sort(account_ids) == Enum.sort([other_user1.id, other_user2.id]) + assert Enum.sort(account_ids) == Enum.sort(Enum.map(other_users, & &1.id)) assert %{} = conn diff --git a/test/pleroma/web/mastodon_api/views/notification_view_test.exs b/test/pleroma/web/mastodon_api/views/notification_view_test.exs index aa11ad682..2d0801589 100644 --- a/test/pleroma/web/mastodon_api/views/notification_view_test.exs +++ b/test/pleroma/web/mastodon_api/views/notification_view_test.exs @@ -98,6 +98,7 @@ defmodule Pleroma.Web.MastodonAPI.NotificationViewTest do {:ok, favorite_activity} = CommonAPI.favorite(create_activity.id, another_user) {:ok, [notification]} = Notification.create_notifications(favorite_activity) create_activity = Activity.get_by_id(create_activity.id) + assert is_binary(notification.group_key) expected = %{ id: to_string(notification.id), @@ -119,6 +120,7 @@ defmodule Pleroma.Web.MastodonAPI.NotificationViewTest do {:ok, reblog_activity} = CommonAPI.repeat(create_activity.id, another_user) {:ok, [notification]} = Notification.create_notifications(reblog_activity) reblog_activity = Activity.get_by_id(create_activity.id) + assert is_binary(notification.group_key) expected = %{ id: to_string(notification.id), @@ -138,6 +140,7 @@ defmodule Pleroma.Web.MastodonAPI.NotificationViewTest do followed = insert(:user) {:ok, followed, follower, _activity} = CommonAPI.follow(followed, follower) notification = Notification |> Repo.one() |> Repo.preload(:activity) + assert is_binary(notification.group_key) expected = %{ id: to_string(notification.id), From 1db2b99e4873a5196cbbb393c294419e10045a15 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sat, 23 May 2026 14:00:17 +0400 Subject: [PATCH 076/183] Run group key migration before notification backfill --- ...ions.exs => 20200602125217_add_group_key_to_notifications.exs} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename priv/repo/migrations/{20260523070000_add_group_key_to_notifications.exs => 20200602125217_add_group_key_to_notifications.exs} (100%) diff --git a/priv/repo/migrations/20260523070000_add_group_key_to_notifications.exs b/priv/repo/migrations/20200602125217_add_group_key_to_notifications.exs similarity index 100% rename from priv/repo/migrations/20260523070000_add_group_key_to_notifications.exs rename to priv/repo/migrations/20200602125217_add_group_key_to_notifications.exs From a2a23b327b9c32420b5a6aed4198073b4f363864 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sat, 23 May 2026 14:27:11 +0400 Subject: [PATCH 077/183] Keep cursor cleanup scoped to grouped notifications --- lib/pleroma/web/controller_helper.ex | 10 +++++++--- .../controllers/notification_controller.ex | 4 ++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/web/controller_helper.ex b/lib/pleroma/web/controller_helper.ex index eecd1774f..0c85e79f9 100644 --- a/lib/pleroma/web/controller_helper.ex +++ b/lib/pleroma/web/controller_helper.ex @@ -56,7 +56,11 @@ defmodule Pleroma.Web.ControllerHelper do # TODO: Only fetch the params from open_api_spex when everything is converted @id_keys Pagination.page_keys() -- ["limit", "order"] @id_key_atoms Enum.map(@id_keys, &String.to_atom/1) + @drop_id_params_key :drop_id_params defp build_pagination_fields(conn, min_id, max_id, extra_params, order) do + drop_id_params? = Map.get(extra_params, @drop_id_params_key, false) + extra_params = Map.delete(extra_params, @drop_id_params_key) + path_param_keys = conn.path_params |> Map.keys() @@ -70,9 +74,9 @@ defmodule Pleroma.Web.ControllerHelper do end |> Map.drop(path_param_keys) |> Map.merge(extra_params) - # OpenApiSpex casts params to atoms while uncast conn params are string-keyed. Drop both so - # generated links replace pagination cursors instead of preserving stale request cursors. - |> Map.drop(@id_keys ++ @id_key_atoms) + # Some OpenApiSpex routes cast cursor params to atoms. Keep the historical default for + # existing endpoints, but allow grouped notifications to opt out of stale request cursors. + |> Map.drop(@id_keys ++ if(drop_id_params?, do: @id_key_atoms, else: [])) {{next_id, nid}, {prev_id, pid}} = if order == :desc, diff --git a/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex b/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex index 2cc31db0c..8a5f8e7b6 100644 --- a/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex @@ -5,7 +5,7 @@ defmodule Pleroma.Web.MastodonAPI.NotificationController do use Pleroma.Web, :controller - import Pleroma.Web.ControllerHelper, only: [add_link_headers: 2] + import Pleroma.Web.ControllerHelper, only: [add_link_headers: 2, add_link_headers: 3] alias Pleroma.Notification alias Pleroma.User @@ -184,7 +184,7 @@ defmodule Pleroma.Web.MastodonAPI.NotificationController do MastodonAPI.get_grouped_notification_page(user, params) conn - |> add_link_headers(page_notifications) + |> add_link_headers(page_notifications, %{drop_id_params: true}) |> render("grouped_index.json", notification_groups: notification_groups, notification_group_counts: notification_group_counts, From a84d16831f788d82ec77eceb51debf359cf397d2 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Mon, 25 May 2026 18:09:18 +0400 Subject: [PATCH 078/183] Fix account favourites ordering --- changelog.d/account-favourites-order.fix | 1 + lib/pleroma/web/activity_pub/activity_pub.ex | 17 +++- .../controllers/account_controller_test.exs | 84 +++++++++++++++++-- 3 files changed, 90 insertions(+), 12 deletions(-) create mode 100644 changelog.d/account-favourites-order.fix diff --git a/changelog.d/account-favourites-order.fix b/changelog.d/account-favourites-order.fix new file mode 100644 index 000000000..94e27e499 --- /dev/null +++ b/changelog.d/account-favourites-order.fix @@ -0,0 +1 @@ +Fixed `/api/v1/pleroma/accounts/:id/favourites` ordering and pagination to use the favourite date. diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 1e8bfe3dd..be7b63725 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -537,9 +537,12 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do # and extra sorting on "activities.id DESC NULLS LAST" would worse the query plan opts = Map.put(opts, :skip_extra_order, true) - Pagination.fetch_paginated(query, opts, pagination) + Pagination.fetch_paginated(query, opts, pagination, pagination_binding(opts)) end + defp pagination_binding(%{favorited_by: _}), do: :favorited_activity + defp pagination_binding(_), do: nil + def fetch_activities(recipients, opts \\ %{}, pagination \\ :keyset) do list_memberships = Pleroma.List.memberships(opts[:user]) @@ -786,6 +789,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp restrict_since(query, %{since_id: ""}), do: query + defp restrict_since(query, %{favorited_by: _, since_id: _}), do: query + defp restrict_since(query, %{since_id: since_id}) do from(activity in query, where: activity.id > ^since_id) end @@ -1013,8 +1018,14 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp restrict_favorited_by(query, %{favorited_by: ap_id}) do from( - [_activity, object] in query, - where: fragment("(?)->'likes' \\? (?)", object.data, ^ap_id) + [activity, object: object] in query, + join: favorited_activity in Activity, + as: :favorited_activity, + on: + favorited_activity.actor == ^ap_id and + fragment("?->>'type' = ?", favorited_activity.data, "Like") and + fragment("associated_object_id(?) = (?)->>'id'", favorited_activity.data, object.data), + select: %Activity{activity | pagination_id: favorited_activity.id} ) end diff --git a/test/pleroma/web/pleroma_api/controllers/account_controller_test.exs b/test/pleroma/web/pleroma_api/controllers/account_controller_test.exs index 38a928f30..e93b277cc 100644 --- a/test/pleroma/web/pleroma_api/controllers/account_controller_test.exs +++ b/test/pleroma/web/pleroma_api/controllers/account_controller_test.exs @@ -105,6 +105,28 @@ defmodule Pleroma.Web.PleromaAPI.AccountControllerTest do assert length(response) == 1 end + test "orders statuses by favorite date", %{ + conn: conn, + user: user + } do + other_user = insert(:user) + + {:ok, first_post} = CommonAPI.post(other_user, %{status: "first post"}) + {:ok, second_post} = CommonAPI.post(other_user, %{status: "second post"}) + {:ok, third_post} = CommonAPI.post(other_user, %{status: "third post"}) + + {:ok, _} = CommonAPI.favorite(third_post.id, user) + {:ok, _} = CommonAPI.favorite(first_post.id, user) + {:ok, _} = CommonAPI.favorite(second_post.id, user) + + response = + conn + |> get("/api/v1/pleroma/accounts/#{user.id}/favourites") + |> json_response_and_validate_schema(:ok) + + assert Enum.map(response, & &1["id"]) == [second_post.id, first_post.id, third_post.id] + end + test "returns favorited DM only when user is logged in and he is one of recipients", %{ current_user: current_user, user: user @@ -164,23 +186,67 @@ defmodule Pleroma.Web.PleromaAPI.AccountControllerTest do } do activities = insert_list(10, :note_activity) - Enum.each(activities, fn activity -> - CommonAPI.favorite(activity.id, user) - end) + favorite_pairs = + Enum.map(activities, fn activity -> + {:ok, favorite} = CommonAPI.favorite(activity.id, user) + {favorite, activity} + end) + |> Enum.sort_by(fn {favorite, _activity} -> FlakeId.from_string(favorite.id) end) - third_activity = Enum.at(activities, 2) - seventh_activity = Enum.at(activities, 6) + {third_favorite, _} = Enum.at(favorite_pairs, 2) + {seventh_favorite, _} = Enum.at(favorite_pairs, 6) + + expected_ids = + favorite_pairs + |> Enum.slice(3, 3) + |> Enum.reverse() + |> Enum.map(fn {_favorite, activity} -> activity.id end) response = conn |> get( - "/api/v1/pleroma/accounts/#{user.id}/favourites?since_id=#{third_activity.id}&max_id=#{seventh_activity.id}" + "/api/v1/pleroma/accounts/#{user.id}/favourites?since_id=#{third_favorite.id}&max_id=#{seventh_favorite.id}" ) |> json_response_and_validate_schema(:ok) - assert length(response) == 3 - refute third_activity in response - refute seventh_activity in response + assert Enum.map(response, & &1["id"]) == expected_ids + end + + test "paginates favorites using min_id and limit", %{ + conn: conn, + user: user + } do + activities = insert_list(10, :note_activity) + + favorite_pairs = + Enum.map(activities, fn activity -> + {:ok, favorite} = CommonAPI.favorite(activity.id, user) + {favorite, activity} + end) + |> Enum.sort_by(fn {favorite, _activity} -> FlakeId.from_string(favorite.id) end) + + {third_favorite, _} = Enum.at(favorite_pairs, 2) + {fourth_favorite, _} = Enum.at(favorite_pairs, 3) + {fifth_favorite, _} = Enum.at(favorite_pairs, 4) + + expected_ids = + favorite_pairs + |> Enum.slice(3, 2) + |> Enum.reverse() + |> Enum.map(fn {_favorite, activity} -> activity.id end) + + conn = + get( + conn, + "/api/v1/pleroma/accounts/#{user.id}/favourites?min_id=#{third_favorite.id}&limit=2" + ) + + response = json_response_and_validate_schema(conn, :ok) + + assert Enum.map(response, & &1["id"]) == expected_ids + assert [link_header] = get_resp_header(conn, "link") + assert link_header =~ "max_id=#{fourth_favorite.id}" + assert link_header =~ "min_id=#{fifth_favorite.id}" end test "limits favorites using limit parameter", %{ From cd28c2089c2fddce4e480bd097830ce22f923c47 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Mon, 25 May 2026 18:45:47 +0400 Subject: [PATCH 079/183] Fix ParadeDB search indexing --- changelog.d/paradedb-search.add | 2 +- config/config.exs | 2 +- docs/configuration/search.md | 8 +- lib/pleroma/search.ex | 74 +++++++++------ .../search/{paradedb.ex => parade_db.ex} | 15 +-- .../search/{paradedb => parade_db}/client.ex | 0 .../search/{paradedb => parade_db}/repo.ex | 0 .../activity_pub/mrf/anti_followbot_policy.ex | 6 +- .../web/activity_pub/mrf/simple_policy.ex | 2 +- .../web/mastodon_api/views/status_view.ex | 4 +- lib/pleroma/workers/mailer_worker.ex | 4 +- test/pleroma/search/meilisearch_test.exs | 2 +- ...est.exs => parade_db_integration_test.exs} | 0 .../{paradedb_test.exs => parade_db_test.exs} | 20 ++-- test/pleroma/search_test.exs | 94 +++++++++++++++++-- 15 files changed, 175 insertions(+), 58 deletions(-) rename lib/pleroma/search/{paradedb.ex => parade_db.ex} (96%) rename lib/pleroma/search/{paradedb => parade_db}/client.ex (100%) rename lib/pleroma/search/{paradedb => parade_db}/repo.ex (100%) rename test/pleroma/search/{paradedb_integration_test.exs => parade_db_integration_test.exs} (100%) rename test/pleroma/search/{paradedb_test.exs => parade_db_test.exs} (94%) diff --git a/changelog.d/paradedb-search.add b/changelog.d/paradedb-search.add index 00a6e2361..f72ebca14 100644 --- a/changelog.d/paradedb-search.add +++ b/changelog.d/paradedb-search.add @@ -1 +1 @@ -Add ParadeDB (pg_search) as an optional external search backend. +Add ParadeDB (pg_search) as an optional external search backend, indexing status content, content warnings, and attachment descriptions. diff --git a/config/config.exs b/config/config.exs index 2e405413c..3c308f093 100644 --- a/config/config.exs +++ b/config/config.exs @@ -946,7 +946,7 @@ config :pleroma, Pleroma.Search.ParadeDB, fuzzy_distance: 0 config :pleroma, Pleroma.Search.ParadeDB.Repo, - pool_size: 10, + pool_size: 2, prepare: :unnamed config :pleroma, Pleroma.Application, diff --git a/docs/configuration/search.md b/docs/configuration/search.md index c0d3f8462..1d5056ddf 100644 --- a/docs/configuration/search.md +++ b/docs/configuration/search.md @@ -85,7 +85,7 @@ You will see a "Default Admin API Key", this is the key you actually put into yo ### Initial indexing -After setting up the configuration, you'll want to index all of your already existing posts. Only public posts are indexed. You'll only +After setting up the configuration, you'll want to index all of your already existing posts. Only public/unlisted posts are indexed. You'll only have to do it one time, but it might take a while, depending on the amount of posts your instance has seen. This is also a fairly RAM consuming process for `meilisearch`, and it will take a lot of RAM when running if you have a lot of posts (seems to be around 5G for ~1.2 million posts while idle and up to 7G while indexing initially, but your experience may be different). @@ -153,8 +153,10 @@ external search backend by pointing Pleroma at a separate Postgres instance with Pleroma will maintain a small `pleroma_search_documents` table in that database (via the existing search indexing queue) and run search queries against it. +The indexed text includes status content, content warnings, and attachment descriptions. To use it, set the search module to `Pleroma.Search.ParadeDB` and configure the ParadeDB database URL: +These settings must be in static/runtime config (or `PARADEDB_DATABASE_URL`) so the dedicated ParadeDB Repo is started with the application. > config :pleroma, Pleroma.Search, module: Pleroma.Search.ParadeDB > @@ -162,6 +164,10 @@ To use it, set the search module to `Pleroma.Search.ParadeDB` and configure the > url: System.get_env("PARADEDB_DATABASE_URL") || "postgres://postgres:postgres@127.0.0.1:5432/paradedb", > table: "pleroma_search_documents", > fuzzy_distance: 0 +> +> config :pleroma, Pleroma.Search.ParadeDB.Repo, +> pool_size: 2, +> prepare: :unnamed Then, create the table and BM25 index once: diff --git a/lib/pleroma/search.ex b/lib/pleroma/search.ex index 9cd2768c4..cd4907026 100644 --- a/lib/pleroma/search.ex +++ b/lib/pleroma/search.ex @@ -5,13 +5,12 @@ defmodule Pleroma.Search do alias Pleroma.Workers.SearchIndexingWorker @spec add_to_index(Activity.t()) :: {:ok, Oban.Job.t() | :noop} | {:error, Oban.Job.changeset()} - def add_to_index(%Activity{id: activity_id, object: %Object{} = object} = activity) do - with {_, true} <- {:indexable, indexable?(activity)}, - {_, "public"} <- {:visibility, Visibility.get_visibility(object)} do + def add_to_index(%Activity{id: activity_id, object: %Object{}} = activity) do + if indexable?(activity) do SearchIndexingWorker.new(%{"op" => "add_to_index", "activity" => activity_id}) |> Oban.insert() else - _ -> {:ok, :noop} + {:ok, :noop} end end @@ -41,40 +40,59 @@ defmodule Pleroma.Search do def object_to_search_data(%Object{} = object) do data = object.data - content_str = - case data["content"] do - [nil | rest] -> to_string(rest) - str -> str - end - content = - with {:ok, scrubbed} <- - FastSanitize.Sanitizer.scrub(content_str, Pleroma.HTML.Scrubber.SearchIndexing), - trimmed <- String.trim(scrubbed) do - trimmed - end - - # Make sure we have a non-empty string - if content != "" do - {:ok, published, _} = DateTime.from_iso8601(data["published"]) + data + |> search_texts() + |> Enum.map(&sanitize_search_text/1) + |> Enum.reject(&(&1 in ["", "."])) + |> Enum.join(" ") + with true <- content != "", + published when is_binary(published) <- data["published"], + {:ok, published, _} <- DateTime.from_iso8601(published) do %{ id: object.id, content: content, ap: data["id"], published: published |> DateTime.to_unix() } + else + _ -> nil end end - defp indexable?(%Activity{ - data: %{"type" => "Create"}, - object: %Object{ - data: %{"content" => content, "published" => published, "type" => "Note"} - } - }) - when not is_nil(content) and content not in ["", "."] and not is_nil(published), - do: true + def indexable?(%Activity{ + data: %{"type" => "Create"}, + object: %Object{data: %{"published" => published, "type" => "Note"}} = object + }) + when not is_nil(published) do + Visibility.get_visibility(object) in ["public", "unlisted"] and + not is_nil(object_to_search_data(object)) + end - defp indexable?(_), do: false + def indexable?(_), do: false + + defp search_texts(data) do + [data["content"], data["summary"] | attachment_names(data["attachment"])] + end + + defp attachment_names(attachments) when is_list(attachments) do + Enum.map(attachments, fn + %{"name" => name} -> name + _ -> nil + end) + end + + defp attachment_names(_), do: [] + + defp sanitize_search_text([nil | rest]), do: sanitize_search_text(to_string(rest)) + + defp sanitize_search_text(text) when is_binary(text) do + case FastSanitize.Sanitizer.scrub(text, Pleroma.HTML.Scrubber.SearchIndexing) do + {:ok, scrubbed} -> String.trim(scrubbed) + _ -> "" + end + end + + defp sanitize_search_text(_), do: "" end diff --git a/lib/pleroma/search/paradedb.ex b/lib/pleroma/search/parade_db.ex similarity index 96% rename from lib/pleroma/search/paradedb.ex rename to lib/pleroma/search/parade_db.ex index 20be5b6f9..ee00e74af 100644 --- a/lib/pleroma/search/paradedb.ex +++ b/lib/pleroma/search/parade_db.ex @@ -66,9 +66,10 @@ defmodule Pleroma.Search.ParadeDB do @impl true def drop_index do - with {:ok, _} <- client_impl().query("DROP TABLE IF EXISTS #{table()}", []) do - :ok - else + case client_impl().query("DROP TABLE IF EXISTS #{table()}", []) do + {:ok, _} -> + :ok + {:error, error} -> Logger.error("ParadeDB drop_index failed: #{inspect(error)}") {:error, error} @@ -80,9 +81,9 @@ defmodule Pleroma.Search.ParadeDB do @impl true def add_to_index(activity) do - maybe_search_data = Search.object_to_search_data(activity.object) - - if activity.data["type"] == "Create" and maybe_search_data do + with true <- Search.indexable?(activity), + maybe_search_data when not is_nil(maybe_search_data) <- + Search.object_to_search_data(activity.object) do dumped_activity_id = dump_activity_id(activity.id) actor_ap_id = activity.data["actor"] published_at = DateTime.from_unix!(maybe_search_data.published) @@ -122,7 +123,7 @@ defmodule Pleroma.Search.ParadeDB do end end else - :ok + _ -> :ok end end diff --git a/lib/pleroma/search/paradedb/client.ex b/lib/pleroma/search/parade_db/client.ex similarity index 100% rename from lib/pleroma/search/paradedb/client.ex rename to lib/pleroma/search/parade_db/client.ex diff --git a/lib/pleroma/search/paradedb/repo.ex b/lib/pleroma/search/parade_db/repo.ex similarity index 100% rename from lib/pleroma/search/paradedb/repo.ex rename to lib/pleroma/search/parade_db/repo.ex diff --git a/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex b/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex index 8ea61aec2..d0320ce49 100644 --- a/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex @@ -24,7 +24,11 @@ defmodule Pleroma.Web.ActivityPub.MRF.AntiFollowbotPolicy do defp score_displayname("fedibot"), do: 1.0 defp score_displayname(_), do: 0.0 - defp determine_if_followbot(%User{nickname: nickname, name: displayname, actor_type: actor_type}) do + defp determine_if_followbot(%User{ + nickname: nickname, + name: displayname, + actor_type: actor_type + }) do # nickname will be a binary string except when following a relay nick_score = if is_binary(nickname) do diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex index a97e8db7b..93f0616ec 100644 --- a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex @@ -108,7 +108,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do end defp intersection(list1, list2) do - list1 -- list1 -- list2 + list1 -- (list1 -- list2) end defp check_followers_only(%{host: actor_host} = _actor_info, activity) do diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 6949c5f2d..5b18261a7 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -680,7 +680,9 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do %{ancestors: ancestors, descendants: descendants} = activities |> Enum.reverse() - |> Enum.group_by(fn %{id: id} -> if id < activity.id, do: :ancestors, else: :descendants end) + |> Enum.group_by(fn %{id: id} -> + if id < activity.id, do: :ancestors, else: :descendants + end) |> Map.put_new(:ancestors, []) |> Map.put_new(:descendants, []) diff --git a/lib/pleroma/workers/mailer_worker.ex b/lib/pleroma/workers/mailer_worker.ex index b0259b191..9ef6a80c9 100644 --- a/lib/pleroma/workers/mailer_worker.ex +++ b/lib/pleroma/workers/mailer_worker.ex @@ -6,7 +6,9 @@ defmodule Pleroma.Workers.MailerWorker do use Oban.Worker, queue: :background @impl true - def perform(%Job{args: %{"op" => "email", "encoded_email" => encoded_email, "config" => config}}) do + def perform(%Job{ + args: %{"op" => "email", "encoded_email" => encoded_email, "config" => config} + }) do encoded_email |> Base.decode64!() |> :erlang.binary_to_term() diff --git a/test/pleroma/search/meilisearch_test.exs b/test/pleroma/search/meilisearch_test.exs index ff32491c5..6e4a42c95 100644 --- a/test/pleroma/search/meilisearch_test.exs +++ b/test/pleroma/search/meilisearch_test.exs @@ -99,7 +99,7 @@ defmodule Pleroma.Search.MeilisearchTest do %{method: :delete, url: "http://127.0.0.1:7700/indexes/objects/documents/" <> id} -> send(self(), "called_delete") - assert String.length(id) > 1 + assert id != "" json(%{}) end) diff --git a/test/pleroma/search/paradedb_integration_test.exs b/test/pleroma/search/parade_db_integration_test.exs similarity index 100% rename from test/pleroma/search/paradedb_integration_test.exs rename to test/pleroma/search/parade_db_integration_test.exs diff --git a/test/pleroma/search/paradedb_test.exs b/test/pleroma/search/parade_db_test.exs similarity index 94% rename from test/pleroma/search/paradedb_test.exs rename to test/pleroma/search/parade_db_test.exs index 3a3b8298f..3133b6b38 100644 --- a/test/pleroma/search/paradedb_test.exs +++ b/test/pleroma/search/parade_db_test.exs @@ -75,15 +75,21 @@ defmodule Pleroma.Search.ParadeDBTest do }) args = %{"op" => "add_to_index", "activity" => activity.id} - assert_enqueued(worker: SearchIndexingWorker, args: args) + refute_enqueued(worker: SearchIndexingWorker, args: args) + end) + end - Config - |> expect(:get, fn - [Pleroma.Search, :module], nil -> - ParadeDB - end) + test "backend skips non-public posts when called directly" do + user = insert(:user) - assert :ok = perform_job(SearchIndexingWorker, args) + Enum.each(["private", "direct"], fn visibility -> + {:ok, activity} = + CommonAPI.post(user, %{ + status: "guys i just don't wanna leave the swamp", + visibility: visibility + }) + + assert :ok = ParadeDB.add_to_index(activity) end) end diff --git a/test/pleroma/search_test.exs b/test/pleroma/search_test.exs index d777bcda2..00d8b10ed 100644 --- a/test/pleroma/search_test.exs +++ b/test/pleroma/search_test.exs @@ -3,26 +3,32 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.SearchTest do + require Pleroma.Constants + use Pleroma.DataCase, async: true use Oban.Testing, repo: Pleroma.Repo import Pleroma.Factory + alias Pleroma.Object + alias Pleroma.Search alias Pleroma.Web.CommonAPI alias Pleroma.Workers.SearchIndexingWorker - test "indexes posts that are public" do + test "indexes posts that are public or unlisted" do user = insert(:user) - {:ok, activity} = - CommonAPI.post(user, %{ - status: "Well this is a story all about how my life got flipped turned upside down", - visibility: "public" - }) + Enum.each(["public", "unlisted"], fn visibility -> + {:ok, activity} = + CommonAPI.post(user, %{ + status: "Well this is a story all about how my life got flipped turned upside down", + visibility: visibility + }) - args = %{"op" => "add_to_index", "activity" => activity.id} + args = %{"op" => "add_to_index", "activity" => activity.id} - assert_enqueued(worker: SearchIndexingWorker, args: args) + assert_enqueued(worker: SearchIndexingWorker, args: args) + end) end test "doesn't index posts that are not public" do @@ -66,4 +72,76 @@ defmodule Pleroma.SearchTest do refute_enqueued(worker: SearchIndexingWorker, args: args) end + + test "object_to_search_data includes content warnings and attachment descriptions" do + object = %Object{ + id: 1, + data: %{ + "id" => "https://example.com/objects/1", + "type" => "Note", + "content" => "body text", + "summary" => "subject text", + "attachment" => [ + %{"type" => "Document", "name" => "image description"}, + %{"type" => "Document", "name" => ""} + ], + "published" => DateTime.utc_now() |> DateTime.to_iso8601() + } + } + + assert %{content: content} = Search.object_to_search_data(object) + assert content =~ "body text" + assert content =~ "subject text" + assert content =~ "image description" + end + + test "object_to_search_data ignores objects with invalid published dates" do + for published <- [nil, 123, "not a date"] do + object = %Object{ + id: 1, + data: %{ + "id" => "https://example.com/objects/1", + "type" => "Note", + "content" => "body text", + "published" => published + } + } + + refute Search.object_to_search_data(object) + end + end + + test "indexable? accepts searchable content warnings and attachment descriptions" do + public = Pleroma.Constants.as_public() + published = DateTime.utc_now() |> DateTime.to_iso8601() + + for data <- [ + %{"content" => ".", "summary" => "subject text", "attachment" => []}, + %{ + "content" => ".", + "summary" => "", + "attachment" => [%{"type" => "Document", "name" => "image description"}] + } + ] do + object = %Object{ + id: 1, + data: + Map.merge(data, %{ + "id" => "https://example.com/objects/1", + "type" => "Note", + "to" => [public], + "published" => published + }) + } + + activity = %Pleroma.Activity{ + id: Ecto.UUID.generate(), + data: %{"type" => "Create"}, + object: object, + recipients: [public] + } + + assert Search.indexable?(activity) + end + end end From fc9e6793ce3aaf1c9f4b305d2b9692ed59d3c058 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Fri, 29 May 2026 12:30:33 +0400 Subject: [PATCH 080/183] Address grouped notification review cleanup --- lib/pleroma/web/mastodon_api/mastodon_api.ex | 23 +++----- .../notification_controller_test.exs | 59 +++++++++++-------- 2 files changed, 43 insertions(+), 39 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/mastodon_api.ex b/lib/pleroma/web/mastodon_api/mastodon_api.ex index cb11a3f4a..d6f2dd3a3 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api.ex @@ -73,7 +73,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPI do def get_grouped_notification_page(user, params \\ %{}) do grouped_types = params - |> Map.get("grouped_types", Map.get(params, :grouped_types)) + |> Map.get("grouped_types") |> Notification.normalize_grouped_types() query = notifications_query(user, params) @@ -98,13 +98,6 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPI do } end - def get_grouped_notification_groups(user, params \\ %{}) do - {groups, _notifications, _notification_group_counts, _notification_group_bounds} = - get_grouped_notification_page(user, params) - - groups - end - def get_notification_group_result(user, group_key, params \\ %{}) do notifications = notification_group_sample(user, params, group_key) metadata = notification_group_metadata(user, group_key, params) @@ -163,21 +156,21 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPI do defp group_pagination(params) do cond do - min_id = Map.get(params, "min_id", Map.get(params, :min_id)) -> + min_id = Map.get(params, "min_id") -> cursor_filters = [{:gt, min_id}] cursor_filters = - case Map.get(params, "max_id", Map.get(params, :max_id)) do + case Map.get(params, "max_id") do nil -> cursor_filters max_id -> [{:lt, max_id} | cursor_filters] end {:asc, cursor_filters} - since_id = Map.get(params, "since_id", Map.get(params, :since_id)) -> + since_id = Map.get(params, "since_id") -> {:desc, [{:gt, since_id}]} - max_id = Map.get(params, "max_id", Map.get(params, :max_id)) -> + max_id = Map.get(params, "max_id") -> {:desc, [{:lt, max_id}]} true -> @@ -187,7 +180,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPI do defp grouped_limit(params) do params - |> Map.get("limit", Map.get(params, :limit, 40)) + |> Map.get("limit", 40) |> parse_limit(40) |> min(80) end @@ -195,7 +188,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPI do def unread_notification_group_count(user, params \\ %{}) do grouped_types = params - |> Map.get("grouped_types", Map.get(params, :grouped_types)) + |> Map.get("grouped_types") |> Notification.normalize_grouped_types() limit = unread_count_limit(params) @@ -406,7 +399,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPI do defp unread_count_limit(params) do params - |> Map.get("limit", Map.get(params, :limit, 100)) + |> Map.get("limit", 100) |> parse_limit(100) |> min(1000) end diff --git a/test/pleroma/web/mastodon_api/controllers/notification_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/notification_controller_test.exs index e9c521d6b..55df41ec6 100644 --- a/test/pleroma/web/mastodon_api/controllers/notification_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/notification_controller_test.exs @@ -837,40 +837,28 @@ defmodule Pleroma.Web.MastodonAPI.NotificationControllerTest do refute link_header =~ "since_id=" end - test "lists accounts from and dismisses a notification group" do - %{user: user, conn: conn} = oauth_access(["read:notifications", "write:notifications"]) - other_users = insert_list(9, :user) + test "lists accounts from a notification group" do + %{user: user, conn: conn} = oauth_access(["read:notifications"]) - {:ok, status} = CommonAPI.post(user, %{status: "hello"}) - - Enum.each(other_users, fn other_user -> - {:ok, _} = CommonAPI.favorite(status.id, other_user) - end) - - %{ - "notification_groups" => [ - %{ - "group_key" => group_key, - "notifications_count" => 9, - "sample_account_ids" => sample_account_ids - } - ] - } = - conn - |> get("/api/v2/notifications") - |> json_response_and_validate_schema(200) + {other_users, %{"group_key" => group_key, "sample_account_ids" => sample_account_ids}} = + create_favourite_notification_group(user, conn) assert length(sample_account_ids) == 8 - %{conn: read_conn} = oauth_access(["read:notifications"], user: user) - account_ids = - read_conn + conn |> get("/api/v2/notifications/#{group_key}/accounts") |> json_response_and_validate_schema(200) |> Enum.map(& &1["id"]) assert Enum.sort(account_ids) == Enum.sort(Enum.map(other_users, & &1.id)) + end + + test "dismisses a notification group" do + %{user: user, conn: conn} = oauth_access(["read:notifications", "write:notifications"]) + + {_other_users, %{"group_key" => group_key}} = + create_favourite_notification_group(user, conn) assert %{} = conn @@ -1012,6 +1000,29 @@ defmodule Pleroma.Web.MastodonAPI.NotificationControllerTest do |> to_string() end + defp create_favourite_notification_group(user, conn) do + other_users = insert_list(9, :user) + + {:ok, status} = CommonAPI.post(user, %{status: "hello"}) + + Enum.each(other_users, fn other_user -> + {:ok, _} = CommonAPI.favorite(status.id, other_user) + end) + + %{ + "notification_groups" => [ + %{ + "notifications_count" => 9 + } = group + ] + } = + conn + |> get("/api/v2/notifications") + |> json_response_and_validate_schema(200) + + {other_users, group} + end + defp params_to_query(%{} = params) do Enum.map_join(params, "&", fn {k, v} when is_list(v) -> Enum.map_join(v, "&", &"#{k}[]=#{&1}") From e676ec350713d766b653f60f906fceec34c2c6a1 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Fri, 29 May 2026 18:35:15 +0400 Subject: [PATCH 081/183] Reject unsupported HTTP Message Signatures --- .../activitypub-bot-rfc9421-fallback.fix | 1 + .../activity_pub/activity_pub_controller.ex | 33 ++++++-- .../activity_pub_controller_test.exs | 77 +++++++++++++++++++ 3 files changed, 103 insertions(+), 8 deletions(-) create mode 100644 changelog.d/activitypub-bot-rfc9421-fallback.fix diff --git a/changelog.d/activitypub-bot-rfc9421-fallback.fix b/changelog.d/activitypub-bot-rfc9421-fallback.fix new file mode 100644 index 000000000..22642fee1 --- /dev/null +++ b/changelog.d/activitypub-bot-rfc9421-fallback.fix @@ -0,0 +1 @@ +Reject unsupported RFC 9421 inbox signatures instead of queueing a doomed failed-signature retry, allowing clients to fall back to legacy HTTP Signatures. diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index 3d1875599..dd644c132 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -353,15 +353,21 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do end def inbox(%{assigns: %{valid_signature: false}} = conn, params) do - Federator.incoming_failed_signature_ap_doc(%{ - method: conn.method, - req_headers: conn.req_headers, - request_path: conn.request_path, - params: params, - query_string: conn.query_string - }) + if unsupported_http_message_signature?(conn) do + conn + |> put_status(:unauthorized) + |> json("error, unsupported HTTP Message Signature") + else + Federator.incoming_failed_signature_ap_doc(%{ + method: conn.method, + req_headers: conn.req_headers, + request_path: conn.request_path, + params: params, + query_string: conn.query_string + }) - json(conn, "ok") + json(conn, "ok") + end end # POST /relay/inbox -or- POST /internal/fetch/inbox @@ -381,6 +387,17 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do |> json("error, missing HTTP Signature") end + defp unsupported_http_message_signature?(conn) do + Plug.Conn.get_req_header(conn, "signature-input") != [] and + not cavage_signature?(conn) + end + + defp cavage_signature?(conn) do + conn + |> Plug.Conn.get_req_header("signature") + |> Enum.any?(&Regex.match?(~r/(^|,)\s*keyId\s*=/, &1)) + end + defp post_inbox_relayed_create(conn, params) do Logger.debug( "Signature missing or not from author, relayed Create message, fetching object from source" diff --git a/test/pleroma/web/activity_pub/activity_pub_controller_test.exs b/test/pleroma/web/activity_pub/activity_pub_controller_test.exs index 68e39b3a0..e9a777758 100644 --- a/test/pleroma/web/activity_pub/activity_pub_controller_test.exs +++ b/test/pleroma/web/activity_pub/activity_pub_controller_test.exs @@ -757,6 +757,83 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do assert Activity.get_by_ap_id(data["id"]) end + test "rejects unsupported HTTP Message Signatures so clients can fall back", %{conn: conn} do + data = %{ + "type" => "Create", + "actor" => "https://activitypubbot.example/user/ok", + "id" => "https://activitypubbot.example/activities/rfc9421-create", + "to" => ["https://www.w3.org/ns/activitystreams#Public"], + "cc" => [], + "object" => %{ + "type" => "Note", + "id" => "https://activitypubbot.example/objects/rfc9421-note", + "actor" => "https://activitypubbot.example/user/ok", + "attributedTo" => "https://activitypubbot.example/user/ok", + "content" => "hello from activitypub-bot", + "published" => "2026-05-29T13:33:31Z", + "to" => ["https://www.w3.org/ns/activitystreams#Public"], + "cc" => [] + } + } + + Mox.expect(Pleroma.StubbedHTTPSignaturesMock, :validate_conn, fn _conn -> false end) + + conn = + conn + |> put_req_header("content-type", "application/activity+json") + |> put_req_header("date", "Fri, 29 May 2026 13:33:31 GMT") + |> put_req_header("content-digest", "sha-256=:fake-digest:") + |> put_req_header( + "signature-input", + "sig1=(\"@method\" \"@target-uri\" \"date\" \"content-type\" \"content-digest\");keyid=\"https://activitypubbot.example/user/ok/publickey\";alg=\"rsa-v1_5-sha256\";created=1780061611" + ) + |> put_req_header("signature", "sig1=:fake-signature:") + |> post("/inbox", data) + + assert "error, unsupported HTTP Message Signature" == json_response(conn, 401) + assert [] == all_enqueued(worker: SignatureRetryWorker) + end + + test "keeps failed-signature retry for legacy signatures with Signature-Input", %{conn: conn} do + data = %{ + "type" => "Create", + "actor" => "https://activitypubbot.example/user/ok", + "id" => "https://activitypubbot.example/activities/legacy-create", + "to" => ["https://www.w3.org/ns/activitystreams#Public"], + "cc" => [], + "object" => %{ + "type" => "Note", + "id" => "https://activitypubbot.example/objects/legacy-note", + "actor" => "https://activitypubbot.example/user/ok", + "attributedTo" => "https://activitypubbot.example/user/ok", + "content" => "hello from activitypub-bot", + "published" => "2026-05-29T13:33:31Z", + "to" => ["https://www.w3.org/ns/activitystreams#Public"], + "cc" => [] + } + } + + Mox.expect(Pleroma.StubbedHTTPSignaturesMock, :validate_conn, fn _conn -> false end) + + conn = + conn + |> put_req_header("content-type", "application/activity+json") + |> put_req_header("date", "Fri, 29 May 2026 13:33:31 GMT") + |> put_req_header("digest", "SHA-256=fake-digest") + |> put_req_header( + "signature-input", + "sig1=(\"@method\" \"@target-uri\" \"date\" \"content-type\" \"digest\");keyid=\"https://activitypubbot.example/user/ok/publickey\";alg=\"rsa-v1_5-sha256\";created=1780061611" + ) + |> put_req_header( + "signature", + "keyId=\"https://activitypubbot.example/user/ok#main-key\",algorithm=\"rsa-sha256\",headers=\"(request-target) host date digest content-type\",signature=\"fake-signature\"" + ) + |> post("/inbox", data) + + assert "ok" == json_response(conn, 200) + assert [_] = all_enqueued(worker: SignatureRetryWorker) + end + test "does not create a forged post after failed signature retry", %{conn: conn} do alice = insert(:user, local: false, ap_id: "https://one.com/users/alice") bob = insert(:user, local: false, ap_id: "https://two.com/users/bob") From 86dd9663fc3e1162d5747fc140774374dcc65896 Mon Sep 17 00:00:00 2001 From: Henry Jameson Date: Mon, 1 Jun 2026 22:06:12 +0300 Subject: [PATCH 082/183] allow
(used by mfm) --- priv/scrubbers/default.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/priv/scrubbers/default.ex b/priv/scrubbers/default.ex index cc5eba027..8e2d16650 100644 --- a/priv/scrubbers/default.ex +++ b/priv/scrubbers/default.ex @@ -46,6 +46,7 @@ defmodule Pleroma.HTML.Scrubber.Default do Meta.allow_tag_with_these_attributes(:b, ["lang"]) Meta.allow_tag_with_these_attributes(:blockquote, ["lang"]) Meta.allow_tag_with_these_attributes(:br, ["lang"]) + Meta.allow_tag_with_these_attributes(:center, []) Meta.allow_tag_with_these_attributes(:cite, ["lang"]) Meta.allow_tag_with_these_attributes(:code, ["lang"]) Meta.allow_tag_with_these_attributes(:del, ["lang"]) From 07c65adb0c5ad94610a998730cd81439bf498f8b Mon Sep 17 00:00:00 2001 From: Phantasm Date: Wed, 10 Jun 2026 22:26:16 +0200 Subject: [PATCH 083/183] Grouped Notifications: Add feature flag to instance view and nodeinfo --- changelog.d/grouped_notifications_flag.skip | 0 lib/pleroma/web/mastodon_api/views/instance_view.ex | 1 + test/pleroma/web/node_info_test.exs | 1 + 3 files changed, 2 insertions(+) create mode 100644 changelog.d/grouped_notifications_flag.skip diff --git a/changelog.d/grouped_notifications_flag.skip b/changelog.d/grouped_notifications_flag.skip new file mode 100644 index 000000000..e69de29bb diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index d64ce4fb5..4fccdd01e 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -166,6 +166,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do [ "pleroma_api", "mastodon_api", + "mastodon_api_grouped_notifications", "mastodon_api_streaming", "polls", "v2_suggestions", diff --git a/test/pleroma/web/node_info_test.exs b/test/pleroma/web/node_info_test.exs index afe4ebb36..e2bf3400a 100644 --- a/test/pleroma/web/node_info_test.exs +++ b/test/pleroma/web/node_info_test.exs @@ -160,6 +160,7 @@ defmodule Pleroma.Web.NodeInfoTest do default_features = [ "pleroma_api", "mastodon_api", + "mastodon_api_grouped_notifications", "mastodon_api_streaming", "polls", "pleroma_explicit_addressing", From 9782bcf979c12b06b7aefde14ed3708e74ff0657 Mon Sep 17 00:00:00 2001 From: Phantasm Date: Tue, 16 Jun 2026 18:22:23 +0200 Subject: [PATCH 084/183] Elixir 1.19: Fix most MRFs being absent from AdminAPI suggestions. Elixir 1.19 started lazily loading modules, so modules only get loaded when referenced. Thus :code.all_loaded would no longer include all MRF policies which is used on startup to generate sugestions for AdminAPI. --- lib/pleroma/docs/generator.ex | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/docs/generator.ex b/lib/pleroma/docs/generator.ex index 93b19484f..70413837a 100644 --- a/lib/pleroma/docs/generator.ex +++ b/lib/pleroma/docs/generator.ex @@ -12,13 +12,27 @@ defmodule Pleroma.Docs.Generator do @spec list_behaviour_implementations(behaviour :: module()) :: [module()] def list_behaviour_implementations(behaviour) do - :code.all_loaded() - |> Enum.filter(fn {module, _} -> + # :code.all_loaded() no longer works on Elixir 1.19 and newer since modules are lazily loaded. + # We can get a complete list of modules by interating over Application.loaded_applications/0, + # but that includes every module which would get loaded below, which is very slow. + # Hardcode the Pleroma application instead, which is good enough for now. + application_modules = + case :application.get_key(:pleroma, :modules) do + {:ok, modules} -> modules + :undefined -> [] + end + + # At compile time or in tests some modules will be outside of the Pleroma application. + loaded_modules = Enum.map(:code.all_loaded(), fn {module, _} -> module end) + + (application_modules ++ loaded_modules) + |> Enum.uniq() + |> Enum.filter(fn module -> + Code.ensure_loaded(module) + # This shouldn't be needed as all modules are expected to have module_info/1, # but in test environments some transient modules `:elixir_compiler_XX` # are loaded for some reason (where XX is a random integer). - Code.ensure_loaded(module) - if function_exported?(module, :module_info, 1) do module.module_info(:attributes) |> Keyword.get_values(:behaviour) @@ -26,7 +40,6 @@ defmodule Pleroma.Docs.Generator do |> Enum.member?(behaviour) end end) - |> Enum.map(fn {module, _} -> module end) end @doc """ From c6d1cead8323ac8ce7ebd13395f84d90fea43518 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?nicole=20miko=C5=82ajczyk?= Date: Tue, 23 Jun 2026 16:08:52 +0200 Subject: [PATCH 085/183] Allow attaching emojis to user lists MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: nicole mikołajczyk --- changelog.d/list-emoji.add | 1 + lib/pleroma/list.ex | 42 ++++++++++++++++++- .../web/api_spec/operations/list_operation.ex | 9 ++-- lib/pleroma/web/api_spec/schemas/list.ex | 17 +++++++- .../web/mastodon_api/views/list_view.ex | 23 +++++++++- .../20260623000000_add_emoji_to_lists.exs | 9 ++++ test/pleroma/list_test.exs | 23 ++++++++++ .../controllers/list_controller_test.exs | 33 +++++++++++++++ .../web/mastodon_api/views/list_view_test.exs | 18 +++++++- 9 files changed, 168 insertions(+), 7 deletions(-) create mode 100644 changelog.d/list-emoji.add create mode 100644 priv/repo/migrations/20260623000000_add_emoji_to_lists.exs diff --git a/changelog.d/list-emoji.add b/changelog.d/list-emoji.add new file mode 100644 index 000000000..dce4874d3 --- /dev/null +++ b/changelog.d/list-emoji.add @@ -0,0 +1 @@ +Allow attaching emojis to user lists diff --git a/lib/pleroma/list.ex b/lib/pleroma/list.ex index cd23bd0a9..3e8a83342 100644 --- a/lib/pleroma/list.ex +++ b/lib/pleroma/list.ex @@ -9,6 +9,7 @@ defmodule Pleroma.List do import Ecto.Changeset alias Pleroma.Activity + alias Pleroma.Emoji alias Pleroma.Repo alias Pleroma.User @@ -18,14 +19,53 @@ defmodule Pleroma.List do field(:following, {:array, :string}, default: []) field(:ap_id, :string) field(:exclusive, :boolean, default: false) + field(:emoji, :string) timestamps() end def update_changeset(list, attrs \\ %{}) do list - |> cast(attrs, [:title, :exclusive]) + |> cast(attrs, [:title, :exclusive, :emoji]) |> validate_required([:title]) + |> fix_emoji() + |> validate_emoji() + end + + defp fix_emoji(changeset) do + with {:emoji_field, emoji} when is_binary(emoji) <- + {:emoji_field, get_field(changeset, :emoji)}, + {:fixed_emoji, emoji} <- + {:fixed_emoji, + emoji + |> Emoji.fully_qualify_emoji() + |> Emoji.maybe_quote()} do + put_change(changeset, :emoji, emoji) + else + {:emoji_field, _} -> changeset + end + end + + defp validate_emoji(changeset) do + validate_change(changeset, :emoji, fn + :emoji, nil -> + [] + + :emoji, emoji -> + if Emoji.unicode?(emoji) or valid_local_custom_emoji?(emoji) do + [] + else + [emoji: "Invalid emoji"] + end + end) + end + + defp valid_local_custom_emoji?(emoji) do + with %{file: _path} <- Emoji.get(emoji) do + true + else + _ -> false + end end def follow_changeset(list, attrs \\ %{}) do diff --git a/lib/pleroma/web/api_spec/operations/list_operation.ex b/lib/pleroma/web/api_spec/operations/list_operation.ex index 87189edc2..8aec1c352 100644 --- a/lib/pleroma/web/api_spec/operations/list_operation.ex +++ b/lib/pleroma/web/api_spec/operations/list_operation.ex @@ -41,7 +41,8 @@ defmodule Pleroma.Web.ApiSpec.ListOperation do responses: %{ 200 => Operation.response("List", "application/json", List), 400 => Operation.response("Error", "application/json", ApiError), - 404 => Operation.response("Error", "application/json", ApiError) + 404 => Operation.response("Error", "application/json", ApiError), + 422 => Operation.response("Error", "application/json", ApiError) } } end @@ -175,7 +176,8 @@ defmodule Pleroma.Web.ApiSpec.ListOperation do exclusive: %Schema{ type: :boolean, description: "Whether members of the list should be removed from the “Home” feed" - } + }, + emoji: %Schema{type: :string, description: "List emoji", nullable: true} }, required: [:title] }, @@ -194,7 +196,8 @@ defmodule Pleroma.Web.ApiSpec.ListOperation do exclusive: %Schema{ type: :boolean, description: "Whether members of the list should be removed from the “Home” feed" - } + }, + emoji: %Schema{type: :string, description: "List emoji", nullable: true} } }, required: true diff --git a/lib/pleroma/web/api_spec/schemas/list.ex b/lib/pleroma/web/api_spec/schemas/list.ex index 5df674894..ff52b1def 100644 --- a/lib/pleroma/web/api_spec/schemas/list.ex +++ b/lib/pleroma/web/api_spec/schemas/list.ex @@ -17,11 +17,26 @@ defmodule Pleroma.Web.ApiSpec.Schemas.List do exclusive: %Schema{ type: :boolean, description: "Whether members of the list should be removed from the “Home” feed" + }, + pleroma: %Schema{ + type: :object, + properties: %{ + emoji: %Schema{type: :string, description: "List emoji", nullable: true}, + emoji_url: %Schema{ + type: :string, + description: "URL of the list emoji if it's a custom emoji, null otherwise", + nullable: true + } + } } }, example: %{ "id" => "12249", - "title" => "Friends" + "title" => "Friends", + "pleroma" => %{ + "emoji" => nil, + "emoji_url" => nil + } } }) end diff --git a/lib/pleroma/web/mastodon_api/views/list_view.ex b/lib/pleroma/web/mastodon_api/views/list_view.ex index 740f458d4..508169fc1 100644 --- a/lib/pleroma/web/mastodon_api/views/list_view.ex +++ b/lib/pleroma/web/mastodon_api/views/list_view.ex @@ -4,6 +4,8 @@ defmodule Pleroma.Web.MastodonAPI.ListView do use Pleroma.Web, :view + + alias Pleroma.Emoji alias Pleroma.Web.MastodonAPI.ListView def render("index.json", %{lists: lists} = opts) do @@ -11,10 +13,29 @@ defmodule Pleroma.Web.MastodonAPI.ListView do end def render("show.json", %{list: list}) do + {emoji, emoji_url} = get_emoji(list.emoji) + %{ id: to_string(list.id), title: list.title, - exclusive: list.exclusive + exclusive: list.exclusive, + pleroma: %{ + emoji: emoji, + emoji_url: emoji_url + } } end + + defp get_emoji(nil), do: {nil, nil} + + defp get_emoji(emoji) do + if Emoji.unicode?(emoji) do + {emoji, nil} + else + case Emoji.get(emoji) do + nil -> {nil, nil} + emoji_data -> {emoji, Emoji.local_url(emoji_data.file)} + end + end + end end diff --git a/priv/repo/migrations/20260623000000_add_emoji_to_lists.exs b/priv/repo/migrations/20260623000000_add_emoji_to_lists.exs new file mode 100644 index 000000000..b72ff0c13 --- /dev/null +++ b/priv/repo/migrations/20260623000000_add_emoji_to_lists.exs @@ -0,0 +1,9 @@ +defmodule Pleroma.Repo.Migrations.AddEmojiToLists do + use Ecto.Migration + + def change do + alter table(:lists) do + add(:emoji, :string) + end + end +end diff --git a/test/pleroma/list_test.exs b/test/pleroma/list_test.exs index d44310689..32121b8fd 100644 --- a/test/pleroma/list_test.exs +++ b/test/pleroma/list_test.exs @@ -23,6 +23,29 @@ defmodule Pleroma.ListTest do assert changeset.errors == [title: {"can't be blank", [validation: :required]}] end + test "creating a list with an emoji" do + user = insert(:user) + {:ok, %Pleroma.List{} = list} = Pleroma.List.create(%{title: "title", emoji: "🕓"}, user) + %Pleroma.List{emoji: emoji} = Pleroma.List.get(list.id, user) + assert emoji == "🕓" + end + + test "validates emoji" do + user = insert(:user) + + assert {:error, changeset} = + Pleroma.List.create(%{title: "title", emoji: "not an emoji"}, user) + + assert changeset.errors == [emoji: {"Invalid emoji", []}] + end + + test "updating a list's emoji" do + user = insert(:user) + {:ok, list} = Pleroma.List.create(%{title: "title"}, user) + {:ok, %{emoji: emoji}} = Pleroma.List.update(list, %{emoji: "🕓"}) + assert "🕓" == emoji + end + test "getting a list not belonging to the user" do user = insert(:user) other_user = insert(:user) diff --git a/test/pleroma/web/mastodon_api/controllers/list_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/list_controller_test.exs index bfda48be4..2845f67ed 100644 --- a/test/pleroma/web/mastodon_api/controllers/list_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/list_controller_test.exs @@ -19,6 +19,16 @@ defmodule Pleroma.Web.MastodonAPI.ListControllerTest do |> json_response_and_validate_schema(:ok) end + test "creating a list with an emoji" do + %{conn: conn} = oauth_access(["write:lists"]) + + assert %{"title" => "cuties", "pleroma" => %{"emoji" => "🕓", "emoji_url" => nil}} = + conn + |> put_req_header("content-type", "application/json") + |> post("/api/v1/lists", %{"title" => "cuties", "emoji" => "🕓"}) + |> json_response_and_validate_schema(:ok) + end + test "renders error for invalid params" do %{conn: conn} = oauth_access(["write:lists"]) @@ -31,6 +41,18 @@ defmodule Pleroma.Web.MastodonAPI.ListControllerTest do json_response_and_validate_schema(conn, 400) end + test "renders error for an invalid emoji" do + %{conn: conn} = oauth_access(["write:lists"]) + + conn = + conn + |> put_req_header("content-type", "application/json") + |> post("/api/v1/lists", %{"title" => "cuties", "emoji" => "not an emoji"}) + + assert %{"error" => "Invalid emoji"} = + json_response_and_validate_schema(conn, :unprocessable_entity) + end + test "listing a user's lists" do %{conn: conn} = oauth_access(["read:lists", "write:lists"]) @@ -159,6 +181,17 @@ defmodule Pleroma.Web.MastodonAPI.ListControllerTest do |> json_response_and_validate_schema(:ok) end + test "updating a list's emoji" do + %{user: user, conn: conn} = oauth_access(["write:lists"]) + {:ok, list} = Pleroma.List.create(%{title: "name"}, user) + + assert %{"pleroma" => %{"emoji" => "🕓", "emoji_url" => nil}} = + conn + |> put_req_header("content-type", "application/json") + |> put("/api/v1/lists/#{list.id}", %{"title" => "name", "emoji" => "🕓"}) + |> json_response_and_validate_schema(:ok) + end + test "validates title when renaming a list" do %{user: user, conn: conn} = oauth_access(["write:lists"]) {:ok, list} = Pleroma.List.create(%{title: "name"}, user) diff --git a/test/pleroma/web/mastodon_api/views/list_view_test.exs b/test/pleroma/web/mastodon_api/views/list_view_test.exs index ae0593b6b..86f461ded 100644 --- a/test/pleroma/web/mastodon_api/views/list_view_test.exs +++ b/test/pleroma/web/mastodon_api/views/list_view_test.exs @@ -15,12 +15,28 @@ defmodule Pleroma.Web.MastodonAPI.ListViewTest do expected = %{ id: to_string(list.id), title: title, - exclusive: false + exclusive: false, + pleroma: %{ + emoji: nil, + emoji_url: nil + } } assert expected == ListView.render("show.json", %{list: list}) end + test "show with a unicode emoji" do + user = insert(:user) + {:ok, list} = Pleroma.List.create(%{title: "mortal enemies", emoji: "🕓"}, user) + + assert %{ + pleroma: %{ + emoji: "🕓", + emoji_url: nil + } + } = ListView.render("show.json", %{list: list}) + end + test "index" do user = insert(:user) From 19a84baf82da5b8e8204bff9e305eeac5bdb4189 Mon Sep 17 00:00:00 2001 From: Phantasm Date: Wed, 10 Jun 2026 22:54:22 +0200 Subject: [PATCH 086/183] InstanceView: Add feature flag tests --- .../mastodon_api/views/instance_view_test.exs | 98 +++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100644 test/pleroma/web/mastodon_api/views/instance_view_test.exs diff --git a/test/pleroma/web/mastodon_api/views/instance_view_test.exs b/test/pleroma/web/mastodon_api/views/instance_view_test.exs new file mode 100644 index 000000000..46bb80ba4 --- /dev/null +++ b/test/pleroma/web/mastodon_api/views/instance_view_test.exs @@ -0,0 +1,98 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2026 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do + use Pleroma.DataCase, async: true + + alias Pleroma.Language.LanguageDetectorMock + alias Pleroma.StaticStubbedConfigMock + alias Pleroma.Web.MastodonAPI.InstanceView + + import Mox + + @features [ + "pleroma_api", + "mastodon_api", + "mastodon_api_grouped_notifications", + "mastodon_api_streaming", + "polls", + "v2_suggestions", + "pleroma_explicit_addressing", + "shareable_emoji_packs", + "multifetch", + "pleroma:api/v1/notifications:include_types_filter", + "editing", + "quote_posting", + "pleroma_emoji_reactions", + "pleroma_custom_emoji_reactions", + "pleroma_chat_messages", + "pleroma:pin_chats", + "pleroma:get:main/ostatus", + "pleroma:group_actors", + "pleroma:bookmark_folders", + "pleroma:block_expiration" + ] + + @configureable_features [ + "blockers_visible", + "media_proxy", + "gopher", + "chat", + "shout", + "relay", + "safe_dm_mentions", + "exposable_reactions", + "profile_directory", + "pleroma:language_detection" + ] + @configurable_features_flags [ + [:activitypub, :blockers_visible], + [:media_proxy, :enabled], + [:gopher, :enabled], + [:shout, :enabled], + [:instance, :allow_relay], + [:instance, :safe_dm_mentions], + [:instance, :show_reactions], + [:instance, :profile_directory] + ] + + # When this fails, a new feature flag was probably added. Add it to the macros above. + describe "uncofigurable features" do + setup do + Enum.each(@configurable_features_flags, &clear_config(&1, false)) + end + + test "returns always enabled features" do + features = @features + + assert Enum.sort(InstanceView.features()) == Enum.sort(features) + end + end + + describe "all features" do + setup do + Enum.each(@configurable_features_flags, &clear_config(&1, true)) + + # Stub the StaticStubbedConfigMock to return our mock for the provider + StaticStubbedConfigMock + |> stub(:get, fn + [Pleroma.Language.LanguageDetector, :provider] -> LanguageDetectorMock + _other -> nil + end) + + # Stub the LanguageDetectorMock with default implementations + LanguageDetectorMock + |> stub(:missing_dependencies, fn -> [] end) + |> stub(:configured?, fn -> true end) + + :ok + end + + test "returns all features including configurable ones" do + features = @features ++ @configureable_features + + assert Enum.sort(InstanceView.features()) == Enum.sort(features) + end + end +end From 7e9b8a766340248578a675df772410b1960ee22c Mon Sep 17 00:00:00 2001 From: Phantasm Date: Tue, 23 Jun 2026 18:20:16 +0200 Subject: [PATCH 087/183] InstanceView: Add federation/0 func test --- .../mastodon_api/views/instance_view_test.exs | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/test/pleroma/web/mastodon_api/views/instance_view_test.exs b/test/pleroma/web/mastodon_api/views/instance_view_test.exs index 46bb80ba4..d34aad25b 100644 --- a/test/pleroma/web/mastodon_api/views/instance_view_test.exs +++ b/test/pleroma/web/mastodon_api/views/instance_view_test.exs @@ -95,4 +95,72 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do assert Enum.sort(InstanceView.features()) == Enum.sort(features) end end + + describe "federation/0" do + setup do: clear_config([:mrf, :transparency], true) + + test "quarantined instances" do + clear_config([:instance, :quarantined_instances], [{"quarantine.example.com", "crimes"}]) + + output = InstanceView.federation() + assert %{quarantined_instances: ["quarantine.example.com"]} = output + assert %{quarantined_instances_info: %{"quarantined_instances" => %{"quarantine.example.com" => %{"reason" => "crimes"}}}} = output + end + + test "rejected instances" do + clear_config([:instance, :rejected_instances], [{"rejected.example.com", "not enough #cofe posting"}]) + + output = InstanceView.federation() + assert %{rejected_instances: %{"rejected.example.com" => %{"reason" => "not enough #cofe posting"}}} = output + end + + test "federating" do + clear_config([:instance, :federating], true) + + output = InstanceView.federation() + assert %{enabled: true} = output + end + + test "transparency" do + clear_config([:instance, :quarantined_instances], [{"quarantine.example.com", "crimes"}]) + clear_config([:instance, :rejected_instances], [{"rejected.example.com", "not enough #cofe posting"}]) + clear_config([:instance, :federating], true) + clear_config([:mrf, :transparency], false) + + output = InstanceView.federation() + assert %{enabled: true} == output + end + + test "MRFs" do + clear_config([:mrf, :policies], [Pleroma.Web.ActivityPub.MRF.SimplePolicy]) + clear_config([:mrf, :transparency_exclusions], [{"media1.example.com", "the Fediverse doesn't need to know"}]) + clear_config([:mrf_simple, :media_removal], [{"media1.example.com", "NSFW"}, {"media2.example.com", "usual suspects"}]) + clear_config([:mrf_simple, :reject], [{"rejected.example.com", "not enough #cofe posting"}]) + + output = InstanceView.federation() + + expected = %{ + mrf_simple: %{ + reject: ["rejected.example.com"], + media_removal: ["media2.example.com"], + }, + mrf_hashtag: %{ + sensitive: ["nsfw"], + }, + exclusions: true, + mrf_policies: ["SimplePolicy", "HashtagPolicy"], + mrf_simple_info: %{ + reject: %{ + "rejected.example.com" => %{"reason" => "not enough #cofe posting"} + }, + media_removal: %{"media2.example.com" => %{"reason" => "usual suspects"}} + } + } + + assert expected.mrf_simple.reject == output.mrf_simple.reject + assert expected.mrf_simple.media_removal == output.mrf_simple.media_removal + assert expected.mrf_simple_info == output.mrf_simple_info + assert expected.exclusions == output.exclusions + end + end end From 86725b5a000a6992cad522be06d0aa99dd829bcb Mon Sep 17 00:00:00 2001 From: Phantasm Date: Tue, 23 Jun 2026 22:22:49 +0200 Subject: [PATCH 088/183] InstanceView: add rule(s), translation render tests --- .../mastodon_api/views/instance_view_test.exs | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/test/pleroma/web/mastodon_api/views/instance_view_test.exs b/test/pleroma/web/mastodon_api/views/instance_view_test.exs index d34aad25b..d3e850766 100644 --- a/test/pleroma/web/mastodon_api/views/instance_view_test.exs +++ b/test/pleroma/web/mastodon_api/views/instance_view_test.exs @@ -163,4 +163,49 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do assert expected.exclusions == output.exclusions end end + + describe "render rule.json" do + setup do + rule = Pleroma.Rule.create(%{text: "hambaga"}) + + %{rule: rule} + end + + test "renders properly", %{rule: rule} do + output = InstanceView.render("rule.json", %{rule: rule}) + + assert %{text: "hambaga"} = output + end + end + + describe "render rules.json" do + setup do + Pleroma.Rule.create(%{text: "rule 1"}) + Pleroma.Rule.create(%{text: "rule 2"}) + Pleroma.Rule.create(%{text: "rule 3"}) + Pleroma.Rule.create(%{text: "rule 4"}) + + :ok + end + + test "renders properly" do + output = InstanceView.render("rules.json", %{}) + keys = Enum.map(output, fn %{text: text} -> text end) + + assert length(keys) == 4 + assert Enum.all?(output, fn %{text: "rule " <> num} -> String.to_integer(num) end) + end + end + + describe "render translation_languages.json" do + setup do + clear_config([Pleroma.Language.Translation, :provider], TranslationMock) + end + + test "returns language matrix" do + output = InstanceView.render("translation_languages.json", %{}) + + assert %{"en" => ["pl"], "pl" => ["en"]} == output + end + end end From df67f219748418b18063d9e788a1945d9dbf14a8 Mon Sep 17 00:00:00 2001 From: Phantasm Date: Tue, 23 Jun 2026 22:24:44 +0200 Subject: [PATCH 089/183] InstanceView: use shorter setup syntax when applicable --- .../pleroma/web/mastodon_api/views/instance_view_test.exs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/test/pleroma/web/mastodon_api/views/instance_view_test.exs b/test/pleroma/web/mastodon_api/views/instance_view_test.exs index d3e850766..f1852cadd 100644 --- a/test/pleroma/web/mastodon_api/views/instance_view_test.exs +++ b/test/pleroma/web/mastodon_api/views/instance_view_test.exs @@ -59,9 +59,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do # When this fails, a new feature flag was probably added. Add it to the macros above. describe "uncofigurable features" do - setup do - Enum.each(@configurable_features_flags, &clear_config(&1, false)) - end + setup do: Enum.each(@configurable_features_flags, &clear_config(&1, false)) test "returns always enabled features" do features = @features @@ -198,9 +196,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do end describe "render translation_languages.json" do - setup do - clear_config([Pleroma.Language.Translation, :provider], TranslationMock) - end + setup do: clear_config([Pleroma.Language.Translation, :provider], TranslationMock) test "returns language matrix" do output = InstanceView.render("translation_languages.json", %{}) From e45dad21046435a3f5e93412a183955c34df16fb Mon Sep 17 00:00:00 2001 From: Phantasm Date: Tue, 23 Jun 2026 22:42:15 +0200 Subject: [PATCH 090/183] InstanceView: Add domain_blocks.json render test --- .../mastodon_api/views/instance_view_test.exs | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/test/pleroma/web/mastodon_api/views/instance_view_test.exs b/test/pleroma/web/mastodon_api/views/instance_view_test.exs index f1852cadd..9701fab85 100644 --- a/test/pleroma/web/mastodon_api/views/instance_view_test.exs +++ b/test/pleroma/web/mastodon_api/views/instance_view_test.exs @@ -195,6 +195,46 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do end end + describe "render domain_blocks.json" do + setup do + clear_config([:mrf, :transparency], true) + clear_config([:mrf, :transparency_exclusions], [{"removed1.example.com", "the Fediverse doesn't need to know"}]) + clear_config([:mrf_simple, :media_removal], [{"media1.example.com", "NSFW"}]) + clear_config([:mrf_simple, :reject], [{"rejected.example.com", "not enough #cofe posting"}, {"rejected-without-reason.example.com", ""}]) + clear_config([:mrf_simple, :federated_timeline_removal], [{"removed1.example.com", "spam"}, {"removed2.example.com", "more spam"}]) + + :ok + end + + test "renders properly" do + output = InstanceView.render("domain_blocks.json", %{}) + + # Media removal is not in @block_severities in the view. + # NOTE: Order of these two matters! + expected = [ + %{ + domain: "removed2.example.com", + comment: "more spam", + severity: "silence", + digest: "5f6649cfc780fac5172dbbcc8a552953f6621a14368d82728b6e8faf51295dc0" + }, + %{ + domain: "rejected.example.com", + comment: "not enough #cofe posting", + severity: "suspend", + digest: "c15082d347fb7eccd251e199a98dcdce94a8dfc1d33d6ece80889eed374b6f92" + }, + %{ + domain: "rejected-without-reason.example.com", + severity: "suspend", + digest: "48297155018c626c8867ca0bba4a2621eccdf51c6f7ac91b7099ed8b2ee9863f" + } + ] + + assert expected == output + end + end + describe "render translation_languages.json" do setup do: clear_config([Pleroma.Language.Translation, :provider], TranslationMock) From c290f022de1b262d8e4d84be84ad7095a89043b5 Mon Sep 17 00:00:00 2001 From: Phantasm Date: Wed, 24 Jun 2026 00:28:44 +0200 Subject: [PATCH 091/183] InstanceView: Add show.json render test --- .../web/mastodon_api/views/instance_view.ex | 2 + .../mastodon_api/views/instance_view_test.exs | 194 ++++++++++++++++++ 2 files changed, 196 insertions(+) diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index 4fccdd01e..1a0f63ffa 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -17,6 +17,8 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do reject: "suspend" } + def mastodon_api_level, do: @mastodon_api_level + def render("show.json", _) do instance = Config.get(:instance) diff --git a/test/pleroma/web/mastodon_api/views/instance_view_test.exs b/test/pleroma/web/mastodon_api/views/instance_view_test.exs index 9701fab85..3b24e8e5f 100644 --- a/test/pleroma/web/mastodon_api/views/instance_view_test.exs +++ b/test/pleroma/web/mastodon_api/views/instance_view_test.exs @@ -5,12 +5,63 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do use Pleroma.DataCase, async: true + import Pleroma.Factory + alias Pleroma.Language.LanguageDetectorMock alias Pleroma.StaticStubbedConfigMock alias Pleroma.Web.MastodonAPI.InstanceView import Mox + @atom_common_information_keys [ + :languages, + :rules, + :title, + :version + ] + + @common_information_keys Enum.map(@atom_common_information_keys, &to_string/1) + + @atom_configuration_keys [ + :accounts, + :statuses, + :media_attachments, + :polls + ] + + @atom_pleroma_configuration_keys [ + :metadata, + :stats, + :vapid_public_key + ] + + @show_keys @common_information_keys ++ [ + "uri", + "description", + "short_description", + "email", + "urls", + "stats", + "thumbnail", + "registrations", + "approval_required", + "contact_account", + "configuration", + # Extra (not present in Mastodon)" + "max_toot_chars", + "max_media_attachments", + "poll_limits", + "upload_limit", + "avatar_upload_limit", + "background_upload_limit", + "banner_upload_limit", + "background_image", + "shout_limit", + "description_limit", + "chat_limit", + "pleroma" + ] + @features [ "pleroma_api", "mastodon_api", @@ -57,6 +108,96 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do [:instance, :profile_directory] ] + defp check_common_information(info) do + filtered_info = Map.reject(info, fn {key, _value} -> key not in @atom_common_information_keys end) + + expected = %{ + languages: ["en", "cs"], + rules: InstanceView.render("rules.json", %{}), + title: "test title", + version: InstanceView.mastodon_api_level() <> " (compatible; #{Pleroma.Application.named_version()})" + } + + Map.equal?(expected, filtered_info) + end + + defp check_contact_name(info, user) do + contact = info[:contact_account] + view = Pleroma.Web.MastodonAPI.AccountView.render("show.json", %{user: user, for: nil}) + + contact == view + end + + defp check_configuration(info) do + configuration = info[:configuration] + filtered_configuration = Map.reject(configuration, fn {key, _value} -> key not in @atom_configuration_keys end) + + expected = %{ + accounts: %{ + max_featured_tags: 0 + }, + statuses: %{ + max_characters: 4096, + max_media_attachments: 1 + }, + media_attachments: %{ + image_size_limit: 1024, + video_size_limit: 1024, + supported_mime_types: ["application/octet-stream"] + }, + polls: %{ + max_options: 4, + max_characters_per_option: 120, + min_expiration: 1, + max_expiration: 2 + } + } + + Map.equal?(expected, filtered_configuration) + end + + defp check_pleroma_configuration(info) do + configuration = info[:pleroma] + filtered_configuration = Map.reject(configuration, fn {key, _value} -> key not in @atom_pleroma_configuration_keys end) + metadata = Map.fetch!(filtered_configuration, :metadata) + # Tested elsewhere already + filtered_metadata = Map.reject(metadata, fn {key, _value} -> key in [:features, :federation] end) + filtered_configuration = %{filtered_configuration | metadata: filtered_metadata} + + expected = %{ + metadata: %{ + account_activation_required: true, + fields_limits: %{ + max_fields: 1, + max_remote_fields: 1, + name_length: 1, + value_length: 1 + }, + post_formats: ["text/plain"], + birthday_required: true, + birthday_min_age: 1, + translation: + %{ + source_languages: ["en", "pl"], + target_languages: ["en", "pl"] + }, + base_urls: %{ + media_proxy: "https://mediaproxy.example.com", + upload: "https://upload.example.com" + }, + markup: %{ + allow_inline_images: true, + allow_headings: true, + allow_tables: true + } + }, + stats: %{mau: Pleroma.User.active_user_count()}, + vapid_public_key: Keyword.get(Pleroma.Web.Push.vapid_config(), :public_key) + } + + Map.equal?(expected, filtered_configuration) + end + # When this fails, a new feature flag was probably added. Add it to the macros above. describe "uncofigurable features" do setup do: Enum.each(@configurable_features_flags, &clear_config(&1, false)) @@ -162,6 +303,59 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do end end + describe "render show.json" do + setup do + Pleroma.Rule.create(%{text: "rule 1"}) + Pleroma.Rule.create(%{text: "rule 2"}) + + user = insert(:user) + + # Wall of configs for testing whether they are rendered properly + clear_config([Pleroma.Language.Translation, :provider], TranslationMock) + + clear_config([:instance, :languages], ["en", "cs"]) + clear_config([:instance, :name], "test title") + clear_config([:instance, :contact_username], user.nickname) + clear_config([:instance, :limit], 4096) + clear_config([:instance, :max_media_attachments], 1) + clear_config([:instance, :upload_limit], 1024) + clear_config([:instance, :poll_limits, :max_options], 4) + clear_config([:instance, :poll_limits, :max_option_chars], 120) + clear_config([:instance, :poll_limits, :min_expiration], 1) + clear_config([:instance, :poll_limits, :max_expiration], 2) + clear_config([:instance, :account_activation_required], true) + clear_config([:instance, :max_account_fields], 1) + clear_config([:instance, :max_remote_account_fields], 1) + clear_config([:instance, :account_field_name_length], 1) + clear_config([:instance, :account_field_value_length], 1) + clear_config([:instance, :allowed_post_formats], ["text/plain"]) + clear_config([:instance, :birthday_required], true) + clear_config([:instance, :birthday_min_age], 1) + + clear_config([:media_proxy, :enabled], true) + clear_config([:media_proxy, :base_url], "https://mediaproxy.example.com") + + clear_config([Pleroma.Upload, :base_url], "https://upload.example.com") + + clear_config([:markup, :allow_inline_images], true) + clear_config([:markup, :allow_headings], true) + clear_config([:markup, :allow_tables], true) + + %{user: user} + end + + test "renders properly", %{user: user} do + output = InstanceView.render("show.json", %{}) + expected_keys = Enum.sort(@show_keys) + + assert check_common_information(output) + assert check_contact_name(output, user) + assert check_configuration(output) + assert check_pleroma_configuration(output) + assert expected_keys == Enum.sort(Enum.map(Map.keys(output), &to_string/1)) + end + end + describe "render rule.json" do setup do rule = Pleroma.Rule.create(%{text: "hambaga"}) From 0337bd0ae4ff1adf26dda28af5405a092c4782e1 Mon Sep 17 00:00:00 2001 From: Phantasm Date: Thu, 25 Jun 2026 00:19:15 +0200 Subject: [PATCH 092/183] InstanceView: Add show2.json render test --- .../mastodon_api/views/instance_view_test.exs | 194 +++++++++++++++++- 1 file changed, 192 insertions(+), 2 deletions(-) diff --git a/test/pleroma/web/mastodon_api/views/instance_view_test.exs b/test/pleroma/web/mastodon_api/views/instance_view_test.exs index 3b24e8e5f..b4622191b 100644 --- a/test/pleroma/web/mastodon_api/views/instance_view_test.exs +++ b/test/pleroma/web/mastodon_api/views/instance_view_test.exs @@ -29,12 +29,20 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do :polls ] + @atom_configuration2_keys @atom_configuration_keys ++ [ + :urls, + :vapid, + :translation, + :timelines_access + ] + @atom_pleroma_configuration_keys [ :metadata, :stats, :vapid_public_key ] + # TODO: Simplify this, so ideally only atoms are macroed @show_keys @common_information_keys ++ [ "uri", "description", @@ -62,6 +70,19 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do "pleroma" ] + @show2_keys @common_information_keys ++ [ + "domain", + "source_url", + "description", + "usage", + "thumbnail", + "configuration", + "registrations", + "contact", + # Extra (not present in Mastodon): + "pleroma" + ] + @features [ "pleroma_api", "mastodon_api", @@ -121,6 +142,27 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do Map.equal?(expected, filtered_info) end + defp check_thumbnail(info) do + thumbnail = info[:thumbnail] + + expected = %{ + url: URI.merge(Pleroma.Web.Endpoint.url(), "https://example.com/thumbnail.png") |> to_string + } + + expected == thumbnail + end + + defp check_contact(info, user) do + contact = info[:contact] + + expected = %{ + email: "nonexist@example.com", + account: Pleroma.Web.MastodonAPI.AccountView.render("show.json", %{user: user, for: nil}) + } + + Map.equal?(expected, contact) + end + defp check_contact_name(info, user) do contact = info[:contact_account] view = Pleroma.Web.MastodonAPI.AccountView.render("show.json", %{user: user, for: nil}) @@ -156,6 +198,70 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do Map.equal?(expected, filtered_configuration) end + defp check_configuration2(info) do + configuration = info[:configuration] + filtered_configuration = Map.reject(configuration, fn {key, _value} -> key not in @atom_configuration2_keys end) + + # configuration2 also includes parts from configuration + expected = %{ + accounts: %{ + max_featured_tags: 0, + max_pinned_statuses: 1, + max_profile_fields: 1, + profile_field_name_limit: 1, + profile_field_value_limit: 1 + }, + statuses: %{ + max_characters: 4096, + max_media_attachments: 1, + characters_reserved_per_url: 0 + }, + media_attachments: %{ + image_size_limit: 1024, + video_size_limit: 1024, + supported_mime_types: ["application/octet-stream"] + }, + polls: %{ + max_options: 4, + max_characters_per_option: 120, + min_expiration: 1, + max_expiration: 2 + }, + urls: %{ + streaming: Pleroma.Web.Endpoint.websocket_url(), + status: "https://status.example.com" + }, + vapid: %{ + public_key: Keyword.get(Pleroma.Web.Push.vapid_config(), :public_key) + }, + translation: %{enabled: true}, + timelines_access: %{ + live_feeds: %{local: "authenticated", remote: "authenticated"}, + hashtag_feeds: %{local: "authenticated", remote: "authenticated"}, + # not implemented in Pleroma + trending_link_feeds: %{ + local: "disabled", + remote: "disabled" + } + } + } + + Map.equal?(expected, filtered_configuration) + end + + defp check_registrations(info) do + registrations = info[:registrations] + + expected = %{ + enabled: false, + approval_required: true, + message: nil, + url: nil + } + + Map.equal?(expected, registrations) + end + defp check_pleroma_configuration(info) do configuration = info[:pleroma] filtered_configuration = Map.reject(configuration, fn {key, _value} -> key not in @atom_pleroma_configuration_keys end) @@ -198,6 +304,56 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do Map.equal?(expected, filtered_configuration) end + defp check_pleroma_configuration2(info) do + configuration = info[:pleroma] + filtered_configuration = Map.reject(configuration, fn {key, _value} -> key not in @atom_pleroma_configuration_keys end) + metadata = Map.fetch!(filtered_configuration, :metadata) + # Tested elsewhere already + filtered_metadata = Map.reject(metadata, fn {key, _value} -> key in [:features, :federation] end) + filtered_configuration = %{filtered_configuration | metadata: filtered_metadata} + + # pleroma_configuration2 also includes parts from pleroma_configuration + expected = %{ + metadata: %{ + account_activation_required: true, + fields_limits: %{ + max_fields: 1, + max_remote_fields: 1, + name_length: 1, + value_length: 1 + }, + post_formats: ["text/plain"], + birthday_required: true, + birthday_min_age: 1, + translation: + %{ + source_languages: ["en", "pl"], + target_languages: ["en", "pl"] + }, + base_urls: %{ + media_proxy: "https://mediaproxy.example.com", + upload: "https://upload.example.com" + }, + markup: %{ + allow_inline_images: true, + allow_headings: true, + allow_tables: true + }, + avatar_upload_limit: 1024, + background_upload_limit: 1024, + banner_upload_limit: 1024, + background_image: Pleroma.Web.Endpoint.url() <> "/media/background.png", + chat_limit: 120, + description_limit: 120, + shout_limit: 120 + }, + stats: %{mau: Pleroma.User.active_user_count()}, + vapid_public_key: Keyword.get(Pleroma.Web.Push.vapid_config(), :public_key) + } + + Map.equal?(expected, filtered_configuration) + end + # When this fails, a new feature flag was probably added. Add it to the macros above. describe "uncofigurable features" do setup do: Enum.each(@configurable_features_flags, &clear_config(&1, false)) @@ -303,7 +459,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do end end - describe "render show.json" do + describe "render show.json/show2.json" do setup do Pleroma.Rule.create(%{text: "rule 1"}) Pleroma.Rule.create(%{text: "rule 2"}) @@ -332,6 +488,23 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do clear_config([:instance, :birthday_required], true) clear_config([:instance, :birthday_min_age], 1) + clear_config([:instance, :short_description], "Corndog Emporium!") + clear_config([:instance, :instance_thumbnail], "https://example.com/thumbnail.png") + clear_config([:instance, :registrations_open], false) + clear_config([:instance, :account_approval_required], true) + + clear_config([:instance, :max_pinned_statuses], 1) + clear_config([:instance, :status_page], "https://status.example.com") + + clear_config([:instance, :email], "nonexist@example.com") + + clear_config([:instance, :avatar_upload_limit], 1024) + clear_config([:instance, :background_upload_limit], 1024) + clear_config([:instance, :banner_upload_limit], 1024) + clear_config([:instance, :background_image], "/media/background.png") + clear_config([:instance, :chat_limit], 120) + clear_config([:instance, :description_limit], 120) + clear_config([:media_proxy, :enabled], true) clear_config([:media_proxy, :base_url], "https://mediaproxy.example.com") @@ -341,10 +514,14 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do clear_config([:markup, :allow_headings], true) clear_config([:markup, :allow_tables], true) + clear_config([:restrict_unauthenticated, :timelines], %{local: true, federated: true}) + + clear_config([:shout, :limit], 120) + %{user: user} end - test "renders properly", %{user: user} do + test "renders show.json properly", %{user: user} do output = InstanceView.render("show.json", %{}) expected_keys = Enum.sort(@show_keys) @@ -354,6 +531,19 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do assert check_pleroma_configuration(output) assert expected_keys == Enum.sort(Enum.map(Map.keys(output), &to_string/1)) end + + test "renders show2.json properly", %{user: user} do + output = InstanceView.render("show2.json", %{}) + expected_keys = Enum.sort(@show2_keys) + + assert check_common_information(output) + assert check_thumbnail(output) + assert check_configuration2(output) + assert check_registrations(output) + assert check_contact(output, user) + assert check_pleroma_configuration2(output) + assert expected_keys == Enum.sort(Enum.map(Map.keys(output), &to_string/1)) + end end describe "render rule.json" do From 6393fe0a9e62051393c44acc1d66bd196616ea54 Mon Sep 17 00:00:00 2001 From: Phantasm Date: Thu, 25 Jun 2026 23:21:09 +0200 Subject: [PATCH 093/183] InstanceView: removing unneeded doubling of attributes in tests --- .../mastodon_api/views/instance_view_test.exs | 97 +++++++++---------- 1 file changed, 47 insertions(+), 50 deletions(-) diff --git a/test/pleroma/web/mastodon_api/views/instance_view_test.exs b/test/pleroma/web/mastodon_api/views/instance_view_test.exs index b4622191b..93b5c60c9 100644 --- a/test/pleroma/web/mastodon_api/views/instance_view_test.exs +++ b/test/pleroma/web/mastodon_api/views/instance_view_test.exs @@ -13,74 +13,71 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do import Mox - @atom_common_information_keys [ + @common_information_keys [ :languages, :rules, :title, :version ] - @common_information_keys Enum.map(@atom_common_information_keys, &to_string/1) - - @atom_configuration_keys [ + @configuration_keys [ :accounts, :statuses, :media_attachments, :polls ] - @atom_configuration2_keys @atom_configuration_keys ++ [ + @configuration2_keys @configuration_keys ++ [ :urls, :vapid, :translation, :timelines_access ] - @atom_pleroma_configuration_keys [ + @pleroma_configuration_keys [ :metadata, :stats, :vapid_public_key ] - # TODO: Simplify this, so ideally only atoms are macroed @show_keys @common_information_keys ++ [ - "uri", - "description", - "short_description", - "email", - "urls", - "stats", - "thumbnail", - "registrations", - "approval_required", - "contact_account", - "configuration", - # Extra (not present in Mastodon)" - "max_toot_chars", - "max_media_attachments", - "poll_limits", - "upload_limit", - "avatar_upload_limit", - "background_upload_limit", - "banner_upload_limit", - "background_image", - "shout_limit", - "description_limit", - "chat_limit", - "pleroma" + :uri, + :description, + :short_description, + :email, + :urls, + :stats, + :thumbnail, + :registrations, + :approval_required, + :contact_account, + :configuration, + # Extra (not present in Mastodon) + :max_toot_chars, + :max_media_attachments, + :poll_limits, + :upload_limit, + :avatar_upload_limit, + :background_upload_limit, + :banner_upload_limit, + :background_image, + :shout_limit, + :description_limit, + :chat_limit, + :pleroma ] @show2_keys @common_information_keys ++ [ - "domain", - "source_url", - "description", - "usage", - "thumbnail", - "configuration", - "registrations", - "contact", - # Extra (not present in Mastodon): - "pleroma" + :domain, + :source_url, + :description, + :usage, + :thumbnail, + :configuration, + :registrations, + :contact, + # Extra (not present in Mastodon) + :pleroma ] @features [ @@ -106,7 +103,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do "pleroma:block_expiration" ] - @configureable_features [ + @configurable_features [ "blockers_visible", "media_proxy", "gopher", @@ -130,7 +127,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do ] defp check_common_information(info) do - filtered_info = Map.reject(info, fn {key, _value} -> key not in @atom_common_information_keys end) + filtered_info = Map.reject(info, fn {key, _value} -> key not in @common_information_keys end) expected = %{ languages: ["en", "cs"], @@ -172,7 +169,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do defp check_configuration(info) do configuration = info[:configuration] - filtered_configuration = Map.reject(configuration, fn {key, _value} -> key not in @atom_configuration_keys end) + filtered_configuration = Map.reject(configuration, fn {key, _value} -> key not in @configuration_keys end) expected = %{ accounts: %{ @@ -200,7 +197,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do defp check_configuration2(info) do configuration = info[:configuration] - filtered_configuration = Map.reject(configuration, fn {key, _value} -> key not in @atom_configuration2_keys end) + filtered_configuration = Map.reject(configuration, fn {key, _value} -> key not in @configuration2_keys end) # configuration2 also includes parts from configuration expected = %{ @@ -264,7 +261,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do defp check_pleroma_configuration(info) do configuration = info[:pleroma] - filtered_configuration = Map.reject(configuration, fn {key, _value} -> key not in @atom_pleroma_configuration_keys end) + filtered_configuration = Map.reject(configuration, fn {key, _value} -> key not in @pleroma_configuration_keys end) metadata = Map.fetch!(filtered_configuration, :metadata) # Tested elsewhere already filtered_metadata = Map.reject(metadata, fn {key, _value} -> key in [:features, :federation] end) @@ -306,7 +303,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do defp check_pleroma_configuration2(info) do configuration = info[:pleroma] - filtered_configuration = Map.reject(configuration, fn {key, _value} -> key not in @atom_pleroma_configuration_keys end) + filtered_configuration = Map.reject(configuration, fn {key, _value} -> key not in @pleroma_configuration_keys end) metadata = Map.fetch!(filtered_configuration, :metadata) # Tested elsewhere already filtered_metadata = Map.reject(metadata, fn {key, _value} -> key in [:features, :federation] end) @@ -385,7 +382,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do end test "returns all features including configurable ones" do - features = @features ++ @configureable_features + features = @features ++ @configurable_features assert Enum.sort(InstanceView.features()) == Enum.sort(features) end @@ -529,7 +526,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do assert check_contact_name(output, user) assert check_configuration(output) assert check_pleroma_configuration(output) - assert expected_keys == Enum.sort(Enum.map(Map.keys(output), &to_string/1)) + assert expected_keys == Enum.sort(Map.keys(output)) end test "renders show2.json properly", %{user: user} do @@ -542,7 +539,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do assert check_registrations(output) assert check_contact(output, user) assert check_pleroma_configuration2(output) - assert expected_keys == Enum.sort(Enum.map(Map.keys(output), &to_string/1)) + assert expected_keys == Enum.sort(Map.keys(output)) end end From 93fe0f5ab5ed258d6b6e3b42e007ca37336218fb Mon Sep 17 00:00:00 2001 From: Phantasm Date: Fri, 26 Jun 2026 01:41:32 +0200 Subject: [PATCH 094/183] InstanceView: Filter out *configuration keys from *configuration2 --- .../mastodon_api/views/instance_view_test.exs | 88 ++++++++----------- 1 file changed, 39 insertions(+), 49 deletions(-) diff --git a/test/pleroma/web/mastodon_api/views/instance_view_test.exs b/test/pleroma/web/mastodon_api/views/instance_view_test.exs index 93b5c60c9..e6c8185c1 100644 --- a/test/pleroma/web/mastodon_api/views/instance_view_test.exs +++ b/test/pleroma/web/mastodon_api/views/instance_view_test.exs @@ -34,12 +34,26 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do :timelines_access ] + # Used for filtering out values from configuration.json render in configuration2.json render + @configuration2_filter [ + {:accounts, [:max_pinned_statuses, :max_profile_fields, :profile_field_name_limit, :profile_field_value_limit]}, + {:statuses, [:characters_reserved_per_url]}, + {:urls, [:streaming, :status]}, + {:vapid, [:public_key]}, + {:translation, [:enabled]}, + {:timelines_access, [:live_feeds, :hashtag_feeds, :trending_link_feeds]} + ] + @pleroma_configuration_keys [ :metadata, :stats, :vapid_public_key ] + @pleroma_configuration2_filter [ + {:metadata, [:avatar_upload_limit, :background_upload_limit, :banner_upload_limit, :background_image, :chat_limit, :description_limit, :shout_limit]} + ] + @show_keys @common_information_keys ++ [ :uri, :description, @@ -126,6 +140,17 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do [:instance, :profile_directory] ] + # Filters out values from one layer deep nested Maps. + defp filter_render(%{}, [], acc), do: acc + + defp filter_render(input, filter, acc) do + {{filter_key, filters}, filter_rest} = List.pop_at(filter, 0) + {config_key_values, input_rest} = Map.pop!(input, filter_key) + filtered = Map.reject(config_key_values, fn {key, _} -> key not in filters end) + + filter_render(input_rest, filter_rest, Map.put(acc, filter_key, filtered)) + end + defp check_common_information(info) do filtered_info = Map.reject(info, fn {key, _value} -> key not in @common_information_keys end) @@ -197,33 +222,21 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do defp check_configuration2(info) do configuration = info[:configuration] - filtered_configuration = Map.reject(configuration, fn {key, _value} -> key not in @configuration2_keys end) + filtered_configuration = + configuration + |> Map.reject(fn {key, _value} -> key not in @configuration2_keys end) + |> filter_render(@configuration2_filter, %{}) - # configuration2 also includes parts from configuration expected = %{ accounts: %{ - max_featured_tags: 0, max_pinned_statuses: 1, max_profile_fields: 1, profile_field_name_limit: 1, profile_field_value_limit: 1 }, statuses: %{ - max_characters: 4096, - max_media_attachments: 1, characters_reserved_per_url: 0 }, - media_attachments: %{ - image_size_limit: 1024, - video_size_limit: 1024, - supported_mime_types: ["application/octet-stream"] - }, - polls: %{ - max_options: 4, - max_characters_per_option: 120, - min_expiration: 1, - max_expiration: 2 - }, urls: %{ streaming: Pleroma.Web.Endpoint.websocket_url(), status: "https://status.example.com" @@ -262,8 +275,9 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do defp check_pleroma_configuration(info) do configuration = info[:pleroma] filtered_configuration = Map.reject(configuration, fn {key, _value} -> key not in @pleroma_configuration_keys end) - metadata = Map.fetch!(filtered_configuration, :metadata) + # Tested elsewhere already + metadata = Map.fetch!(filtered_configuration, :metadata) filtered_metadata = Map.reject(metadata, fn {key, _value} -> key in [:features, :federation] end) filtered_configuration = %{filtered_configuration | metadata: filtered_metadata} @@ -304,38 +318,16 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do defp check_pleroma_configuration2(info) do configuration = info[:pleroma] filtered_configuration = Map.reject(configuration, fn {key, _value} -> key not in @pleroma_configuration_keys end) - metadata = Map.fetch!(filtered_configuration, :metadata) - # Tested elsewhere already - filtered_metadata = Map.reject(metadata, fn {key, _value} -> key in [:features, :federation] end) - filtered_configuration = %{filtered_configuration | metadata: filtered_metadata} - # pleroma_configuration2 also includes parts from pleroma_configuration + # Tested elsewhere already + metadata = Map.fetch!(filtered_configuration, :metadata) + filtered_metadata = Map.reject(metadata, fn {key, _value} -> key in [:features, :federation] end) + filtered_configuration = + %{filtered_configuration | metadata: filtered_metadata} + |> filter_render(@pleroma_configuration2_filter, %{}) + expected = %{ metadata: %{ - account_activation_required: true, - fields_limits: %{ - max_fields: 1, - max_remote_fields: 1, - name_length: 1, - value_length: 1 - }, - post_formats: ["text/plain"], - birthday_required: true, - birthday_min_age: 1, - translation: - %{ - source_languages: ["en", "pl"], - target_languages: ["en", "pl"] - }, - base_urls: %{ - media_proxy: "https://mediaproxy.example.com", - upload: "https://upload.example.com" - }, - markup: %{ - allow_inline_images: true, - allow_headings: true, - allow_tables: true - }, avatar_upload_limit: 1024, background_upload_limit: 1024, banner_upload_limit: 1024, @@ -343,9 +335,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do chat_limit: 120, description_limit: 120, shout_limit: 120 - }, - stats: %{mau: Pleroma.User.active_user_count()}, - vapid_public_key: Keyword.get(Pleroma.Web.Push.vapid_config(), :public_key) + } } Map.equal?(expected, filtered_configuration) From 9912b2d16af01d322a1e3a4eb78efbd6f42efdf5 Mon Sep 17 00:00:00 2001 From: Phantasm Date: Fri, 26 Jun 2026 01:57:32 +0200 Subject: [PATCH 095/183] InstanceView: lint tests --- .../mastodon_api/views/instance_view_test.exs | 207 ++++++++++++------ 1 file changed, 136 insertions(+), 71 deletions(-) diff --git a/test/pleroma/web/mastodon_api/views/instance_view_test.exs b/test/pleroma/web/mastodon_api/views/instance_view_test.exs index e6c8185c1..cae704b90 100644 --- a/test/pleroma/web/mastodon_api/views/instance_view_test.exs +++ b/test/pleroma/web/mastodon_api/views/instance_view_test.exs @@ -27,16 +27,23 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do :polls ] - @configuration2_keys @configuration_keys ++ [ - :urls, - :vapid, - :translation, - :timelines_access - ] + @configuration2_keys @configuration_keys ++ + [ + :urls, + :vapid, + :translation, + :timelines_access + ] # Used for filtering out values from configuration.json render in configuration2.json render @configuration2_filter [ - {:accounts, [:max_pinned_statuses, :max_profile_fields, :profile_field_name_limit, :profile_field_value_limit]}, + {:accounts, + [ + :max_pinned_statuses, + :max_profile_fields, + :profile_field_name_limit, + :profile_field_value_limit + ]}, {:statuses, [:characters_reserved_per_url]}, {:urls, [:streaming, :status]}, {:vapid, [:public_key]}, @@ -51,48 +58,59 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do ] @pleroma_configuration2_filter [ - {:metadata, [:avatar_upload_limit, :background_upload_limit, :banner_upload_limit, :background_image, :chat_limit, :description_limit, :shout_limit]} + {:metadata, + [ + :avatar_upload_limit, + :background_upload_limit, + :banner_upload_limit, + :background_image, + :chat_limit, + :description_limit, + :shout_limit + ]} ] - @show_keys @common_information_keys ++ [ - :uri, - :description, - :short_description, - :email, - :urls, - :stats, - :thumbnail, - :registrations, - :approval_required, - :contact_account, - :configuration, - # Extra (not present in Mastodon) - :max_toot_chars, - :max_media_attachments, - :poll_limits, - :upload_limit, - :avatar_upload_limit, - :background_upload_limit, - :banner_upload_limit, - :background_image, - :shout_limit, - :description_limit, - :chat_limit, - :pleroma - ] + @show_keys @common_information_keys ++ + [ + :uri, + :description, + :short_description, + :email, + :urls, + :stats, + :thumbnail, + :registrations, + :approval_required, + :contact_account, + :configuration, + # Extra (not present in Mastodon) + :max_toot_chars, + :max_media_attachments, + :poll_limits, + :upload_limit, + :avatar_upload_limit, + :background_upload_limit, + :banner_upload_limit, + :background_image, + :shout_limit, + :description_limit, + :chat_limit, + :pleroma + ] - @show2_keys @common_information_keys ++ [ - :domain, - :source_url, - :description, - :usage, - :thumbnail, - :configuration, - :registrations, - :contact, - # Extra (not present in Mastodon) - :pleroma - ] + @show2_keys @common_information_keys ++ + [ + :domain, + :source_url, + :description, + :usage, + :thumbnail, + :configuration, + :registrations, + :contact, + # Extra (not present in Mastodon) + :pleroma + ] @features [ "pleroma_api", @@ -158,7 +176,9 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do languages: ["en", "cs"], rules: InstanceView.render("rules.json", %{}), title: "test title", - version: InstanceView.mastodon_api_level() <> " (compatible; #{Pleroma.Application.named_version()})" + version: + InstanceView.mastodon_api_level() <> + " (compatible; #{Pleroma.Application.named_version()})" } Map.equal?(expected, filtered_info) @@ -194,7 +214,9 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do defp check_configuration(info) do configuration = info[:configuration] - filtered_configuration = Map.reject(configuration, fn {key, _value} -> key not in @configuration_keys end) + + filtered_configuration = + Map.reject(configuration, fn {key, _value} -> key not in @configuration_keys end) expected = %{ accounts: %{ @@ -222,6 +244,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do defp check_configuration2(info) do configuration = info[:configuration] + filtered_configuration = configuration |> Map.reject(fn {key, _value} -> key not in @configuration2_keys end) @@ -274,11 +297,16 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do defp check_pleroma_configuration(info) do configuration = info[:pleroma] - filtered_configuration = Map.reject(configuration, fn {key, _value} -> key not in @pleroma_configuration_keys end) + + filtered_configuration = + Map.reject(configuration, fn {key, _value} -> key not in @pleroma_configuration_keys end) # Tested elsewhere already metadata = Map.fetch!(filtered_configuration, :metadata) - filtered_metadata = Map.reject(metadata, fn {key, _value} -> key in [:features, :federation] end) + + filtered_metadata = + Map.reject(metadata, fn {key, _value} -> key in [:features, :federation] end) + filtered_configuration = %{filtered_configuration | metadata: filtered_metadata} expected = %{ @@ -293,19 +321,18 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do post_formats: ["text/plain"], birthday_required: true, birthday_min_age: 1, - translation: - %{ - source_languages: ["en", "pl"], - target_languages: ["en", "pl"] - }, + translation: %{ + source_languages: ["en", "pl"], + target_languages: ["en", "pl"] + }, base_urls: %{ media_proxy: "https://mediaproxy.example.com", upload: "https://upload.example.com" }, markup: %{ - allow_inline_images: true, - allow_headings: true, - allow_tables: true + allow_inline_images: true, + allow_headings: true, + allow_tables: true } }, stats: %{mau: Pleroma.User.active_user_count()}, @@ -317,11 +344,16 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do defp check_pleroma_configuration2(info) do configuration = info[:pleroma] - filtered_configuration = Map.reject(configuration, fn {key, _value} -> key not in @pleroma_configuration_keys end) + + filtered_configuration = + Map.reject(configuration, fn {key, _value} -> key not in @pleroma_configuration_keys end) # Tested elsewhere already metadata = Map.fetch!(filtered_configuration, :metadata) - filtered_metadata = Map.reject(metadata, fn {key, _value} -> key in [:features, :federation] end) + + filtered_metadata = + Map.reject(metadata, fn {key, _value} -> key in [:features, :federation] end) + filtered_configuration = %{filtered_configuration | metadata: filtered_metadata} |> filter_render(@pleroma_configuration2_filter, %{}) @@ -386,14 +418,26 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do output = InstanceView.federation() assert %{quarantined_instances: ["quarantine.example.com"]} = output - assert %{quarantined_instances_info: %{"quarantined_instances" => %{"quarantine.example.com" => %{"reason" => "crimes"}}}} = output + + assert %{ + quarantined_instances_info: %{ + "quarantined_instances" => %{"quarantine.example.com" => %{"reason" => "crimes"}} + } + } = output end test "rejected instances" do - clear_config([:instance, :rejected_instances], [{"rejected.example.com", "not enough #cofe posting"}]) + clear_config([:instance, :rejected_instances], [ + {"rejected.example.com", "not enough #cofe posting"} + ]) output = InstanceView.federation() - assert %{rejected_instances: %{"rejected.example.com" => %{"reason" => "not enough #cofe posting"}}} = output + + assert %{ + rejected_instances: %{ + "rejected.example.com" => %{"reason" => "not enough #cofe posting"} + } + } = output end test "federating" do @@ -405,29 +449,39 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do test "transparency" do clear_config([:instance, :quarantined_instances], [{"quarantine.example.com", "crimes"}]) - clear_config([:instance, :rejected_instances], [{"rejected.example.com", "not enough #cofe posting"}]) clear_config([:instance, :federating], true) clear_config([:mrf, :transparency], false) + clear_config([:instance, :rejected_instances], [ + {"rejected.example.com", "not enough #cofe posting"} + ]) + output = InstanceView.federation() assert %{enabled: true} == output end test "MRFs" do clear_config([:mrf, :policies], [Pleroma.Web.ActivityPub.MRF.SimplePolicy]) - clear_config([:mrf, :transparency_exclusions], [{"media1.example.com", "the Fediverse doesn't need to know"}]) - clear_config([:mrf_simple, :media_removal], [{"media1.example.com", "NSFW"}, {"media2.example.com", "usual suspects"}]) clear_config([:mrf_simple, :reject], [{"rejected.example.com", "not enough #cofe posting"}]) + clear_config([:mrf, :transparency_exclusions], [ + {"media1.example.com", "the Fediverse doesn't need to know"} + ]) + + clear_config([:mrf_simple, :media_removal], [ + {"media1.example.com", "NSFW"}, + {"media2.example.com", "usual suspects"} + ]) + output = InstanceView.federation() expected = %{ mrf_simple: %{ reject: ["rejected.example.com"], - media_removal: ["media2.example.com"], + media_removal: ["media2.example.com"] }, mrf_hashtag: %{ - sensitive: ["nsfw"], + sensitive: ["nsfw"] }, exclusions: true, mrf_policies: ["SimplePolicy", "HashtagPolicy"], @@ -569,10 +623,21 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do describe "render domain_blocks.json" do setup do clear_config([:mrf, :transparency], true) - clear_config([:mrf, :transparency_exclusions], [{"removed1.example.com", "the Fediverse doesn't need to know"}]) clear_config([:mrf_simple, :media_removal], [{"media1.example.com", "NSFW"}]) - clear_config([:mrf_simple, :reject], [{"rejected.example.com", "not enough #cofe posting"}, {"rejected-without-reason.example.com", ""}]) - clear_config([:mrf_simple, :federated_timeline_removal], [{"removed1.example.com", "spam"}, {"removed2.example.com", "more spam"}]) + + clear_config([:mrf, :transparency_exclusions], [ + {"removed1.example.com", "the Fediverse doesn't need to know"} + ]) + + clear_config([:mrf_simple, :reject], [ + {"rejected.example.com", "not enough #cofe posting"}, + {"rejected-without-reason.example.com", ""} + ]) + + clear_config([:mrf_simple, :federated_timeline_removal], [ + {"removed1.example.com", "spam"}, + {"removed2.example.com", "more spam"} + ]) :ok end From 6c79cc792f856027fb8c1e9372ede341272d166f Mon Sep 17 00:00:00 2001 From: Phantasm Date: Sat, 27 Jun 2026 22:41:45 +0200 Subject: [PATCH 096/183] InstanceView: improve filtering in tests, simplify --- .../mastodon_api/views/instance_view_test.exs | 108 +++++++----------- 1 file changed, 42 insertions(+), 66 deletions(-) diff --git a/test/pleroma/web/mastodon_api/views/instance_view_test.exs b/test/pleroma/web/mastodon_api/views/instance_view_test.exs index cae704b90..7bdd9a126 100644 --- a/test/pleroma/web/mastodon_api/views/instance_view_test.exs +++ b/test/pleroma/web/mastodon_api/views/instance_view_test.exs @@ -20,21 +20,6 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do :version ] - @configuration_keys [ - :accounts, - :statuses, - :media_attachments, - :polls - ] - - @configuration2_keys @configuration_keys ++ - [ - :urls, - :vapid, - :translation, - :timelines_access - ] - # Used for filtering out values from configuration.json render in configuration2.json render @configuration2_filter [ {:accounts, @@ -51,12 +36,6 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do {:timelines_access, [:live_feeds, :hashtag_feeds, :trending_link_feeds]} ] - @pleroma_configuration_keys [ - :metadata, - :stats, - :vapid_public_key - ] - @pleroma_configuration2_filter [ {:metadata, [ @@ -159,14 +138,37 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do ] # Filters out values from one layer deep nested Maps. - defp filter_render(%{}, [], acc), do: acc + # Filtering types: + # - :inclusive -> filters out everything that is not in the filters + # - :discriminative -> filters out what is in the filters + defp filter_render(input, filter, type, acc) - defp filter_render(input, filter, acc) do - {{filter_key, filters}, filter_rest} = List.pop_at(filter, 0) - {config_key_values, input_rest} = Map.pop!(input, filter_key) - filtered = Map.reject(config_key_values, fn {key, _} -> key not in filters end) + defp filter_render(_input_rest, [], type, acc) when type == :inclusive, do: acc - filter_render(input_rest, filter_rest, Map.put(acc, filter_key, filtered)) + defp filter_render(input_rest, [], type, acc) when type == :discriminative, + do: Map.merge(input_rest, acc) + + defp filter_render(input, filter, type, acc) do + {current_filter, filter_rest} = List.pop_at(filter, 0) + + case current_filter do + nil -> + filter_render(input, filter_rest, type, acc) + + {filter_key, filters} -> + {config_key_values, input_rest} = Map.pop!(input, filter_key) + + filtered = + case type do + :inclusive -> + Map.reject(config_key_values, fn {key, _} -> key not in filters end) + + :discriminative -> + Map.reject(config_key_values, fn {key, _} -> key in filters end) + end + + filter_render(input_rest, filter_rest, type, Map.put(acc, filter_key, filtered)) + end end defp check_common_information(info) do @@ -215,9 +217,6 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do defp check_configuration(info) do configuration = info[:configuration] - filtered_configuration = - Map.reject(configuration, fn {key, _value} -> key not in @configuration_keys end) - expected = %{ accounts: %{ max_featured_tags: 0 @@ -239,16 +238,13 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do } } - Map.equal?(expected, filtered_configuration) + Map.equal?(expected, configuration) end defp check_configuration2(info) do - configuration = info[:configuration] - - filtered_configuration = - configuration - |> Map.reject(fn {key, _value} -> key not in @configuration2_keys end) - |> filter_render(@configuration2_filter, %{}) + configuration = + info[:configuration] + |> filter_render(@configuration2_filter, :inclusive, %{}) expected = %{ accounts: %{ @@ -279,7 +275,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do } } - Map.equal?(expected, filtered_configuration) + Map.equal?(expected, configuration) end defp check_registrations(info) do @@ -296,18 +292,9 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do end defp check_pleroma_configuration(info) do - configuration = info[:pleroma] - - filtered_configuration = - Map.reject(configuration, fn {key, _value} -> key not in @pleroma_configuration_keys end) - - # Tested elsewhere already - metadata = Map.fetch!(filtered_configuration, :metadata) - - filtered_metadata = - Map.reject(metadata, fn {key, _value} -> key in [:features, :federation] end) - - filtered_configuration = %{filtered_configuration | metadata: filtered_metadata} + configuration = + info[:pleroma] + |> filter_render([{:metadata, [:features, :federation]}], :discriminative, %{}) expected = %{ metadata: %{ @@ -339,24 +326,13 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do vapid_public_key: Keyword.get(Pleroma.Web.Push.vapid_config(), :public_key) } - Map.equal?(expected, filtered_configuration) + Map.equal?(expected, configuration) end defp check_pleroma_configuration2(info) do - configuration = info[:pleroma] - - filtered_configuration = - Map.reject(configuration, fn {key, _value} -> key not in @pleroma_configuration_keys end) - - # Tested elsewhere already - metadata = Map.fetch!(filtered_configuration, :metadata) - - filtered_metadata = - Map.reject(metadata, fn {key, _value} -> key in [:features, :federation] end) - - filtered_configuration = - %{filtered_configuration | metadata: filtered_metadata} - |> filter_render(@pleroma_configuration2_filter, %{}) + configuration = + info[:pleroma] + |> filter_render(@pleroma_configuration2_filter, :inclusive, %{}) expected = %{ metadata: %{ @@ -370,7 +346,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do } } - Map.equal?(expected, filtered_configuration) + Map.equal?(expected, configuration) end # When this fails, a new feature flag was probably added. Add it to the macros above. From ef321846d4bc4860789ce24edeeeda6384fede67 Mon Sep 17 00:00:00 2001 From: Phantasm Date: Sat, 27 Jun 2026 22:44:59 +0200 Subject: [PATCH 097/183] InstanceView: Use attribute for pleroma_configuration filter in test --- test/pleroma/web/mastodon_api/views/instance_view_test.exs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/pleroma/web/mastodon_api/views/instance_view_test.exs b/test/pleroma/web/mastodon_api/views/instance_view_test.exs index 7bdd9a126..f7d058365 100644 --- a/test/pleroma/web/mastodon_api/views/instance_view_test.exs +++ b/test/pleroma/web/mastodon_api/views/instance_view_test.exs @@ -36,6 +36,8 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do {:timelines_access, [:live_feeds, :hashtag_feeds, :trending_link_feeds]} ] + @pleroma_configuration_filter [{:metadata, [:features, :federation]}] + @pleroma_configuration2_filter [ {:metadata, [ @@ -294,7 +296,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do defp check_pleroma_configuration(info) do configuration = info[:pleroma] - |> filter_render([{:metadata, [:features, :federation]}], :discriminative, %{}) + |> filter_render(@pleroma_configuration_filter, :discriminative, %{}) expected = %{ metadata: %{ From 6e03f1d7898b8b9cc5ccfe40d81954ef91d24f5c Mon Sep 17 00:00:00 2001 From: Phantasm Date: Sat, 27 Jun 2026 23:34:01 +0200 Subject: [PATCH 098/183] InstanceView: Tighten show.json render test, rename show2-only funcs --- .../mastodon_api/views/instance_view_test.exs | 53 ++++++++++++++++--- 1 file changed, 47 insertions(+), 6 deletions(-) diff --git a/test/pleroma/web/mastodon_api/views/instance_view_test.exs b/test/pleroma/web/mastodon_api/views/instance_view_test.exs index f7d058365..b0ef4b659 100644 --- a/test/pleroma/web/mastodon_api/views/instance_view_test.exs +++ b/test/pleroma/web/mastodon_api/views/instance_view_test.exs @@ -79,6 +79,9 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do :pleroma ] + # Checked in other check_* functions + @show_filter @common_information_keys ++ [:contact_account, :configuration, :pleroma] + @show2_keys @common_information_keys ++ [ :domain, @@ -188,7 +191,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do Map.equal?(expected, filtered_info) end - defp check_thumbnail(info) do + defp check_thumbnail2(info) do thumbnail = info[:thumbnail] expected = %{ @@ -209,7 +212,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do Map.equal?(expected, contact) end - defp check_contact_name(info, user) do + defp check_contact_account(info, user) do contact = info[:contact_account] view = Pleroma.Web.MastodonAPI.AccountView.render("show.json", %{user: user, for: nil}) @@ -280,7 +283,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do Map.equal?(expected, configuration) end - defp check_registrations(info) do + defp check_registrations2(info) do registrations = info[:registrations] expected = %{ @@ -351,6 +354,43 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do Map.equal?(expected, configuration) end + defp check_show(info) do + filter = @show_filter + filtered_info = Map.reject(info, fn {key, _value} -> key in filter end) + + expected = %{ + uri: Pleroma.Web.WebFinger.host(), + description: "Pleroma: An efficient and flexible fediverse server", + short_description: "Corndog Emporium!", + email: "nonexist@example.com", + urls: %{ + streaming_api: Pleroma.Web.Endpoint.websocket_url(), + }, + stats: Pleroma.Stats.get_stats(), + thumbnail: "https://example.com/thumbnail.png", + registrations: false, + approval_required: true, + max_toot_chars: 4096, + max_media_attachments: 1, + poll_limits: %{ + max_options: 4, + max_option_chars: 120, + min_expiration: 1, + max_expiration: 2 + }, + upload_limit: 1024, + avatar_upload_limit: 1024, + background_upload_limit: 1024, + banner_upload_limit: 1024, + background_image: Pleroma.Web.Endpoint.url() <> "/media/background.png", + shout_limit: 120, + description_limit: 120, + chat_limit: 120, + } + + Map.equal?(expected, filtered_info) + end + # When this fails, a new feature flag was probably added. Add it to the macros above. describe "uncofigurable features" do setup do: Enum.each(@configurable_features_flags, &clear_config(&1, false)) @@ -545,7 +585,8 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do expected_keys = Enum.sort(@show_keys) assert check_common_information(output) - assert check_contact_name(output, user) + assert check_show(output) + assert check_contact_account(output, user) assert check_configuration(output) assert check_pleroma_configuration(output) assert expected_keys == Enum.sort(Map.keys(output)) @@ -556,9 +597,9 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do expected_keys = Enum.sort(@show2_keys) assert check_common_information(output) - assert check_thumbnail(output) + assert check_thumbnail2(output) assert check_configuration2(output) - assert check_registrations(output) + assert check_registrations2(output) assert check_contact(output, user) assert check_pleroma_configuration2(output) assert expected_keys == Enum.sort(Map.keys(output)) From d19a9f768c996c845222805d6d6dd9b648580d95 Mon Sep 17 00:00:00 2001 From: Phantasm Date: Sat, 27 Jun 2026 23:39:48 +0200 Subject: [PATCH 099/183] InstanceView: Tighten show2.json --- .../mastodon_api/views/instance_view_test.exs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/test/pleroma/web/mastodon_api/views/instance_view_test.exs b/test/pleroma/web/mastodon_api/views/instance_view_test.exs index b0ef4b659..3f7c83968 100644 --- a/test/pleroma/web/mastodon_api/views/instance_view_test.exs +++ b/test/pleroma/web/mastodon_api/views/instance_view_test.exs @@ -96,6 +96,8 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do :pleroma ] + @show2_filter @common_information_keys ++ [:thumbnail, :configuration, :registrations, :contact, :pleroma] + @features [ "pleroma_api", "mastodon_api", @@ -391,6 +393,20 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do Map.equal?(expected, filtered_info) end + defp check_show2(info) do + filter = @show2_filter + filtered_info = Map.reject(info, fn {key, _value} -> key in filter end) + + expected = %{ + domain: Pleroma.Web.WebFinger.host(), + source_url: Pleroma.Application.repository(), + description: "Corndog Emporium!", + usage: %{users: %{active_month: Pleroma.User.active_user_count()}} + } + + Map.equal?(expected, filtered_info) + end + # When this fails, a new feature flag was probably added. Add it to the macros above. describe "uncofigurable features" do setup do: Enum.each(@configurable_features_flags, &clear_config(&1, false)) @@ -597,6 +613,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do expected_keys = Enum.sort(@show2_keys) assert check_common_information(output) + assert check_show2(output) assert check_thumbnail2(output) assert check_configuration2(output) assert check_registrations2(output) From 3027052b0dbc042af45135b041754b05ffe4893d Mon Sep 17 00:00:00 2001 From: Phantasm Date: Sun, 28 Jun 2026 00:17:28 +0200 Subject: [PATCH 100/183] InstanceView: Remove wall of changed configs in show render tests. Switched to using config defaults instead. --- .../mastodon_api/views/instance_view_test.exs | 180 +++++++----------- 1 file changed, 70 insertions(+), 110 deletions(-) diff --git a/test/pleroma/web/mastodon_api/views/instance_view_test.exs b/test/pleroma/web/mastodon_api/views/instance_view_test.exs index 3f7c83968..6acfbab99 100644 --- a/test/pleroma/web/mastodon_api/views/instance_view_test.exs +++ b/test/pleroma/web/mastodon_api/views/instance_view_test.exs @@ -182,9 +182,9 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do filtered_info = Map.reject(info, fn {key, _value} -> key not in @common_information_keys end) expected = %{ - languages: ["en", "cs"], + languages: ["en"], rules: InstanceView.render("rules.json", %{}), - title: "test title", + title: "Pleroma", version: InstanceView.mastodon_api_level() <> " (compatible; #{Pleroma.Application.named_version()})" @@ -197,7 +197,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do thumbnail = info[:thumbnail] expected = %{ - url: URI.merge(Pleroma.Web.Endpoint.url(), "https://example.com/thumbnail.png") |> to_string + url: URI.merge(Pleroma.Web.Endpoint.url(), "http://localhost:4001/instance/thumbnail.jpeg") |> to_string } expected == thumbnail @@ -207,7 +207,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do contact = info[:contact] expected = %{ - email: "nonexist@example.com", + email: "admin@example.com", account: Pleroma.Web.MastodonAPI.AccountView.render("show.json", %{user: user, for: nil}) } @@ -229,19 +229,19 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do max_featured_tags: 0 }, statuses: %{ - max_characters: 4096, - max_media_attachments: 1 + max_characters: 5_000, + max_media_attachments: 1_000 }, media_attachments: %{ - image_size_limit: 1024, - video_size_limit: 1024, + image_size_limit: 16_000_000, + video_size_limit: 16_000_000, supported_mime_types: ["application/octet-stream"] }, polls: %{ - max_options: 4, - max_characters_per_option: 120, - min_expiration: 1, - max_expiration: 2 + max_options: 20, + max_characters_per_option: 200, + min_expiration: 0, + max_expiration: 365 * 24 * 60 * 60 } } @@ -256,24 +256,24 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do expected = %{ accounts: %{ max_pinned_statuses: 1, - max_profile_fields: 1, - profile_field_name_limit: 1, - profile_field_value_limit: 1 + max_profile_fields: 10, + profile_field_name_limit: 512, + profile_field_value_limit: 2048 }, statuses: %{ characters_reserved_per_url: 0 }, urls: %{ streaming: Pleroma.Web.Endpoint.websocket_url(), - status: "https://status.example.com" + status: nil }, vapid: %{ public_key: Keyword.get(Pleroma.Web.Push.vapid_config(), :public_key) }, translation: %{enabled: true}, timelines_access: %{ - live_feeds: %{local: "authenticated", remote: "authenticated"}, - hashtag_feeds: %{local: "authenticated", remote: "authenticated"}, + live_feeds: %{local: "public", remote: "public"}, + hashtag_feeds: %{local: "public", remote: "public"}, # not implemented in Pleroma trending_link_feeds: %{ local: "disabled", @@ -289,8 +289,8 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do registrations = info[:registrations] expected = %{ - enabled: false, - approval_required: true, + enabled: true, + approval_required: false, message: nil, url: nil } @@ -305,28 +305,31 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do expected = %{ metadata: %{ - account_activation_required: true, + account_activation_required: false, fields_limits: %{ - max_fields: 1, - max_remote_fields: 1, - name_length: 1, - value_length: 1 + max_fields: 10, + max_remote_fields: 20, + name_length: 512, + value_length: 2048 }, - post_formats: ["text/plain"], - birthday_required: true, - birthday_min_age: 1, + post_formats: [ + "text/plain", + "text/html", + "text/markdown", + "text/bbcode", + "text/x.misskeymarkdown" + ], + birthday_required: false, + birthday_min_age: 0, translation: %{ source_languages: ["en", "pl"], target_languages: ["en", "pl"] }, - base_urls: %{ - media_proxy: "https://mediaproxy.example.com", - upload: "https://upload.example.com" - }, + base_urls: %{}, markup: %{ allow_inline_images: true, - allow_headings: true, - allow_tables: true + allow_headings: false, + allow_tables: false } }, stats: %{mau: Pleroma.User.active_user_count()}, @@ -343,13 +346,13 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do expected = %{ metadata: %{ - avatar_upload_limit: 1024, - background_upload_limit: 1024, - banner_upload_limit: 1024, - background_image: Pleroma.Web.Endpoint.url() <> "/media/background.png", - chat_limit: 120, - description_limit: 120, - shout_limit: 120 + avatar_upload_limit: 2_000_000, + background_upload_limit: 4_000_000, + banner_upload_limit: 4_000_000, + background_image: Pleroma.Web.Endpoint.url() <> "/images/city.jpg", + chat_limit: 5_000, + description_limit: 5_000, + shout_limit: 5_000 } } @@ -363,31 +366,31 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do expected = %{ uri: Pleroma.Web.WebFinger.host(), description: "Pleroma: An efficient and flexible fediverse server", - short_description: "Corndog Emporium!", - email: "nonexist@example.com", + short_description: "", + email: "admin@example.com", urls: %{ streaming_api: Pleroma.Web.Endpoint.websocket_url(), }, stats: Pleroma.Stats.get_stats(), - thumbnail: "https://example.com/thumbnail.png", - registrations: false, - approval_required: true, - max_toot_chars: 4096, - max_media_attachments: 1, + thumbnail: "http://localhost:4001/instance/thumbnail.jpeg", + registrations: true, + approval_required: false, + max_toot_chars: 5_000, + max_media_attachments: 1_000, poll_limits: %{ - max_options: 4, - max_option_chars: 120, - min_expiration: 1, - max_expiration: 2 + max_options: 20, + max_option_chars: 200, + min_expiration: 0, + max_expiration: 365 * 24 * 60 * 60 }, - upload_limit: 1024, - avatar_upload_limit: 1024, - background_upload_limit: 1024, - banner_upload_limit: 1024, - background_image: Pleroma.Web.Endpoint.url() <> "/media/background.png", - shout_limit: 120, - description_limit: 120, - chat_limit: 120, + upload_limit: 16_000_000, + avatar_upload_limit: 2_000_000, + background_upload_limit: 4_000_000, + banner_upload_limit: 4_000_000, + background_image: Pleroma.Web.Endpoint.url() <> "/images/city.jpg", + shout_limit: 5_000, + description_limit: 5_000, + chat_limit: 5_000, } Map.equal?(expected, filtered_info) @@ -400,7 +403,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do expected = %{ domain: Pleroma.Web.WebFinger.host(), source_url: Pleroma.Application.repository(), - description: "Corndog Emporium!", + description: "", usage: %{users: %{active_month: Pleroma.User.active_user_count()}} } @@ -541,57 +544,14 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do user = insert(:user) - # Wall of configs for testing whether they are rendered properly + # contact in show2.json + clear_config([:instance, :contact_username], user.nickname) + + # Pleroma.Language.Translation.configured?() -> true clear_config([Pleroma.Language.Translation, :provider], TranslationMock) - clear_config([:instance, :languages], ["en", "cs"]) - clear_config([:instance, :name], "test title") - clear_config([:instance, :contact_username], user.nickname) - clear_config([:instance, :limit], 4096) - clear_config([:instance, :max_media_attachments], 1) - clear_config([:instance, :upload_limit], 1024) - clear_config([:instance, :poll_limits, :max_options], 4) - clear_config([:instance, :poll_limits, :max_option_chars], 120) - clear_config([:instance, :poll_limits, :min_expiration], 1) - clear_config([:instance, :poll_limits, :max_expiration], 2) - clear_config([:instance, :account_activation_required], true) - clear_config([:instance, :max_account_fields], 1) - clear_config([:instance, :max_remote_account_fields], 1) - clear_config([:instance, :account_field_name_length], 1) - clear_config([:instance, :account_field_value_length], 1) - clear_config([:instance, :allowed_post_formats], ["text/plain"]) - clear_config([:instance, :birthday_required], true) - clear_config([:instance, :birthday_min_age], 1) - - clear_config([:instance, :short_description], "Corndog Emporium!") - clear_config([:instance, :instance_thumbnail], "https://example.com/thumbnail.png") - clear_config([:instance, :registrations_open], false) - clear_config([:instance, :account_approval_required], true) - - clear_config([:instance, :max_pinned_statuses], 1) - clear_config([:instance, :status_page], "https://status.example.com") - - clear_config([:instance, :email], "nonexist@example.com") - - clear_config([:instance, :avatar_upload_limit], 1024) - clear_config([:instance, :background_upload_limit], 1024) - clear_config([:instance, :banner_upload_limit], 1024) - clear_config([:instance, :background_image], "/media/background.png") - clear_config([:instance, :chat_limit], 120) - clear_config([:instance, :description_limit], 120) - - clear_config([:media_proxy, :enabled], true) - clear_config([:media_proxy, :base_url], "https://mediaproxy.example.com") - - clear_config([Pleroma.Upload, :base_url], "https://upload.example.com") - - clear_config([:markup, :allow_inline_images], true) - clear_config([:markup, :allow_headings], true) - clear_config([:markup, :allow_tables], true) - - clear_config([:restrict_unauthenticated, :timelines], %{local: true, federated: true}) - - clear_config([:shout, :limit], 120) + # Deprecated, but still used in view + clear_config([:instance, :chat_limit], 5_000) %{user: user} end From dd070883778107e2dba73821ceb66aeefb3d6a82 Mon Sep 17 00:00:00 2001 From: Phantasm Date: Sun, 28 Jun 2026 00:19:38 +0200 Subject: [PATCH 101/183] InstanceView: lint tests --- .../mastodon_api/views/instance_view_test.exs | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/test/pleroma/web/mastodon_api/views/instance_view_test.exs b/test/pleroma/web/mastodon_api/views/instance_view_test.exs index 6acfbab99..8f066a6f5 100644 --- a/test/pleroma/web/mastodon_api/views/instance_view_test.exs +++ b/test/pleroma/web/mastodon_api/views/instance_view_test.exs @@ -96,7 +96,8 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do :pleroma ] - @show2_filter @common_information_keys ++ [:thumbnail, :configuration, :registrations, :contact, :pleroma] + @show2_filter @common_information_keys ++ + [:thumbnail, :configuration, :registrations, :contact, :pleroma] @features [ "pleroma_api", @@ -197,7 +198,9 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do thumbnail = info[:thumbnail] expected = %{ - url: URI.merge(Pleroma.Web.Endpoint.url(), "http://localhost:4001/instance/thumbnail.jpeg") |> to_string + url: + URI.merge(Pleroma.Web.Endpoint.url(), "http://localhost:4001/instance/thumbnail.jpeg") + |> to_string } expected == thumbnail @@ -313,11 +316,11 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do value_length: 2048 }, post_formats: [ - "text/plain", - "text/html", - "text/markdown", - "text/bbcode", - "text/x.misskeymarkdown" + "text/plain", + "text/html", + "text/markdown", + "text/bbcode", + "text/x.misskeymarkdown" ], birthday_required: false, birthday_min_age: 0, @@ -369,7 +372,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do short_description: "", email: "admin@example.com", urls: %{ - streaming_api: Pleroma.Web.Endpoint.websocket_url(), + streaming_api: Pleroma.Web.Endpoint.websocket_url() }, stats: Pleroma.Stats.get_stats(), thumbnail: "http://localhost:4001/instance/thumbnail.jpeg", @@ -390,7 +393,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do background_image: Pleroma.Web.Endpoint.url() <> "/images/city.jpg", shout_limit: 5_000, description_limit: 5_000, - chat_limit: 5_000, + chat_limit: 5_000 } Map.equal?(expected, filtered_info) From 964feb1e470ba666b70c9acd3d329043a849e030 Mon Sep 17 00:00:00 2001 From: Phantasm Date: Sun, 28 Jun 2026 00:36:43 +0200 Subject: [PATCH 102/183] InstanceView: Remove unreachable guards in filter_render --- .../mastodon_api/views/instance_view_test.exs | 27 +++++++------------ 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/test/pleroma/web/mastodon_api/views/instance_view_test.exs b/test/pleroma/web/mastodon_api/views/instance_view_test.exs index 8f066a6f5..66485be0e 100644 --- a/test/pleroma/web/mastodon_api/views/instance_view_test.exs +++ b/test/pleroma/web/mastodon_api/views/instance_view_test.exs @@ -157,26 +157,19 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do do: Map.merge(input_rest, acc) defp filter_render(input, filter, type, acc) do - {current_filter, filter_rest} = List.pop_at(filter, 0) + {{filter_key, filters}, filter_rest} = List.pop_at(filter, 0) + {config_key_values, input_rest} = Map.pop!(input, filter_key) - case current_filter do - nil -> - filter_render(input, filter_rest, type, acc) + filtered = + case type do + :inclusive -> + Map.reject(config_key_values, fn {key, _} -> key not in filters end) - {filter_key, filters} -> - {config_key_values, input_rest} = Map.pop!(input, filter_key) + :discriminative -> + Map.reject(config_key_values, fn {key, _} -> key in filters end) + end - filtered = - case type do - :inclusive -> - Map.reject(config_key_values, fn {key, _} -> key not in filters end) - - :discriminative -> - Map.reject(config_key_values, fn {key, _} -> key in filters end) - end - - filter_render(input_rest, filter_rest, type, Map.put(acc, filter_key, filtered)) - end + filter_render(input_rest, filter_rest, type, Map.put(acc, filter_key, filtered)) end defp check_common_information(info) do From a020b8a1901720238e8749d8487ddb218b82472b Mon Sep 17 00:00:00 2001 From: Phantasm Date: Mon, 29 Jun 2026 23:47:43 +0200 Subject: [PATCH 103/183] Elixir 1.19: Ensure default primary logger configg level is debug Setting the default handler and ex_syslogger handler levels does not necessarily have to set the primary logger log level, at least in the future. Without this, the primary log level might be lower than the configured console/syslog levels, which would cause some desired logs to not be logged. --- config/config.exs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/config/config.exs b/config/config.exs index 37a2590ca..80183a3d9 100644 --- a/config/config.exs +++ b/config/config.exs @@ -133,12 +133,17 @@ config :pleroma, Pleroma.Web.Endpoint, ] # Configures Elixir's Logger +# Primary config +config :logger, level: :debug + +# Console config config :logger, :default_handler, level: :debug config :logger, :default_formatter, format: "\n$time $metadata[$level] $message\n", metadata: [:actor, :path, :type, :user] +# Syslog config config :logger, :ex_syslogger, level: :debug, ident: "pleroma", From 7d9175350d0f2ce715048ac1795ec4790d192c3b Mon Sep 17 00:00:00 2001 From: Phantasm Date: Tue, 30 Jun 2026 00:24:48 +0200 Subject: [PATCH 104/183] Elixir 1.19: Completely overwrite formatter for default handler :logger.update_handler_config/3 can merge configs, :logger.set_handler_config/3 instead resets the rest to defaults. --- lib/pleroma/config/deprecation_warnings.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/config/deprecation_warnings.ex b/lib/pleroma/config/deprecation_warnings.ex index e77785483..8f8786ee7 100644 --- a/lib/pleroma/config/deprecation_warnings.ex +++ b/lib/pleroma/config/deprecation_warnings.ex @@ -435,7 +435,7 @@ defmodule Pleroma.Config.DeprecationWarnings do """) if log_level, do: :logger.update_handler_config(:default, :level, log_level) - :logger.update_handler_config(:default, :formatter, formatter) + :logger.set_handler_config(:default, :formatter, formatter) end end From f4b415c54e475f891cffe9363d4e19d58a062ad3 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 22 May 2026 12:12:11 -0700 Subject: [PATCH 105/183] Improve RSS compliance The core feed reader issue was that RSS 2.0 is a simple text element and can only appear once per . By having multiple tags with rel attributes, parsers were getting confused and resolving entries to the wrong URL. Now each item has exactly one : the post URL. Everything else uses valid or dedicated extension elements where appropriate. --- changelog.d/rss.fix | 1 + lib/pleroma/web/feed/feed_view.ex | 151 +++++++++++++++++- .../templates/feed/feed/_activity.atom.eex | 24 +-- .../web/templates/feed/feed/_activity.rss.eex | 35 ++-- .../feed/feed/_tag_activity.atom.eex | 32 ++-- .../templates/feed/feed/_tag_activity.xml.eex | 22 ++- .../web/templates/feed/feed/tag.atom.eex | 6 +- .../web/templates/feed/feed/tag.rss.eex | 5 + .../web/templates/feed/feed/user.atom.eex | 6 +- .../web/templates/feed/feed/user.rss.eex | 12 +- test/pleroma/web/feed/tag_controller_test.exs | 2 +- 11 files changed, 212 insertions(+), 84 deletions(-) create mode 100644 changelog.d/rss.fix diff --git a/changelog.d/rss.fix b/changelog.d/rss.fix new file mode 100644 index 000000000..a3455698a --- /dev/null +++ b/changelog.d/rss.fix @@ -0,0 +1 @@ +Fix RSS feeds to be W3C compliant and render better in RSS readers diff --git a/lib/pleroma/web/feed/feed_view.ex b/lib/pleroma/web/feed/feed_view.ex index e1ee33d62..a433b8403 100644 --- a/lib/pleroma/web/feed/feed_view.ex +++ b/lib/pleroma/web/feed/feed_view.ex @@ -97,6 +97,64 @@ defmodule Pleroma.Web.Feed.FeedView do end end + def rss_content_encoded(data) do + data + |> feed_content_encoded() + |> String.replace("]]>", "]]>") + end + + def atom_content_encoded(data), do: feed_content_encoded(data) + + defp feed_content_encoded(data) do + base_content = + case activity_content(data) do + "" -> + data["summary"] || data["type"] || + "" + |> escape() + + content -> + content + end + + attachments_html = + (data["attachment"] || []) + |> Enum.map(&rss_attachment_preview/1) + |> Enum.reject(&(&1 == "")) + |> Enum.join("

") + + [base_content, attachments_html] + |> Enum.reject(&(&1 == "")) + |> Enum.join("

") + end + + defp rss_attachment_preview(attachment) do + href = attachment_href(attachment) + + if is_binary(href) and href != "" do + media_type = attachment_type(attachment) || "application/octet-stream" + escaped_href = escape(href) + name = attachment["name"] || "Attachment" + escaped_name = escape(name) + + cond do + String.starts_with?(media_type, "image/") -> + ~s(

#{escaped_name}

) + + String.starts_with?(media_type, "video/") -> + ~s(

#{escaped_name}

) + + String.starts_with?(media_type, "audio/") -> + ~s(

#{escaped_name}

) + + true -> + ~s(

#{escaped_name}

) + end + else + "" + end + end + def activity_content(%{"content" => content}) do content |> String.replace(~r/[\n\r]/, "") @@ -106,16 +164,99 @@ defmodule Pleroma.Web.Feed.FeedView do def activity_context(activity), do: escape(activity.data["context"]) + def feed_self_url(conn), do: Phoenix.Controller.current_url(conn) + + def activity_link(activity, data) do + cond do + activity.local -> data["id"] || data["url"] + true -> data["external_url"] || data["id"] || data["url"] + end + end + def attachment_href(attachment) do - attachment["url"] - |> hd() + attachment + |> attachment_url_data() |> Map.get("href") end def attachment_type(attachment) do - attachment["url"] - |> hd() - |> Map.get("mediaType") + attachment + |> attachment_url_data() + |> Map.get("mediaType") || attachment["mediaType"] + end + + def attachment_size(attachment) do + attachment + |> attachment_url_data() + |> Map.get("size", 0) + end + + def attachment_size_positive(attachment) do + case attachment_size(attachment) do + size when is_integer(size) and size > 0 -> + size + + size when is_binary(size) -> + case Integer.parse(size) do + {parsed, ""} when parsed > 0 -> parsed + _ -> nil + end + + _ -> + nil + end + end + + def attachment_medium(attachment) do + case attachment_type(attachment) do + "image/" <> _rest -> "image" + "video/" <> _rest -> "video" + "audio/" <> _rest -> "audio" + _ -> "document" + end + end + + def attachment_previewable?(attachment) do + attachment_medium(attachment) in ["image", "video", "audio"] + end + + def attachment_description(attachment) do + attachment["name"] || "" + end + + def media_content_xml(attachment) do + href = escape(attachment_href(attachment) || "") + type = escape(attachment_type(attachment) || "application/octet-stream") + medium = escape(attachment_medium(attachment)) + + file_size_attr = + case attachment_size_positive(attachment) do + size when is_integer(size) -> ~s( fileSize="#{size}") + _ -> "" + end + + description_xml = + case attachment_description(attachment) do + "" -> + "" + + description -> + ~s(\n #{escape(description)}) + end + + """ + + nonadult#{description_xml} + + """ + end + + defp attachment_url_data(attachment) do + case attachment["url"] do + [first | _] when is_map(first) -> first + %{} = map -> map + _ -> %{} + end end def get_href(id) do diff --git a/lib/pleroma/web/templates/feed/feed/_activity.atom.eex b/lib/pleroma/web/templates/feed/feed/_activity.atom.eex index b774f7984..f87d97cb4 100644 --- a/lib/pleroma/web/templates/feed/feed/_activity.atom.eex +++ b/lib/pleroma/web/templates/feed/feed/_activity.atom.eex @@ -3,7 +3,7 @@ http://activitystrea.ms/schema/1.0/post <%= @data["id"] %> <%= activity_title(@data, Keyword.get(@feed_config, :post_title, %{})) %> - <%= activity_description(@data) %> + <%= escape(atom_content_encoded(@data)) %> <%= to_rfc3339(@data["published"]) %> <%= to_rfc3339(@data["published"]) %> @@ -15,35 +15,23 @@ <%= escape(@data["summary"]) %> <% end %> - <%= if @activity.local do %> - - - <% else %> - - <% end %> + + <%= for tag <- Pleroma.Object.hashtags(@object) do %> <% end %> <%= for attachment <- @data["attachment"] || [] do %> - + <%= unless attachment_previewable?(attachment) do %> + + <% end %> <% end %> <%= if @data["inReplyTo"] do %> <% end %> - <%= for id <- @activity.recipients do %> - <%= if id == Pleroma.Constants.as_public() do %> - - <% else %> - <%= unless Regex.match?(~r/^#{Pleroma.Web.Endpoint.url()}.+followers$/, id) do %> - - <% end %> - <% end %> - <% end %> - <%= for {emoji, file} <- @data["emoji"] || %{} do %> <% end %> diff --git a/lib/pleroma/web/templates/feed/feed/_activity.rss.eex b/lib/pleroma/web/templates/feed/feed/_activity.rss.eex index 7de98f736..a8c2aa342 100644 --- a/lib/pleroma/web/templates/feed/feed/_activity.rss.eex +++ b/lib/pleroma/web/templates/feed/feed/_activity.rss.eex @@ -1,45 +1,32 @@ http://activitystrea.ms/schema/1.0/note http://activitystrea.ms/schema/1.0/post - <%= @data["id"] %> + <%= @data["id"] %> <%= activity_title(@data, Keyword.get(@feed_config, :post_title, %{})) %> <%= activity_description(@data) %> + ]]> <%= to_rfc2822(@data["published"]) %> <%= activity_context(@activity) %> - <%= if @activity.local do %> - <%= @data["id"] %> - <% else %> - <%= @data["external_url"] %> - <% end %> - - <%= activity_context(@activity) %> + <%= activity_link(@activity, @data) %> <%= for tag <- Pleroma.Object.hashtags(@object) do %> - + <%= if tag && tag != "" do %> + <%= tag %> + <% end %> <% end %> <%= for attachment <- @data["attachment"] || [] do %> - + <%= size = attachment_size_positive(attachment) %> + <%= unless attachment_previewable?(attachment) do %> + + <% end %> + <%= media_content_xml(attachment) %> <% end %> <%= if @data["inReplyTo"] do %> <% end %> - - <%= for id <- @activity.recipients do %> - <%= if id == Pleroma.Constants.as_public() do %> - http://activityschema.org/collection/public - <% else %> - <%= unless Regex.match?(~r/^#{Pleroma.Web.Endpoint.url()}.+followers$/, id) do %> - <%= id %> - <% end %> - <% end %> - <% end %> - - <%= for {emoji, file} <- @data["emoji"] || %{} do %> - <%= file %> - <% end %> diff --git a/lib/pleroma/web/templates/feed/feed/_tag_activity.atom.eex b/lib/pleroma/web/templates/feed/feed/_tag_activity.atom.eex index 03c222975..ae762f9ed 100644 --- a/lib/pleroma/web/templates/feed/feed/_tag_activity.atom.eex +++ b/lib/pleroma/web/templates/feed/feed/_tag_activity.atom.eex @@ -6,7 +6,7 @@ <%= @data["id"] %> <%= activity_title(@data, Keyword.get(@feed_config, :post_title, %{})) %> - <%= activity_description(@data) %> + <%= escape(atom_content_encoded(@data)) %> <%= to_rfc3339(@data["published"]) %> <%= to_rfc3339(@data["published"]) %> @@ -15,34 +15,22 @@ <%= if @data["summary"] != "" do %> - <%= @data["summary"] %> + <%= escape(@data["summary"]) %> <% end %> - <%= if @activity.local do %> - - - <% else %> - - <% end %> - - <%= for id <- @activity.recipients do %> - <%= if id == Pleroma.Constants.as_public() do %> - - <% else %> - <%= unless Regex.match?(~r/^#{Pleroma.Web.Endpoint.url()}.+followers$/, id) do %> - - <% end %> - <% end %> - <% end %> + + <%= for tag <- Pleroma.Object.hashtags(@object) do %> <% end %> + <%= for attachment <- @data["attachment"] || [] do %> + <%= unless attachment_previewable?(attachment) do %> + + <% end %> + <% end %> + <%= for {emoji, file} <- @data["emoji"] || %{} do %> <% end %> diff --git a/lib/pleroma/web/templates/feed/feed/_tag_activity.xml.eex b/lib/pleroma/web/templates/feed/feed/_tag_activity.xml.eex index 1b8c34b87..6ae3c5a55 100644 --- a/lib/pleroma/web/templates/feed/feed/_tag_activity.xml.eex +++ b/lib/pleroma/web/templates/feed/feed/_tag_activity.xml.eex @@ -1,14 +1,22 @@ - <%= activity_title(@data, Keyword.get(@feed_config, :post_title, %{})) %> - - - <%= activity_context(@activity) %> - <%= activity_context(@activity) %> + <%= activity_link(@activity, @data) %> + <%= activity_link(@activity, @data) %> <%= to_rfc2822(@data["published"]) %> - + <%= activity_title(@data, Keyword.get(@feed_config, :post_title, %{})) %> <%= activity_description(@data) %> + ]]> + <%= for attachment <- @data["attachment"] || [] do %> - + <%= size = attachment_size_positive(attachment) %> + <%= unless attachment_previewable?(attachment) do %> + + <% end %> + <%= media_content_xml(attachment) %> <% end %> + <%= for tag <- Pleroma.Object.hashtags(@object) do %> + <%= if tag && tag != "" do %> + <%= tag %> + <% end %> + <% end %> diff --git a/lib/pleroma/web/templates/feed/feed/tag.atom.eex b/lib/pleroma/web/templates/feed/feed/tag.atom.eex index 3449c97ff..1646aeec7 100644 --- a/lib/pleroma/web/templates/feed/feed/tag.atom.eex +++ b/lib/pleroma/web/templates/feed/feed/tag.atom.eex @@ -9,10 +9,12 @@ <%= Routes.tag_feed_url(@conn, :feed, @tag) <> ".atom" %> #<%= @tag %> - <%= Gettext.dpgettext("static_pages", "tag feed description", "These are public toots tagged with #%{tag}. You can interact with them if you have an account anywhere in the fediverse.", tag: @tag) %> + <%= with subtitle when subtitle not in [nil, ""] <- Gettext.dpgettext("static_pages", "tag feed description", "These are public toots tagged with #%{tag}. You can interact with them if you have an account anywhere in the fediverse.", tag: @tag) do %> + <%= subtitle %> + <% end %> <%= feed_logo() %> <%= most_recent_update(@activities) %> - " type="application/atom+xml"/> + <%= for activity <- @activities do %> <%= render Phoenix.Controller.view_module(@conn), "_tag_activity.atom", Map.merge(assigns, prepare_activity(activity, actor: true)) %> diff --git a/lib/pleroma/web/templates/feed/feed/tag.rss.eex b/lib/pleroma/web/templates/feed/feed/tag.rss.eex index a87f9bf50..618b89ba3 100644 --- a/lib/pleroma/web/templates/feed/feed/tag.rss.eex +++ b/lib/pleroma/web/templates/feed/feed/tag.rss.eex @@ -1,6 +1,8 @@ @@ -8,7 +10,10 @@ <%= Gettext.dpgettext("static_pages", "tag feed description", "These are public toots tagged with #%{tag}. You can interact with them if you have an account anywhere in the fediverse.", tag: @tag) %> <%= "#{Routes.tag_feed_url(@conn, :feed, @tag)}.rss" %> <%= feed_logo() %> + <%= feed_logo() %> 2b90d9 + <%= most_recent_update(@activities) || to_rfc2822(NaiveDateTime.utc_now()) %> + <%= Pleroma.Application.named_version() %> <%= for activity <- @activities do %> <%= render Phoenix.Controller.view_module(@conn), "_tag_activity.xml", Map.merge(assigns, prepare_activity(activity)) %> <% end %> diff --git a/lib/pleroma/web/templates/feed/feed/user.atom.eex b/lib/pleroma/web/templates/feed/feed/user.atom.eex index c2c77cfed..cc6883727 100644 --- a/lib/pleroma/web/templates/feed/feed/user.atom.eex +++ b/lib/pleroma/web/templates/feed/feed/user.atom.eex @@ -8,10 +8,12 @@ <%= Routes.user_feed_url(@conn, :feed, @user.nickname) <> ".atom" %> <%= @user.nickname <> "'s timeline" %> - <%= escape(@user.bio) %> + <%= if @user.bio && @user.bio != "" do %> + <%= escape(@user.bio) %> + <% end %> <%= most_recent_update(@activities, @user, :atom) %> <%= logo(@user) %> - " type="application/atom+xml"/> + <%= render Phoenix.Controller.view_module(@conn), "_author.atom", assigns %> diff --git a/lib/pleroma/web/templates/feed/feed/user.rss.eex b/lib/pleroma/web/templates/feed/feed/user.rss.eex index b907a7e57..e2150d8b6 100644 --- a/lib/pleroma/web/templates/feed/feed/user.rss.eex +++ b/lib/pleroma/web/templates/feed/feed/user.rss.eex @@ -1,26 +1,32 @@ <%= @user.nickname <> "'s timeline" %> - <%= "#{Routes.user_feed_url(@conn, :feed, @user.nickname)}.rss" %> - " + + <%= "#{Routes.user_feed_url(@conn, :feed, @user.nickname)}.rss" %> <%= escape(@user.bio) %> <%= logo(@user) %> <%= @user.nickname <> "'s timeline" %> <%= "#{Routes.user_feed_url(@conn, :feed, @user.nickname)}.rss" %> + <%= most_recent_update(@activities, @user, :rss) %> + <%= logo(@user) %> + <%= Pleroma.Application.named_version() %> <%= render Phoenix.Controller.view_module(@conn), "_author.rss", assigns %> <%= if last_activity(@activities) do %> - <%= "#{Routes.user_feed_url(@conn, :feed, @user.nickname)}.rss?max_id=#{last_activity(@activities).id}" %> + - <%= activity_link(@activity, @data) %> + <%= escape(activity_link(@activity, @data) || "") %> <%= for tag <- Pleroma.Object.hashtags(@object) do %> <%= if tag && tag != "" do %> diff --git a/lib/pleroma/web/templates/feed/feed/_tag_activity.xml.eex b/lib/pleroma/web/templates/feed/feed/_tag_activity.xml.eex index 09945605c..a8796353f 100644 --- a/lib/pleroma/web/templates/feed/feed/_tag_activity.xml.eex +++ b/lib/pleroma/web/templates/feed/feed/_tag_activity.xml.eex @@ -1,6 +1,6 @@ - <%= activity_link(@activity, @data) %> - <%= activity_link(@activity, @data) %> + <%= escape(activity_link(@activity, @data) || "") %> + <%= escape(activity_link(@activity, @data) || "") %> <%= to_rfc2822(@data["published"]) %> <%= activity_title(@data, Keyword.get(@feed_config, :post_title, %{})) %> <%= activity_description(@data) %> diff --git a/test/pleroma/web/feed/tag_controller_test.exs b/test/pleroma/web/feed/tag_controller_test.exs index 9e8874455..eef524cc4 100644 --- a/test/pleroma/web/feed/tag_controller_test.exs +++ b/test/pleroma/web/feed/tag_controller_test.exs @@ -167,6 +167,8 @@ defmodule Pleroma.Web.Feed.TagControllerTest do ~c"yeah #PleromaArt" ] + assert xpath(xml, ~x"//channel/item/guid/@isPermaLink"sl) == ["true", "true"] + assert xpath(xml, ~x"//channel/item/pubDate/text()"sl) == [ FeedView.to_rfc2822(activity2.data["published"]), FeedView.to_rfc2822(activity1.data["published"]) @@ -232,6 +234,37 @@ defmodule Pleroma.Web.Feed.TagControllerTest do refute response =~ "" end + test "escapes and round-trips RSS item permalinks", %{conn: conn} do + user = insert(:user, local: false) + permalink = "https://remote.example/post?first=one&second=two" + + note = + insert(:note, + user: user, + object_local: false, + data: %{ + "content" => "remote #rsslink", + "external_url" => permalink, + "summary" => "", + "tag" => ["rsslink"] + } + ) + + insert(:note_activity, user: user, note: note, local: false, object_local: false) + + response = + conn + |> put_req_header("accept", "application/rss+xml") + |> get(tag_feed_path(conn, :feed, "rsslink.rss")) + |> response(200) + + assert response =~ "https://remote.example/post?first=one&second=two" + + xml = parse(response) + assert xpath(xml, ~x"//channel/item/guid/text()"s) == permalink + assert xpath(xml, ~x"//channel/item/link/text()"s) == permalink + end + describe "private instance" do setup do: clear_config([:instance, :public], false) From 2d667e1d2a2cfecea99cdcd660460882b2dfbf20 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Fri, 24 Jul 2026 20:44:58 +0400 Subject: [PATCH 166/183] style(feed): format enclosure fixtures --- test/pleroma/web/feed/tag_controller_test.exs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/test/pleroma/web/feed/tag_controller_test.exs b/test/pleroma/web/feed/tag_controller_test.exs index eef524cc4..a23c25274 100644 --- a/test/pleroma/web/feed/tag_controller_test.exs +++ b/test/pleroma/web/feed/tag_controller_test.exs @@ -30,8 +30,7 @@ defmodule Pleroma.Web.Feed.TagControllerTest do %{ "url" => [ %{ - "href" => - "https://peertube.moe/video.mp4?first=one&second=two", + "href" => "https://peertube.moe/video.mp4?first=one&second=two", "mediaType" => "video/mp4", "size" => "123", "type" => "Link" @@ -115,8 +114,7 @@ defmodule Pleroma.Web.Feed.TagControllerTest do %{ "url" => [ %{ - "href" => - "https://peertube.moe/video.mp4?first=one&second=two", + "href" => "https://peertube.moe/video.mp4?first=one&second=two", "mediaType" => "video/mp4", "size" => "123", "type" => "Link" From 5f71532442c7ad45c34663a6b3cf08f081a35132 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Fri, 24 Jul 2026 20:59:13 +0400 Subject: [PATCH 167/183] fix(feed): tighten enclosure metadata --- lib/pleroma/web/feed/feed_view.ex | 10 ++++++---- .../web/templates/feed/feed/_activity.atom.eex | 2 +- .../web/templates/feed/feed/_activity.rss.eex | 9 ++++----- .../web/templates/feed/feed/_tag_activity.atom.eex | 2 +- .../web/templates/feed/feed/_tag_activity.xml.eex | 9 ++++----- test/pleroma/web/feed/feed_view_test.exs | 13 +++++++++++++ test/pleroma/web/feed/user_controller_test.exs | 9 ++++++--- 7 files changed, 35 insertions(+), 19 deletions(-) diff --git a/lib/pleroma/web/feed/feed_view.ex b/lib/pleroma/web/feed/feed_view.ex index dcbb8404f..fc29db2b7 100644 --- a/lib/pleroma/web/feed/feed_view.ex +++ b/lib/pleroma/web/feed/feed_view.ex @@ -131,7 +131,7 @@ defmodule Pleroma.Web.Feed.FeedView do end attachments_html = - if data["sensitive"] do + if sensitive?(data) do "" else (data["attachment"] || []) @@ -181,6 +181,8 @@ defmodule Pleroma.Web.Feed.FeedView do def activity_context(activity), do: escape(activity.data["context"]) + def sensitive?(data), do: data["sensitive"] in [true, "true"] + def feed_self_url(conn) do conn |> Phoenix.Controller.current_url() @@ -237,15 +239,15 @@ defmodule Pleroma.Web.Feed.FeedView do end end - def attachment_previewable?(attachment) do - attachment_medium(attachment) in ["image", "video", "audio"] + def rss_enclosure_attachment(attachments) do + Enum.find(attachments || [], &(not is_nil(attachment_size_positive(&1)))) end def attachment_description(attachment) do attachment["name"] || "" end - def media_content_xml(_attachment, true), do: "" + def media_content_xml(_attachment, sensitive) when sensitive in [true, "true"], do: "" def media_content_xml(attachment, _sensitive) do href = escape(attachment_href(attachment) || "") diff --git a/lib/pleroma/web/templates/feed/feed/_activity.atom.eex b/lib/pleroma/web/templates/feed/feed/_activity.atom.eex index bc67e12b3..91ef947f7 100644 --- a/lib/pleroma/web/templates/feed/feed/_activity.atom.eex +++ b/lib/pleroma/web/templates/feed/feed/_activity.atom.eex @@ -21,7 +21,7 @@ <% end %> - <%= unless @data["sensitive"] do %> + <%= unless sensitive?(@data) do %> <%= for attachment <- @data["attachment"] || [] do %> <%= if size = attachment_size_positive(attachment) do %> " type="<%= escape(attachment_type(attachment) || "application/octet-stream") %>" length="<%= size %>"/> diff --git a/lib/pleroma/web/templates/feed/feed/_activity.rss.eex b/lib/pleroma/web/templates/feed/feed/_activity.rss.eex index 393501dad..4d71a45c7 100644 --- a/lib/pleroma/web/templates/feed/feed/_activity.rss.eex +++ b/lib/pleroma/web/templates/feed/feed/_activity.rss.eex @@ -18,15 +18,14 @@ <% end %> <% end %> - <%= unless @data["sensitive"] do %> - <%= with [attachment | _] <- @data["attachment"] || [] do %> - <%= size = attachment_size_positive(attachment) %> - " type="<%= escape(attachment_type(attachment) || "application/octet-stream") %>" length="<%= size || 0 %>" /> + <%= unless sensitive?(@data) do %> + <%= with attachment when not is_nil(attachment) <- rss_enclosure_attachment(@data["attachment"]) do %> + " type="<%= escape(attachment_type(attachment) || "application/octet-stream") %>" length="<%= attachment_size_positive(attachment) %>" /> <% end %> <% end %> <%= for attachment <- @data["attachment"] || [] do %> - <%= media_content_xml(attachment, @data["sensitive"]) %> + <%= media_content_xml(attachment, sensitive?(@data)) %> <% end %> <%= if @data["inReplyTo"] do %> diff --git a/lib/pleroma/web/templates/feed/feed/_tag_activity.atom.eex b/lib/pleroma/web/templates/feed/feed/_tag_activity.atom.eex index ce5d62de6..44c4806cf 100644 --- a/lib/pleroma/web/templates/feed/feed/_tag_activity.atom.eex +++ b/lib/pleroma/web/templates/feed/feed/_tag_activity.atom.eex @@ -24,7 +24,7 @@ <% end %> - <%= unless @data["sensitive"] do %> + <%= unless sensitive?(@data) do %> <%= for attachment <- @data["attachment"] || [] do %> <%= if size = attachment_size_positive(attachment) do %> " type="<%= escape(attachment_type(attachment) || "application/octet-stream") %>" length="<%= size %>"/> diff --git a/lib/pleroma/web/templates/feed/feed/_tag_activity.xml.eex b/lib/pleroma/web/templates/feed/feed/_tag_activity.xml.eex index a8796353f..64e4c1d81 100644 --- a/lib/pleroma/web/templates/feed/feed/_tag_activity.xml.eex +++ b/lib/pleroma/web/templates/feed/feed/_tag_activity.xml.eex @@ -6,15 +6,14 @@ <%= activity_description(@data) %> ]]> - <%= unless @data["sensitive"] do %> - <%= with [attachment | _] <- @data["attachment"] || [] do %> - <%= size = attachment_size_positive(attachment) %> - " type="<%= escape(attachment_type(attachment) || "application/octet-stream") %>" length="<%= size || 0 %>"/> + <%= unless sensitive?(@data) do %> + <%= with attachment when not is_nil(attachment) <- rss_enclosure_attachment(@data["attachment"]) do %> + " type="<%= escape(attachment_type(attachment) || "application/octet-stream") %>" length="<%= attachment_size_positive(attachment) %>"/> <% end %> <% end %> <%= for attachment <- @data["attachment"] || [] do %> - <%= media_content_xml(attachment, @data["sensitive"]) %> + <%= media_content_xml(attachment, sensitive?(@data)) %> <% end %> <%= for tag <- Pleroma.Object.hashtags(@object) do %> diff --git a/test/pleroma/web/feed/feed_view_test.exs b/test/pleroma/web/feed/feed_view_test.exs index 45ed3c3ff..786b6df07 100644 --- a/test/pleroma/web/feed/feed_view_test.exs +++ b/test/pleroma/web/feed/feed_view_test.exs @@ -35,6 +35,7 @@ defmodule Pleroma.Web.Feed.FeedViewTest do assert FeedView.atom_content_encoded(data) == "content warning" assert FeedView.rss_content_encoded(data) == "content warning" assert FeedView.media_content_xml(attachment, true) == "" + assert FeedView.media_content_xml(attachment, "true") == "" media_content = FeedView.media_content_xml(attachment, false) assert media_content =~ "spoiler" @@ -43,4 +44,16 @@ defmodule Pleroma.Web.Feed.FeedViewTest do assert FeedView.media_content_xml(Map.delete(attachment, "name"), false) == ~s(\n) end + + test "does not treat a legacy false string as sensitive" do + data = %{ + "attachment" => [ + %{"url" => [%{"href" => "https://example.com/image.png", "mediaType" => "image/png"}]} + ], + "content" => "visible", + "sensitive" => "false" + } + + assert FeedView.atom_content_encoded(data) =~ " response(200) |> parse() - assert xpath(rss_xml, ~x"//item/enclosure/@url"sl) == ["https://example.com/file.bin"] - assert xpath(rss_xml, ~x"//item/enclosure/@type"sl) == ["application/octet-stream"] - assert xpath(rss_xml, ~x"//item/enclosure/@length"sl) == ["0"] + assert xpath(rss_xml, ~x"//item/enclosure/@url"sl) == [] + assert xpath(rss_xml, ~x"//item/media:content/@url"sl) == ["https://example.com/file.bin"] + + assert xpath(rss_xml, ~x"//item/media:content/@type"sl) == [ + "application/octet-stream" + ] end test "returns 404 for a missing feed", %{conn: conn} do From c5230d98c5bbb4ba70516f825fec4d1fb60d836d Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Fri, 24 Jul 2026 21:04:35 +0400 Subject: [PATCH 168/183] fix(feed): complete Atom XML metadata --- lib/pleroma/web/feed/feed_view.ex | 2 +- .../templates/feed/feed/_activity.atom.eex | 2 +- .../feed/feed/_tag_activity.atom.eex | 2 +- .../web/templates/feed/feed/tag.atom.eex | 2 +- test/pleroma/web/feed/tag_controller_test.exs | 43 +++++++++++++++++++ .../pleroma/web/feed/user_controller_test.exs | 28 ++++++++++++ 6 files changed, 75 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/web/feed/feed_view.ex b/lib/pleroma/web/feed/feed_view.ex index fc29db2b7..b409b1923 100644 --- a/lib/pleroma/web/feed/feed_view.ex +++ b/lib/pleroma/web/feed/feed_view.ex @@ -43,7 +43,7 @@ defmodule Pleroma.Web.Feed.FeedView do def most_recent_update(activities, :rss) do activities |> Enum.map(fn activity -> - case Object.normalize(activity, fetch: false) do + case activity.object do %Object{updated_at: updated_at} -> updated_at _ -> activity.updated_at end diff --git a/lib/pleroma/web/templates/feed/feed/_activity.atom.eex b/lib/pleroma/web/templates/feed/feed/_activity.atom.eex index 91ef947f7..f94ba0793 100644 --- a/lib/pleroma/web/templates/feed/feed/_activity.atom.eex +++ b/lib/pleroma/web/templates/feed/feed/_activity.atom.eex @@ -1,7 +1,7 @@ http://activitystrea.ms/schema/1.0/note http://activitystrea.ms/schema/1.0/post - <%= @data["id"] %> + <%= escape(@data["id"] || "") %> <%= activity_title(@data, Keyword.get(@feed_config, :post_title, %{})) %> <%= escape(atom_content_encoded(@data)) %> <%= to_rfc3339(@data["published"]) %> diff --git a/lib/pleroma/web/templates/feed/feed/_tag_activity.atom.eex b/lib/pleroma/web/templates/feed/feed/_tag_activity.atom.eex index 44c4806cf..6c74c2ab7 100644 --- a/lib/pleroma/web/templates/feed/feed/_tag_activity.atom.eex +++ b/lib/pleroma/web/templates/feed/feed/_tag_activity.atom.eex @@ -4,7 +4,7 @@ <%= render Phoenix.Controller.view_module(@conn), "_tag_author.atom", assigns %> - <%= @data["id"] %> + <%= escape(@data["id"] || "") %> <%= activity_title(@data, Keyword.get(@feed_config, :post_title, %{})) %> <%= escape(atom_content_encoded(@data)) %> <%= to_rfc3339(@data["published"]) %> diff --git a/lib/pleroma/web/templates/feed/feed/tag.atom.eex b/lib/pleroma/web/templates/feed/feed/tag.atom.eex index 1646aeec7..fb04544d3 100644 --- a/lib/pleroma/web/templates/feed/feed/tag.atom.eex +++ b/lib/pleroma/web/templates/feed/feed/tag.atom.eex @@ -13,7 +13,7 @@ <%= subtitle %> <% end %> <%= feed_logo() %> - <%= most_recent_update(@activities) %> + <%= most_recent_update(@activities) || to_rfc3339(NaiveDateTime.utc_now()) %> <%= for activity <- @activities do %> diff --git a/test/pleroma/web/feed/tag_controller_test.exs b/test/pleroma/web/feed/tag_controller_test.exs index a23c25274..35e29ed8c 100644 --- a/test/pleroma/web/feed/tag_controller_test.exs +++ b/test/pleroma/web/feed/tag_controller_test.exs @@ -232,6 +232,49 @@ defmodule Pleroma.Web.Feed.TagControllerTest do refute response =~ "" end + test "provides a valid updated timestamp for an empty Atom feed", %{conn: conn} do + response = + conn + |> put_req_header("accept", "application/atom+xml") + |> get(tag_feed_path(conn, :feed, "empty.atom")) + |> response(200) + + updated = response |> parse() |> xpath(~x"//feed/updated/text()"s) + assert {:ok, _datetime, 0} = DateTime.from_iso8601(updated) + end + + test "escapes Atom IDs and tag feed self links", %{conn: conn} do + user = insert(:user, local: false) + object_id = "https://remote.example/object?first=one&second=two" + + note = + insert(:note, + user: user, + object_local: false, + data: %{ + "content" => "remote #atomlink", + "external_url" => object_id, + "id" => object_id, + "summary" => "", + "tag" => ["atomlink"] + } + ) + + insert(:note_activity, user: user, note: note, local: false, object_local: false) + + response = + conn + |> put_req_header("accept", "application/atom+xml") + |> get("/tags/atomlink.atom?third=three&fourth=four") + |> response(200) + + assert response =~ "https://remote.example/object?first=one&second=two" + assert response =~ "third=three&fourth=four" + + xml = parse(response) + assert xpath(xml, ~x"//feed/entry/id/text()"s) == object_id + end + test "escapes and round-trips RSS item permalinks", %{conn: conn} do user = insert(:user, local: false) permalink = "https://remote.example/post?first=one&second=two" diff --git a/test/pleroma/web/feed/user_controller_test.exs b/test/pleroma/web/feed/user_controller_test.exs index b78ddb63a..458f5ec8d 100644 --- a/test/pleroma/web/feed/user_controller_test.exs +++ b/test/pleroma/web/feed/user_controller_test.exs @@ -150,6 +150,34 @@ defmodule Pleroma.Web.Feed.UserControllerTest do end end + test "keeps fallback summary markup inert in rendered Atom content", %{ + conn: conn, + object: object, + user: user + } do + data = + object.data + |> Map.put("content", "") + |> Map.put("summary", "") + + object + |> Ecto.Changeset.change(data: data) + |> Pleroma.Repo.update!() + + response = + conn + |> get("/users/#{user.nickname}/feed.atom") + |> response(200) + + assert response =~ "&lt;script&gt;alert(&#39;feed&#39;)&lt;/script&gt;" + + content_values = response |> parse() |> xpath(~x"//entry/content/text()"sl) + + assert Enum.any?(content_values, fn content -> + String.starts_with?(content, "<script>alert('feed')</script>") + end) + end + test "does not expose sensitive attachments as enclosures", %{ conn: conn, object: object, From 5d8b5a73d2d23dac3b79ff74431bca1b02a7fe70 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Fri, 24 Jul 2026 21:05:16 +0400 Subject: [PATCH 169/183] style(feed): format Atom fallback assertion --- test/pleroma/web/feed/user_controller_test.exs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/pleroma/web/feed/user_controller_test.exs b/test/pleroma/web/feed/user_controller_test.exs index 458f5ec8d..d12904ec0 100644 --- a/test/pleroma/web/feed/user_controller_test.exs +++ b/test/pleroma/web/feed/user_controller_test.exs @@ -169,7 +169,8 @@ defmodule Pleroma.Web.Feed.UserControllerTest do |> get("/users/#{user.nickname}/feed.atom") |> response(200) - assert response =~ "&lt;script&gt;alert(&#39;feed&#39;)&lt;/script&gt;" + assert response =~ + "&lt;script&gt;alert(&#39;feed&#39;)&lt;/script&gt;" content_values = response |> parse() |> xpath(~x"//entry/content/text()"sl) From d5f48101b806f973ded8f5cf3276538b269ec3b4 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sat, 25 Jul 2026 00:06:03 +0400 Subject: [PATCH 170/183] fix(media-proxy): safely detect extensionless images --- changelog.d/mediaproxy-content-type.fix | 1 + lib/pleroma/reverse_proxy.ex | 138 +++++++++-- .../attachment_type_sniffer.ex | 59 ----- .../object_validators/attachment_validator.ex | 21 +- .../web/mastodon_api/views/status_view.ex | 1 + .../web/media_proxy/media_proxy_controller.ex | 1 + test/pleroma/reverse_proxy_test.exs | 178 ++++++-------- .../attachment_type_sniffer_test.exs | 178 -------------- .../attachment_validator_test.exs | 221 ------------------ .../mastodon_api/views/status_view_test.exs | 17 ++ .../media_proxy_controller_test.exs | 5 +- 11 files changed, 211 insertions(+), 609 deletions(-) create mode 100644 changelog.d/mediaproxy-content-type.fix delete mode 100644 lib/pleroma/web/activity_pub/object_validators/attachment_type_sniffer.ex delete mode 100644 test/pleroma/web/activity_pub/object_validators/attachment_type_sniffer_test.exs diff --git a/changelog.d/mediaproxy-content-type.fix b/changelog.d/mediaproxy-content-type.fix new file mode 100644 index 000000000..f36dd6283 --- /dev/null +++ b/changelog.d/mediaproxy-content-type.fix @@ -0,0 +1 @@ +Detect image content types for extensionless MediaProxy responses. diff --git a/lib/pleroma/reverse_proxy.ex b/lib/pleroma/reverse_proxy.ex index 11f041c57..061b2de9b 100644 --- a/lib/pleroma/reverse_proxy.ex +++ b/lib/pleroma/reverse_proxy.ex @@ -17,6 +17,7 @@ defmodule Pleroma.ReverseProxy do @max_read_duration :timer.seconds(30) @max_body_length :infinity @failed_request_ttl :timer.seconds(60) + @sniff_bytes 8 * 1024 @methods ~w(GET HEAD) @allowed_mime_types Pleroma.Config.get([Pleroma.Upload, :allowed_mime_types], []) @@ -63,6 +64,9 @@ defmodule Pleroma.ReverseProxy do is always set (overriding any upstream header) so the media can be embedded in pages; the filename is derived from the content type + * `sniff_content_type` (default `false`): detects an image MIME type from the first + response chunk when the upstream type is missing or `application/octet-stream`. + * `req_headers`, `resp_headers` additional headers. * `http`: options for [hackney](https://github.com/benoitc/hackney) or [gun](https://github.com/ninenines/gun). @@ -71,11 +75,15 @@ defmodule Pleroma.ReverseProxy do @default_options [pool: :media] @inline_content_types [ + "image/apng", + "image/avif", + "image/bmp", "image/gif", "image/jpeg", "image/jpg", "image/png", "image/svg+xml", + "image/webp", "audio/mpeg", "audio/mp3", "video/webm", @@ -94,6 +102,7 @@ defmodule Pleroma.ReverseProxy do | {:req_headers, [{String.t(), String.t()}]} | {:resp_headers, [{String.t(), String.t()}]} | {:inline_content_types, boolean() | list(String.t())} + | {:sniff_content_type, boolean()} | {:redirect_on_failure, boolean()} @spec call(Plug.Conn.t(), String.t(), list(option())) :: Plug.Conn.t() @@ -186,12 +195,14 @@ defmodule Pleroma.ReverseProxy do defp response(conn, client, url, status, headers, opts) do Logger.debug("#{__MODULE__} #{status} #{url} #{inspect(headers)}") + {headers, client, prefetched} = maybe_prefetch_for_content_type(headers, client, opts) + result = conn |> put_resp_headers(build_resp_headers(headers, opts)) |> streaming_compat |> send_chunked(status) - |> chunk_reply(client, opts) + |> chunk_reply(client, opts, prefetched) case result do {:ok, conn} -> @@ -211,18 +222,28 @@ defmodule Pleroma.ReverseProxy do end end - defp chunk_reply(conn, client, opts) do + defp chunk_reply(conn, client, opts, :none) do chunk_reply(conn, client, opts, 0, 0) end - defp chunk_reply(conn, client, opts, sent_so_far, duration) do - with {:ok, duration} <- - check_read_duration( - duration, - Keyword.get(opts, :max_read_duration, @max_read_duration) + defp chunk_reply(conn, _client, _opts, :done), do: {:ok, conn} + defp chunk_reply(conn, _client, _opts, {:error, error}), do: {:error, error, conn} + + defp chunk_reply(conn, client, opts, {:ok, data, duration}) do + with :ok <- + body_size_constraint( + byte_size(data), + Keyword.get(opts, :max_body_length, @max_body_length) ), - {:ok, data, client} <- client().stream_body(client), - {:ok, duration} <- increase_read_duration(duration), + {:ok, conn} <- chunk(conn, data) do + chunk_reply(conn, client, opts, byte_size(data), duration) + else + {:error, error} -> {:error, error, conn} + end + end + + defp chunk_reply(conn, client, opts, sent_so_far, duration) do + with {:ok, data, client, duration} <- read_chunk(client, duration, opts), sent_so_far = sent_so_far + byte_size(data), :ok <- body_size_constraint( @@ -237,6 +258,66 @@ defmodule Pleroma.ReverseProxy do end end + defp read_chunk(client, duration, opts) do + with {:ok, timer} <- + check_read_duration( + duration, + Keyword.get(opts, :max_read_duration, @max_read_duration) + ), + {:ok, data, client} <- client().stream_body(client), + {:ok, duration} <- increase_read_duration(timer) do + {:ok, data, client, duration} + end + end + + defp maybe_prefetch_for_content_type(headers, client, opts) do + if Keyword.get(opts, :sniff_content_type, false) and generic_content_type?(headers) do + case read_chunk(client, 0, opts) do + {:ok, data, client, duration} -> + {maybe_put_image_content_type(headers, data), client, {:ok, data, duration}} + + :done -> + {headers, client, :done} + + {:error, error} -> + {headers, client, {:error, error}} + end + else + {headers, client, :none} + end + end + + defp generic_content_type?(headers) do + headers + |> get_content_type() + |> String.trim() + |> String.downcase() + |> then(&(&1 in ["", "application/octet-stream"])) + end + + defp maybe_put_image_content_type(headers, data) do + with false <- data == "", + {:ok, %{mime_type: "image/" <> _ = content_type}} <- + Majic.perform({:bytes, binary_part(data, 0, min(byte_size(data), @sniff_bytes))}, + pool: Pleroma.MajicPool + ) do + [ + {"content-type", content_type} + | Enum.reject(headers, fn {key, _} -> key == "content-type" end) + ] + else + _ -> headers + end + rescue + error -> + Logger.debug("#{__MODULE__}: content-type sniffing failed: #{Exception.message(error)}") + headers + catch + kind, reason -> + Logger.debug("#{__MODULE__}: content-type sniffing failed: #{kind}: #{inspect(reason)}") + headers + end + defp head_response(conn, url, code, headers, opts) do Logger.debug("#{__MODULE__} #{code} #{url} #{inspect(headers)}") @@ -283,17 +364,34 @@ defmodule Pleroma.ReverseProxy do |> Enum.filter(fn {k, _} -> k in @keep_req_headers end) |> build_req_range_or_encoding_header(opts) |> build_req_user_agent_header(opts) - |> Keyword.merge(Keyword.get(opts, :req_headers, [])) + |> merge_headers(Keyword.get(opts, :req_headers, [])) + |> maybe_force_identity_encoding(opts) + end + + defp merge_headers(headers, extra_headers) do + Enum.reduce(extra_headers, headers, fn {key, value}, headers -> + List.keystore(headers, String.downcase(key), 0, {String.downcase(key), value}) + end) + end + + defp maybe_force_identity_encoding(headers, opts) do + if Keyword.get(opts, :sniff_content_type, false) do + List.keystore(headers, "accept-encoding", 0, {"accept-encoding", "identity"}) + else + headers + end end # Disable content-encoding if any @range_headers are requested (see #1823). defp build_req_range_or_encoding_header(headers, _opts) do range? = Enum.any?(headers, fn {header, _} -> Enum.member?(@range_headers, header) end) - if range? && List.keymember?(headers, "accept-encoding", 0) do - List.keydelete(headers, "accept-encoding", 0) - else - headers + cond do + range? && List.keymember?(headers, "accept-encoding", 0) -> + List.keydelete(headers, "accept-encoding", 0) + + true -> + headers end end @@ -386,7 +484,7 @@ defmodule Pleroma.ReverseProxy do disposition = "attachment; filename=\"#{name}\"" - List.keystore(headers, "content-disposition", 0, {"content-disposition", disposition}) + replace_header(headers, "content-disposition", disposition) else if opt == true do headers @@ -400,11 +498,15 @@ defmodule Pleroma.ReverseProxy do "inline" end - List.keystore(headers, "content-disposition", 0, {"content-disposition", disposition}) + replace_header(headers, "content-disposition", disposition) end end end + defp replace_header(headers, key, value) do + [{key, value} | Enum.reject(headers, fn {header, _} -> header == key end)] + end + defp inline_filename(content_type) do case MIME.extensions(content_type) do [ext | _] when ext != "" -> "inline.#{ext}" @@ -443,7 +545,9 @@ defmodule Pleroma.ReverseProxy do end end - defp check_read_duration(_, _), do: {:ok, :no_duration_limit, :no_duration_limit} + defp check_read_duration(_, _), do: {:ok, :no_duration_limit} + + defp increase_read_duration(:no_duration_limit), do: {:ok, :no_duration_limit} defp increase_read_duration({previous_duration, started}) when is_integer(previous_duration) and is_integer(started) do diff --git a/lib/pleroma/web/activity_pub/object_validators/attachment_type_sniffer.ex b/lib/pleroma/web/activity_pub/object_validators/attachment_type_sniffer.ex deleted file mode 100644 index 19215b627..000000000 --- a/lib/pleroma/web/activity_pub/object_validators/attachment_type_sniffer.ex +++ /dev/null @@ -1,59 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2022 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentTypeSniffer do - @moduledoc """ - Best-effort MIME sniffing for remote attachments whose declared type is - unusable (missing or `application/octet-stream`). - - Fetches the first chunk of the URL and runs it through libmagic. Only - `image/*` results are returned; anything else, or any failure, yields `nil` — - so this never reclassifies non-image media and never blocks ingestion. This - exists because some remotes (e.g. extensionless Cloudflare Images URLs) ship - attachments with no usable `mediaType`, which would otherwise render in the - timeline as a clickable link instead of an inline image. - """ - - alias Pleroma.HTTP - - require Logger - - # libmagic only needs the file header; a few KB is plenty and keeps the - # request cheap even when the origin ignores Range. - @sniff_bytes 8 * 1024 - @timeout 5_000 - - @spec sniff_image_type(binary() | nil) :: {:ok, binary() | nil} - def sniff_image_type(url) when is_binary(url) and url != "" do - do_sniff(url) - rescue - e -> - Logger.debug( - "Attachment type sniff failed for #{url}: #{Exception.format(:error, e, __STACKTRACE__)}" - ) - - {:ok, nil} - end - - def sniff_image_type(_), do: {:ok, nil} - - defp do_sniff(url) do - headers = [{"range", "bytes=0-#{@sniff_bytes - 1}"}] - opts = [pool: :media, timeout: @timeout, recv_timeout: @timeout] - - with {:ok, %Tesla.Env{status: status, body: body}} when status in 200..299 <- - HTTP.get(url, headers, opts), - false <- blank?(body), - {:ok, %{mime_type: mime}} <- Majic.perform({:bytes, body}, pool: Pleroma.MajicPool), - true <- String.starts_with?(mime, "image/") do - {:ok, mime} - else - _ -> {:ok, nil} - end - end - - defp blank?(nil), do: true - defp blank?(""), do: true - defp blank?(_), do: false -end diff --git a/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex b/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex index b6e30a438..5ee9e7549 100644 --- a/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex @@ -6,7 +6,6 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator do use Ecto.Schema alias Pleroma.EctoType.ActivityPub.ObjectValidators - alias Pleroma.Web.ActivityPub.ObjectValidators.AttachmentTypeSniffer import Ecto.Changeset @@ -62,27 +61,9 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator do end def fix_media_type(data) do - current = data["mediaType"] || data["mimeType"] || "application/octet-stream" - - resolved = - if sniffable?(current) do - case AttachmentTypeSniffer.sniff_image_type(data["href"]) do - {:ok, mime} when is_binary(mime) -> mime - _ -> current - end - else - current - end - - Map.put(data, "mediaType", resolved) + Map.put_new(data, "mediaType", data["mimeType"] || "application/octet-stream") end - # Only sniff when the declared type carries no real information. Everything - # else (including unrecognized-but-present types) is left as the remote sent it. - defp sniffable?("application/octet-stream"), do: true - defp sniffable?(""), do: true - defp sniffable?(_), do: false - defp handle_href(href, mediaType, data) do [ %{ diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 6949c5f2d..725dc2ce7 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -620,6 +620,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do String.contains?(media_type, "image") -> "image" String.contains?(media_type, "video") -> "video" String.contains?(media_type, "audio") -> "audio" + attachment["type"] == "Image" -> "image" true -> "unknown" end diff --git a/lib/pleroma/web/media_proxy/media_proxy_controller.ex b/lib/pleroma/web/media_proxy/media_proxy_controller.ex index a0aafc32e..93f692008 100644 --- a/lib/pleroma/web/media_proxy/media_proxy_controller.ex +++ b/lib/pleroma/web/media_proxy/media_proxy_controller.ex @@ -208,6 +208,7 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do defp media_proxy_opts do Config.get([:media_proxy, :proxy_opts], []) + |> Keyword.put_new(:sniff_content_type, true) end defp sandbox(conn, _params) do diff --git a/test/pleroma/reverse_proxy_test.exs b/test/pleroma/reverse_proxy_test.exs index 5f75f2a07..ad1d75f33 100644 --- a/test/pleroma/reverse_proxy_test.exs +++ b/test/pleroma/reverse_proxy_test.exs @@ -222,6 +222,56 @@ defmodule Pleroma.ReverseProxyTest do assert Conn.get_resp_header(conn, "content-type") == ["application/octet-stream"] end + test "sniffs and preserves an image body with a generic content type", %{conn: conn} do + body = File.read!("test/fixtures/image.jpg") + + ClientMock + |> expect(:request, fn :get, "/extensionless", headers, _, _ -> + assert {"accept-encoding", "identity"} in headers + {:ok, 200, [{"content-type", "application/octet-stream"}], %{body: body}} + end) + |> expect(:stream_body, fn %{body: ^body} = client -> + {:ok, body, Map.delete(client, :body)} + end) + |> expect(:stream_body, fn %{} -> :done end) + + conn = + ReverseProxy.call(conn, "/extensionless", + sniff_content_type: true, + max_read_duration: :infinity, + req_headers: [{"accept-encoding", "gzip"}] + ) + + assert conn.resp_body == body + assert Conn.get_resp_header(conn, "content-type") == ["image/jpeg"] + + assert Conn.get_resp_header(conn, "content-disposition") == [ + "inline; filename=\"inline.jpg\"" + ] + end + + test "preserves a generic content type for a non-image body", %{conn: conn} do + body = "not an image" + + ClientMock + |> expect(:request, fn :get, "/extensionless", _, _, _ -> + {:ok, 200, [], %{body: body}} + end) + |> expect(:stream_body, fn %{body: ^body} = client -> + {:ok, body, Map.delete(client, :body)} + end) + |> expect(:stream_body, fn %{} -> :done end) + + conn = ReverseProxy.call(conn, "/extensionless", sniff_content_type: true) + + assert conn.resp_body == body + assert Conn.get_resp_header(conn, "content-type") == ["application/octet-stream"] + + assert Conn.get_resp_header(conn, "content-disposition") == [ + "attachment; filename=\"extensionless\"" + ] + end + defp headers_mock(_) do ClientMock |> expect(:request, fn :get, "/headers", headers, _, _ -> @@ -298,21 +348,21 @@ defmodule Pleroma.ReverseProxyTest do end end - defp disposition_headers_mock(headers, url \\ "/disposition") do + defp disposition_headers_mock(headers) do ClientMock - |> expect(:request, fn :get, ^url, _, _, _ -> - Registry.register(ClientMock, url, 0) + |> expect(:request, fn :get, "/disposition", _, _, _ -> + Registry.register(ClientMock, "/disposition", 0) - {:ok, 200, headers, %{url: url}} + {:ok, 200, headers, %{url: "/disposition"}} end) - |> expect(:stream_body, 2, fn %{url: ^url} = client -> - case Registry.lookup(ClientMock, url) do + |> expect(:stream_body, 2, fn %{url: "/disposition"} = client -> + case Registry.lookup(ClientMock, "/disposition") do [{_, 0}] -> - Registry.update_value(ClientMock, url, &(&1 + 1)) + Registry.update_value(ClientMock, "/disposition", &(&1 + 1)) {:ok, "", client} [{_, 1}] -> - Registry.unregister(ClientMock, url) + Registry.unregister(ClientMock, "/disposition") :done end end) @@ -337,6 +387,7 @@ defmodule Pleroma.ReverseProxyTest do disposition_headers_mock([ {"content-type", "image/png"}, {"content-disposition", "attachment; filename=\"filename.png\""}, + {"content-disposition", "attachment; filename=\"duplicate.png\""}, {"content-length", "0"} ]) @@ -347,23 +398,6 @@ defmodule Pleroma.ReverseProxyTest do refute String.starts_with?(disposition, "attachment") end - test "forces inline based on content type even without a file extension in the url", %{ - conn: conn - } do - disposition_headers_mock( - [ - {"content-type", "image/jpeg"}, - {"content-length", "0"} - ], - "/original" - ) - - conn = ReverseProxy.call(conn, "/original") - - [disposition] = Conn.get_resp_header(conn, "content-disposition") - assert disposition == "inline; filename=\"inline.jpg\"" - end - test "with content-disposition header", %{conn: conn} do disposition_headers_mock([ {"content-disposition", "attachment; filename=\"filename.jpg\""}, @@ -390,69 +424,6 @@ defmodule Pleroma.ReverseProxyTest do assert {"content-disposition", "attachment; filename=\"upstream.png\""} in conn.resp_headers end - test "with inline_content_types: true does not synthesise inline when upstream is absent", %{ - conn: conn - } do - disposition_headers_mock([ - {"content-type", "image/png"}, - {"content-length", "0"} - ]) - - conn = ReverseProxy.call(conn, "/disposition", inline_content_types: true) - - assert Conn.get_resp_header(conn, "content-disposition") == [] - end - - test "with inline_content_types: false forces attachment for everything", %{ - conn: conn - } do - disposition_headers_mock([ - {"content-type", "image/png"}, - {"content-disposition", "inline; filename=\"pic.png\""}, - {"content-length", "0"} - ]) - - conn = ReverseProxy.call(conn, "/disposition", inline_content_types: false) - - [disposition] = Conn.get_resp_header(conn, "content-disposition") - assert String.starts_with?(disposition, "attachment") - end - - test "with inline_content_types: false derives attachment filename from the URL basename when no upstream filename", %{ - conn: conn - } do - # No content-disposition header: the attachment branch falls back to - # attachment_name, which by default comes from MediaProxy.filename/1 - # (the URL basename). - disposition_headers_mock([ - {"content-type", "image/png"}, - {"content-length", "0"} - ]) - - conn = ReverseProxy.call(conn, "/disposition", inline_content_types: false) - - [disposition] = Conn.get_resp_header(conn, "content-disposition") - assert disposition == "attachment; filename=\"disposition\"" - end - - test "with inline_content_types: false honours an explicit attachment_name opt", %{ - conn: conn - } do - disposition_headers_mock([ - {"content-type", "image/png"}, - {"content-length", "0"} - ]) - - conn = - ReverseProxy.call(conn, "/disposition", - inline_content_types: false, - attachment_name: "custom.bin" - ) - - [disposition] = Conn.get_resp_header(conn, "content-disposition") - assert disposition == "attachment; filename=\"custom.bin\"" - end - test "forces bare inline for a whitelisted type with no MIME extension", %{ conn: conn } do @@ -469,36 +440,17 @@ defmodule Pleroma.ReverseProxyTest do assert disposition == "inline" end - test "honours a custom inline_content_types whitelist", %{conn: conn} do - # image/bmp is NOT in the default whitelist; with a custom whitelist - # that includes it, the proxy should force inline. + test "serves modern browser image types inline", %{conn: conn} do disposition_headers_mock([ - {"content-type", "image/bmp"}, - {"content-disposition", "attachment; filename=\"upstream.bmp\""}, + {"content-type", "image/webp"}, {"content-length", "0"} ]) - conn = - ReverseProxy.call(conn, "/disposition", inline_content_types: ["image/bmp"]) + conn = ReverseProxy.call(conn, "/disposition") - [disposition] = Conn.get_resp_header(conn, "content-disposition") - assert String.starts_with?(disposition, "inline") - assert String.ends_with?(disposition, "inline.bmp\"") - end - - test "treats a type outside the whitelist as attachment even on custom whitelist", %{ - conn: conn - } do - disposition_headers_mock([ - {"content-type", "image/png"}, - {"content-disposition", "attachment; filename=\"filename.png\""}, - {"content-length", "0"} - ]) - - conn = - ReverseProxy.call(conn, "/disposition", inline_content_types: ["image/bmp"]) - - assert {"content-disposition", "attachment; filename=\"filename.png\""} in conn.resp_headers + assert Conn.get_resp_header(conn, "content-disposition") == [ + "inline; filename=\"inline.webp\"" + ] end end diff --git a/test/pleroma/web/activity_pub/object_validators/attachment_type_sniffer_test.exs b/test/pleroma/web/activity_pub/object_validators/attachment_type_sniffer_test.exs deleted file mode 100644 index 52b1b67a1..000000000 --- a/test/pleroma/web/activity_pub/object_validators/attachment_type_sniffer_test.exs +++ /dev/null @@ -1,178 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2022 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentTypeSnifferTest do - use Pleroma.DataCase, async: true - - alias Pleroma.Web.ActivityPub.ObjectValidators.AttachmentTypeSniffer - - # @sniff_bytes must mirror the value in the module under test; we assert the - # Range header that the sniffer sends, so the test should fail loudly if the - # constant drifts. - @sniff_bytes 8 * 1024 - @range_header "bytes=0-#{@sniff_bytes - 1}" - - describe "sniff_image_type/1 with a real image body" do - test "returns image/jpeg for a JPEG body" do - url = "https://example.com/media/image.jpg" - jpeg = File.read!("test/fixtures/image.jpg") - - Tesla.Mock.mock(fn %{method: :get, url: ^url} -> - %Tesla.Env{status: 200, body: jpeg} - end) - - assert {:ok, "image/jpeg"} = AttachmentTypeSniffer.sniff_image_type(url) - end - - test "returns image/png for a PNG body" do - url = "https://example.com/media/image.png" - png = File.read!("test/fixtures/image.png") - - Tesla.Mock.mock(fn %{method: :get, url: ^url} -> - %Tesla.Env{status: 200, body: png} - end) - - assert {:ok, "image/png"} = AttachmentTypeSniffer.sniff_image_type(url) - end - - test "returns image/gif for a GIF body" do - url = "https://example.com/media/image.gif" - gif = File.read!("test/fixtures/image.gif") - - Tesla.Mock.mock(fn %{method: :get, url: ^url} -> - %Tesla.Env{status: 200, body: gif} - end) - - assert {:ok, "image/gif"} = AttachmentTypeSniffer.sniff_image_type(url) - end - end - - describe "sniff_image_type/1 sends a ranged request" do - test "sends a Range header asking for the first chunk" do - url = "https://example.com/media/ranged" - jpeg = File.read!("test/fixtures/image.jpg") - - Tesla.Mock.mock(fn - %{method: :get, url: ^url, headers: headers} -> - headers_map = Enum.into(headers, %{}) - assert headers_map["range"] == @range_header - - %Tesla.Env{status: 200, body: jpeg} - - %{method: :get, url: ^url} -> - # If the sniffer did not send the Range header, fail explicitly. - flunk("expected Range header to be sent with the request") - end) - - assert {:ok, "image/jpeg"} = AttachmentTypeSniffer.sniff_image_type(url) - end - end - - describe "sniff_image_type/1 with a non-image body" do - test "returns nil for plain text" do - url = "https://example.com/media/note.txt" - - Tesla.Mock.mock(fn %{method: :get, url: ^url} -> - %Tesla.Env{status: 200, body: "just some plain text, not an image at all"} - end) - - assert {:ok, nil} = AttachmentTypeSniffer.sniff_image_type(url) - end - - test "returns nil for an HTML body" do - url = "https://example.com/page" - html = File.read!("test/fixtures/rel_me_anchor.html") - - Tesla.Mock.mock(fn %{method: :get, url: ^url} -> - %Tesla.Env{status: 200, body: html} - end) - - assert {:ok, nil} = AttachmentTypeSniffer.sniff_image_type(url) - end - - test "returns nil for an empty body" do - url = "https://example.com/media/empty" - - Tesla.Mock.mock(fn %{method: :get, url: ^url} -> - %Tesla.Env{status: 200, body: ""} - end) - - assert {:ok, nil} = AttachmentTypeSniffer.sniff_image_type(url) - end - - test "returns nil for a nil body" do - url = "https://example.com/media/nil-body" - - Tesla.Mock.mock(fn %{method: :get, url: ^url} -> - %Tesla.Env{status: 200, body: nil} - end) - - assert {:ok, nil} = AttachmentTypeSniffer.sniff_image_type(url) - end - end - - describe "sniff_image_type/1 on HTTP failure" do - test "returns nil on a 404" do - url = "https://example.com/media/missing" - - Tesla.Mock.mock(fn %{method: :get, url: ^url} -> - %Tesla.Env{status: 404, body: ""} - end) - - assert {:ok, nil} = AttachmentTypeSniffer.sniff_image_type(url) - end - - test "returns nil on a 500" do - url = "https://example.com/media/oops" - - Tesla.Mock.mock(fn %{method: :get, url: ^url} -> - %Tesla.Env{status: 500, body: "server error"} - end) - - assert {:ok, nil} = AttachmentTypeSniffer.sniff_image_type(url) - end - - test "returns nil on a 3xx (not in 200..299)" do - url = "https://example.com/media/redirect" - - Tesla.Mock.mock(fn %{method: :get, url: ^url} -> - %Tesla.Env{status: 302, body: ""} - end) - - assert {:ok, nil} = AttachmentTypeSniffer.sniff_image_type(url) - end - - test "returns nil on a transport-level error" do - url = "https://example.com/media/econnrefused" - - Tesla.Mock.mock(fn %{method: :get, url: ^url} -> - {:error, :econnrefused} - end) - - assert {:ok, nil} = AttachmentTypeSniffer.sniff_image_type(url) - end - - test "returns nil when the HTTP adapter raises" do - url = "https://example.com/media/raises" - - Tesla.Mock.mock(fn %{method: :get, url: ^url} -> - raise "boom" - end) - - assert {:ok, nil} = AttachmentTypeSniffer.sniff_image_type(url) - end - end - - describe "sniff_image_type/1 guard clauses" do - test "returns nil for nil input without making a request" do - # No Tesla mock is configured; if a request were made, Tesla.Mock would - # raise and fail the test. - assert {:ok, nil} = AttachmentTypeSniffer.sniff_image_type(nil) - end - - test "returns nil for an empty-string URL without making a request" do - assert {:ok, nil} = AttachmentTypeSniffer.sniff_image_type("") - end - end -end diff --git a/test/pleroma/web/activity_pub/object_validators/attachment_validator_test.exs b/test/pleroma/web/activity_pub/object_validators/attachment_validator_test.exs index 870ad90cc..744ae8704 100644 --- a/test/pleroma/web/activity_pub/object_validators/attachment_validator_test.exs +++ b/test/pleroma/web/activity_pub/object_validators/attachment_validator_test.exs @@ -215,226 +215,5 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidatorTest do AttachmentValidator.cast_and_validate(attachment) |> Ecto.Changeset.apply_action(:insert) end - - test "sniffs image/jpeg for octet-stream attachments whose body is an image" do - url = "https://example.com/media/no-extension/original" - jpeg = File.read!("test/fixtures/image.jpg") - - Tesla.Mock.mock(fn %{method: :get, url: ^url} -> - %Tesla.Env{status: 200, body: jpeg} - end) - - attachment = %{ - "type" => "Document", - "mediaType" => "application/octet-stream", - "url" => [%{"type" => "Link", "href" => url, "mediaType" => "application/octet-stream"}] - } - - {:ok, attachment} = - attachment - |> AttachmentValidator.cast_and_validate() - |> Ecto.Changeset.apply_action(:insert) - - assert [%{href: ^url, mediaType: "image/jpeg"}] = attachment.url - end - - test "sniffs image/jpeg for attachments with a missing mediaType" do - url = "https://example.com/media/missing-type/original" - jpeg = File.read!("test/fixtures/image.jpg") - - Tesla.Mock.mock(fn %{method: :get, url: ^url} -> - %Tesla.Env{status: 200, body: jpeg} - end) - - attachment = %{ - "type" => "Document", - "url" => [%{"type" => "Link", "href" => url}] - } - - {:ok, attachment} = - attachment - |> AttachmentValidator.cast_and_validate() - |> Ecto.Changeset.apply_action(:insert) - - assert [%{href: ^url, mediaType: "image/jpeg"}] = attachment.url - end - - test "leaves non-image octet-stream attachments as octet-stream" do - url = "https://example.com/media/some-document" - - Tesla.Mock.mock(fn %{method: :get, url: ^url} -> - %Tesla.Env{status: 200, body: "just some plain text, not an image at all"} - end) - - attachment = %{ - "type" => "Document", - "mediaType" => "application/octet-stream", - "url" => [%{"type" => "Link", "href" => url, "mediaType" => "application/octet-stream"}] - } - - {:ok, attachment} = - attachment - |> AttachmentValidator.cast_and_validate() - |> Ecto.Changeset.apply_action(:insert) - - assert [%{mediaType: "application/octet-stream"}] = attachment.url - end - - test "does not sniff when the remote already provided a real mediaType" do - # No Tesla mock is set up: if the validator tried to fetch, Tesla.Mock - # would raise and fail the test. A real image type must be kept as-is. - attachment = %{ - "type" => "Document", - "mediaType" => "image/png", - "url" => [ - %{"type" => "Link", "href" => "https://example.com/x", "mediaType" => "image/png"} - ] - } - - {:ok, attachment} = - attachment - |> AttachmentValidator.cast_and_validate() - |> Ecto.Changeset.apply_action(:insert) - - assert [%{mediaType: "image/png"}] = attachment.url - end - end - - describe "fix_media_type fallbacks and sniffing" do - test "uses mimeType as a fallback when mediaType is absent (real type, not sniffed)" do - # No Tesla mock: if the validator tried to fetch, the test would fail. - # A real image type coming from mimeType must be preserved as-is on the - # outer attachment and on the synthesised url entry (string url form). - attachment = %{ - "type" => "Document", - "mimeType" => "image/png", - "url" => "https://example.com/x.png" - } - - {:ok, attachment} = - attachment - |> AttachmentValidator.cast_and_validate() - |> Ecto.Changeset.apply_action(:insert) - - assert attachment.mediaType == "image/png" - assert [%{mediaType: "image/png"}] = attachment.url - end - - test "prefers mediaType over mimeType when both are present" do - attachment = %{ - "type" => "Document", - "mediaType" => "image/gif", - "mimeType" => "image/png", - "url" => "https://example.com/x.gif" - } - - {:ok, attachment} = - attachment - |> AttachmentValidator.cast_and_validate() - |> Ecto.Changeset.apply_action(:insert) - - assert attachment.mediaType == "image/gif" - assert [%{mediaType: "image/gif"}] = attachment.url - end - - test "sniffs when the url entry's mediaType is application/octet-stream and the body is an image" do - # The outer attachment mediaType is not sniffed (no top-level href); only - # the url entry is. We assert only on the url entry. - url = "https://example.com/media/mime-type-octet" - jpeg = File.read!("test/fixtures/image.jpg") - - Tesla.Mock.mock(fn %{method: :get, url: ^url} -> - %Tesla.Env{status: 200, body: jpeg} - end) - - attachment = %{ - "type" => "Document", - "mediaType" => "application/octet-stream", - "url" => [%{"type" => "Link", "href" => url, "mediaType" => "application/octet-stream"}] - } - - {:ok, attachment} = - attachment - |> AttachmentValidator.cast_and_validate() - |> Ecto.Changeset.apply_action(:insert) - - assert [%{mediaType: "image/jpeg"}] = attachment.url - end - - test "sniffs when mediaType is the empty string and the body is an image" do - url = "https://example.com/media/empty-mediatype" - png = File.read!("test/fixtures/image.png") - - Tesla.Mock.mock(fn %{method: :get, url: ^url} -> - %Tesla.Env{status: 200, body: png} - end) - - attachment = %{ - "type" => "Document", - "mediaType" => "", - "url" => [%{"type" => "Link", "href" => url, "mediaType" => ""}] - } - - {:ok, attachment} = - attachment - |> AttachmentValidator.cast_and_validate() - |> Ecto.Changeset.apply_action(:insert) - - assert [%{mediaType: "image/png"}] = attachment.url - end - - test "falls back to the declared type when the sniffer HTTP request fails" do - url = "https://example.com/media/sniff-404" - - Tesla.Mock.mock(fn %{method: :get, url: ^url} -> - %Tesla.Env{status: 404, body: ""} - end) - - attachment = %{ - "type" => "Document", - "mediaType" => "application/octet-stream", - "url" => [%{"type" => "Link", "href" => url, "mediaType" => "application/octet-stream"}] - } - - {:ok, attachment} = - attachment - |> AttachmentValidator.cast_and_validate() - |> Ecto.Changeset.apply_action(:insert) - - assert [%{mediaType: "application/octet-stream"}] = attachment.url - end - - test "sniffs each url entry independently" do - jpeg_url = "https://example.com/media/a" - octet_url = "https://example.com/media/b" - jpeg = File.read!("test/fixtures/image.jpg") - - Tesla.Mock.mock(fn - %{method: :get, url: ^jpeg_url} -> - %Tesla.Env{status: 200, body: jpeg} - - %{method: :get, url: ^octet_url} -> - %Tesla.Env{status: 200, body: "definitely not an image"} - end) - - attachment = %{ - "type" => "Document", - "mediaType" => "application/octet-stream", - "url" => [ - %{"type" => "Link", "href" => jpeg_url, "mediaType" => "application/octet-stream"}, - %{"type" => "Link", "href" => octet_url, "mediaType" => "application/octet-stream"} - ] - } - - {:ok, attachment} = - attachment - |> AttachmentValidator.cast_and_validate() - |> Ecto.Changeset.apply_action(:insert) - - assert [ - %{href: ^jpeg_url, mediaType: "image/jpeg"}, - %{href: ^octet_url, mediaType: "application/octet-stream"} - ] = attachment.url - end end end diff --git a/test/pleroma/web/mastodon_api/views/status_view_test.exs b/test/pleroma/web/mastodon_api/views/status_view_test.exs index 76123cd0f..faeca050e 100644 --- a/test/pleroma/web/mastodon_api/views/status_view_test.exs +++ b/test/pleroma/web/mastodon_api/views/status_view_test.exs @@ -668,6 +668,23 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do assert expected == StatusView.render("attachment.json", %{attachment: object}) assert_schema(expected, "Attachment", api_spec) end + + test "uses the ActivityStreams image type when the MIME type is generic" do + object = %{ + "type" => "Image", + "url" => [ + %{ + "mediaType" => "application/octet-stream", + "href" => "https://example.com/extensionless" + } + ] + } + + attachment = StatusView.render("attachment.json", %{attachment: object}) + + assert attachment.type == "image" + assert attachment.pleroma.mime_type == "application/octet-stream" + end end test "put the url advertised in the Activity in to the url attribute" do diff --git a/test/pleroma/web/media_proxy/media_proxy_controller_test.exs b/test/pleroma/web/media_proxy/media_proxy_controller_test.exs index f7e52483c..800159c21 100644 --- a/test/pleroma/web/media_proxy/media_proxy_controller_test.exs +++ b/test/pleroma/web/media_proxy/media_proxy_controller_test.exs @@ -71,7 +71,10 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyControllerTest do test "it performs ReverseProxy.call with valid signature", %{conn: conn, url: url} do with_mock Pleroma.ReverseProxy, - call: fn _conn, _url, _opts -> %Conn{status: :success} end do + call: fn _conn, _url, opts -> + assert opts[:sniff_content_type] + %Conn{status: :success} + end do assert %Conn{status: :success} = get(conn, url) end end From 5ddd337045f6999ad884210055ed4492b63181a7 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sat, 25 Jul 2026 09:26:26 +0400 Subject: [PATCH 171/183] chore: remove development log --- DEVLOG.md | 14 -------------- 1 file changed, 14 deletions(-) delete mode 100644 DEVLOG.md diff --git a/DEVLOG.md b/DEVLOG.md deleted file mode 100644 index 1e5d2b63d..000000000 --- a/DEVLOG.md +++ /dev/null @@ -1,14 +0,0 @@ -# Development Log - -## Gun pool validation - -- The previous unique origin registry forced concurrent HTTP/1 requests onto one connection. -- Pool identity omitted proxy routes and credentials, allowing incompatible routes to reuse a connection. -- Gun 2.2 CONNECT requests require their tunnel stream reference on subsequent requests; Tesla's bundled Gun adapter does not preserve it. -- CONNECT and SOCKS5 setup returned before the destination tunnel was ready and did not pass authentication credentials. -- The existing global `max_connections` limit and idle reclaimer bound HTTP/1 sibling and many-host connection growth. -- SOCKS4 is not supported by Gun 2.2 and is now rejected explicitly by the Gun adapter. -- Gun reports a stale stream error to the process that calls `cancel/2`; pool workers must consume that reply so one HTTP/2 cancellation cannot terminate other multiplexed requests. -- Streamed uploads must be registered before body enumeration and cancelled if enumeration exits, throws, or raises, otherwise a partial HTTP/1 request can remain reusable. -- Gun 2.4 bounds HTTP/1 response header and trailer buffers, rejects invalid request headers by default, and fixes cross-origin HTTP/2 push handling; Gun 2.4.1 includes Cowlib 2.17.1 to repair HTTP/2 HPACK decoding. -- Plain HTTP proxy requests must use absolute-form request targets; CONNECT is reserved for HTTPS tunnels because forwarding proxies commonly reject CONNECT to port 80. From 99befcaf1bf2326d2196e5640a8834009a720290 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sat, 25 Jul 2026 10:51:09 +0400 Subject: [PATCH 172/183] fix(activitypub): deduplicate account favourites --- lib/pleroma/web/activity_pub/activity_pub.ex | 16 +++++++ ...00_add_activity_favorites_lookup_index.exs | 20 ++++++++ .../controllers/account_controller_test.exs | 46 +++++++++++++++++++ 3 files changed, 82 insertions(+) create mode 100644 priv/repo/migrations/20260725000000_add_activity_favorites_lookup_index.exs diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index a44c5b1ff..10d3fe063 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -1025,6 +1025,20 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp restrict_assigned_account(query, _), do: query defp restrict_favorited_by(query, %{favorited_by: ap_id}) do + newer_favorite = + from(newer_favorite in Activity, + where: newer_favorite.actor == ^ap_id, + where: newer_favorite.id > parent_as(:favorited_activity).id, + where: fragment("?->>'type' = ?", newer_favorite.data, "Like"), + where: + fragment( + "associated_object_id(?) = associated_object_id(?)", + newer_favorite.data, + parent_as(:favorited_activity).data + ), + select: 1 + ) + from( [activity, object: object] in query, join: favorited_activity in Activity, @@ -1033,6 +1047,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do favorited_activity.actor == ^ap_id and fragment("?->>'type' = ?", favorited_activity.data, "Like") and fragment("associated_object_id(?) = (?)->>'id'", favorited_activity.data, object.data), + where: fragment("(?)->'likes' \\? (?)", object.data, ^ap_id), + where: not exists(subquery(newer_favorite)), select: %Activity{activity | pagination_id: favorited_activity.id} ) end diff --git a/priv/repo/migrations/20260725000000_add_activity_favorites_lookup_index.exs b/priv/repo/migrations/20260725000000_add_activity_favorites_lookup_index.exs new file mode 100644 index 000000000..34bc04555 --- /dev/null +++ b/priv/repo/migrations/20260725000000_add_activity_favorites_lookup_index.exs @@ -0,0 +1,20 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Repo.Migrations.AddActivityFavoritesLookupIndex do + use Ecto.Migration + + @disable_ddl_transaction true + @disable_migration_lock true + + def change do + create_if_not_exists( + index(:activities, [:actor, "associated_object_id(data)", "id DESC"], + name: :activities_favorites_lookup_index, + where: "data->>'type' = 'Like'", + concurrently: true + ) + ) + end +end diff --git a/test/pleroma/web/pleroma_api/controllers/account_controller_test.exs b/test/pleroma/web/pleroma_api/controllers/account_controller_test.exs index e93b277cc..bdf9f03c1 100644 --- a/test/pleroma/web/pleroma_api/controllers/account_controller_test.exs +++ b/test/pleroma/web/pleroma_api/controllers/account_controller_test.exs @@ -5,9 +5,12 @@ defmodule Pleroma.Web.PleromaAPI.AccountControllerTest do use Pleroma.Web.ConnCase + alias Pleroma.Activity alias Pleroma.Config + alias Pleroma.Repo alias Pleroma.Tests.ObanHelpers alias Pleroma.User + alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.CommonAPI import Pleroma.Factory @@ -127,6 +130,49 @@ defmodule Pleroma.Web.PleromaAPI.AccountControllerTest do assert Enum.map(response, & &1["id"]) == [second_post.id, first_post.id, third_post.id] end + test "returns a status once when duplicate favorite activities exist", %{ + conn: conn, + user: user + } do + older_activity = insert(:note_activity) + {:ok, _} = CommonAPI.favorite(older_activity.id, user) + + activity = insert(:note_activity) + {:ok, favorite} = CommonAPI.favorite(activity.id, user) + + duplicate = + Repo.insert!(%Activity{ + actor: favorite.actor, + data: Map.put(favorite.data, "id", Utils.generate_activity_id()), + local: favorite.local, + recipients: favorite.recipients + }) + + conn = get(conn, "/api/v1/pleroma/accounts/#{user.id}/favourites?limit=1") + response = json_response_and_validate_schema(conn, :ok) + + assert Enum.map(response, & &1["id"]) == [activity.id] + assert [link_header] = get_resp_header(conn, "link") + assert link_header =~ "max_id=#{duplicate.id}" + assert link_header =~ "min_id=#{duplicate.id}" + + response = + conn + |> get("/api/v1/pleroma/accounts/#{user.id}/favourites?max_id=#{duplicate.id}&limit=1") + |> json_response_and_validate_schema(:ok) + + assert Enum.map(response, & &1["id"]) == [older_activity.id] + + {:ok, _} = CommonAPI.unfavorite(activity.id, user) + + response = + conn + |> get("/api/v1/pleroma/accounts/#{user.id}/favourites") + |> json_response_and_validate_schema(:ok) + + assert Enum.map(response, & &1["id"]) == [older_activity.id] + end + test "returns favorited DM only when user is logged in and he is one of recipients", %{ current_user: current_user, user: user From 91089c93ada742ff2b709601e9f0ea217c9f5a06 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sun, 26 Jul 2026 11:38:24 +0400 Subject: [PATCH 173/183] feat(search): add resumable runtime backfill --- lib/mix/tasks/pleroma/search/indexer.ex | 58 ++++------ lib/pleroma/search/backfill.ex | 137 ++++++++++++++++++++++++ test/pleroma/search/backfill_test.exs | 79 ++++++++++++++ 3 files changed, 234 insertions(+), 40 deletions(-) create mode 100644 lib/pleroma/search/backfill.ex create mode 100644 test/pleroma/search/backfill_test.exs diff --git a/lib/mix/tasks/pleroma/search/indexer.ex b/lib/mix/tasks/pleroma/search/indexer.ex index 2a52472f9..62cbf2e50 100644 --- a/lib/mix/tasks/pleroma/search/indexer.ex +++ b/lib/mix/tasks/pleroma/search/indexer.ex @@ -4,9 +4,6 @@ defmodule Mix.Tasks.Pleroma.Search.Indexer do import Mix.Pleroma - import Ecto.Query - - alias Pleroma.Workers.SearchIndexingWorker def run(["create_index"]) do start_pleroma() @@ -35,49 +32,30 @@ defmodule Mix.Tasks.Pleroma.Search.Indexer do strict: [ chunk: :integer, limit: :integer, - step: :integer + step: :integer, + before: :string ] ) start_pleroma() - chunk_size = Keyword.get(options, :chunk, 100) - limit = Keyword.get(options, :limit, 100_000) - per_step = Keyword.get(options, :step, 1000) + result = + Pleroma.Search.Backfill.run( + options ++ + [ + on_page: fn page -> + IO.puts( + "Queued #{page.count} activities (#{page.enqueued} total), " <> + "checkpoint: #{page.next_cursor}" + ) + end + ] + ) - chunks = max(div(limit, per_step), 1) + IO.puts("Queued #{result.enqueued} activities") - 1..chunks - |> Enum.each(fn step -> - q = - from(a in Pleroma.Activity, - limit: ^per_step, - offset: ^per_step * (^step - 1), - select: [:id], - order_by: [desc: :id] - ) - - {:ok, ids} = - Pleroma.Repo.transaction(fn -> - Pleroma.Repo.stream(q, timeout: :infinity) - |> Enum.map(fn a -> - a.id - end) - end) - - IO.puts("Got #{length(ids)} activities, adding to indexer") - - ids - |> Enum.chunk_every(chunk_size) - |> Enum.each(fn chunk -> - IO.puts("Adding #{length(chunk)} activities to indexing queue") - - chunk - |> Enum.map(fn id -> - SearchIndexingWorker.new(%{"op" => "add_to_index", "activity" => id}) - end) - |> Oban.insert_all() - end) - end) + unless result.exhausted do + IO.puts("Resume with --before #{result.next_cursor}") + end end end diff --git a/lib/pleroma/search/backfill.ex b/lib/pleroma/search/backfill.ex new file mode 100644 index 000000000..1b986bb59 --- /dev/null +++ b/lib/pleroma/search/backfill.ex @@ -0,0 +1,137 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2026 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Search.Backfill do + import Ecto.Query + + alias Pleroma.Activity + alias Pleroma.Workers.SearchIndexingWorker + + @default_chunk_size 100 + @default_step_size 1_000 + @default_limit 100_000 + + @doc """ + Enqueues existing Create activities for search indexing. + + The returned `next_cursor` can be passed as `before` to resume a bounded run. + Pass `limit: :infinity` to continue until the oldest activity is reached. + """ + def run(options \\ []) do + options = + Keyword.validate!(options, + chunk: @default_chunk_size, + step: @default_step_size, + limit: @default_limit, + before: nil, + on_page: nil + ) + + chunk_size = positive_option!(options, :chunk, @default_chunk_size) + step_size = positive_option!(options, :step, @default_step_size) + limit = limit_option!(options) + before = cursor_option!(options[:before]) + on_page = callback_option!(options[:on_page]) + + do_run(before, limit, step_size, chunk_size, on_page, 0) + end + + defp do_run(before, remaining, step_size, chunk_size, on_page, enqueued) do + page_size = page_size(remaining, step_size) + ids = fetch_ids(before, page_size) + + case ids do + [] -> + result(enqueued, before, true) + + ids -> + enqueue(ids, chunk_size) + + page_count = length(ids) + next_cursor = List.last(ids) + total = enqueued + page_count + remaining = decrement(remaining, page_count) + + on_page.(%{count: page_count, enqueued: total, next_cursor: next_cursor}) + + cond do + remaining == 0 -> result(total, next_cursor, false) + page_count < page_size -> result(total, next_cursor, true) + true -> do_run(next_cursor, remaining, step_size, chunk_size, on_page, total) + end + end + end + + defp fetch_ids(before, limit) do + Activity + |> where([a], fragment("?->>'type' = 'Create'", a.data)) + |> maybe_before(before) + |> order_by([a], desc: a.id) + |> limit(^limit) + |> select([a], a.id) + |> Pleroma.Repo.all() + end + + defp maybe_before(query, nil), do: query + defp maybe_before(query, before), do: where(query, [a], a.id < ^before) + + defp enqueue(ids, chunk_size) do + ids + |> Enum.chunk_every(chunk_size) + |> Enum.each(fn chunk -> + chunk + |> Enum.map(&SearchIndexingWorker.new(%{"op" => "add_to_index", "activity" => &1})) + |> Oban.insert_all() + end) + end + + defp page_size(:infinity, step_size), do: step_size + defp page_size(remaining, step_size), do: min(remaining, step_size) + + defp decrement(:infinity, _count), do: :infinity + defp decrement(remaining, count), do: remaining - count + + defp result(enqueued, next_cursor, exhausted) do + %{enqueued: enqueued, next_cursor: next_cursor, exhausted: exhausted} + end + + defp positive_option!(options, key, default) do + case Keyword.get(options, key, default) do + value when is_integer(value) and value > 0 -> value + value -> raise ArgumentError, "#{key} must be a positive integer, got: #{inspect(value)}" + end + end + + defp limit_option!(options) do + case Keyword.fetch!(options, :limit) do + :infinity -> :infinity + value when is_integer(value) and value > 0 -> value + value -> raise ArgumentError, "limit must be a positive integer, got: #{inspect(value)}" + end + end + + defp cursor_option!(nil), do: nil + + defp cursor_option!(cursor) do + valid? = + try do + match?({:ok, <<_::binary-size(16)>>}, FlakeId.Ecto.CompatType.dump(cursor)) + rescue + _ -> false + end + + if valid? do + cursor + else + raise ArgumentError, "before must be a valid activity id, got: #{inspect(cursor)}" + end + end + + defp callback_option!(nil), do: fn _ -> :ok end + defp callback_option!(callback) when is_function(callback, 1), do: callback + + defp callback_option!(callback) do + raise ArgumentError, "on_page must be a function of arity 1, got: #{inspect(callback)}" + end +end diff --git a/test/pleroma/search/backfill_test.exs b/test/pleroma/search/backfill_test.exs new file mode 100644 index 000000000..d707524c3 --- /dev/null +++ b/test/pleroma/search/backfill_test.exs @@ -0,0 +1,79 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2026 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Search.BackfillTest do + use Pleroma.DataCase, async: false + use Oban.Testing, repo: Pleroma.Repo + + import Pleroma.Factory + + alias Pleroma.Search.Backfill + alias Pleroma.Workers.SearchIndexingWorker + + test "enqueues the exact requested number across partial pages" do + activities = insert_list(5, :note_activity) |> Enum.sort_by(& &1.id, :desc) + + assert %{enqueued: 3, exhausted: false, next_cursor: cursor} = + Backfill.run(limit: 3, step: 2, chunk: 2) + + assert search_indexing_ids() == Enum.map(Enum.take(activities, 3), & &1.id) + assert cursor == Enum.at(activities, 2).id + end + + test "resumes after the last processed activity without duplicates" do + activities = insert_list(5, :note_activity) |> Enum.sort_by(& &1.id, :desc) + + assert %{enqueued: 2, next_cursor: cursor} = Backfill.run(limit: 2, step: 1) + + Pleroma.Repo.delete_all(Oban.Job) + + assert %{enqueued: 2, next_cursor: next_cursor} = + Backfill.run(before: cursor, limit: 2, step: 1) + + assert search_indexing_ids() == + activities + |> Enum.slice(2, 2) + |> Enum.map(& &1.id) + + assert next_cursor == Enum.at(activities, 3).id + end + + test "only queues Create activities" do + create = insert(:note_activity) + insert(:like_activity, note_activity: create) + + assert %{enqueued: 1, exhausted: true, next_cursor: cursor} = Backfill.run(step: 2) + assert search_indexing_ids() == [create.id] + assert cursor == create.id + end + + test "rejects unknown options and malformed cursors before enqueueing" do + assert_raise ArgumentError, fn -> Backfill.run(limt: 1) end + assert_raise ArgumentError, fn -> Backfill.run(before: "not-an-activity-id") end + assert_raise ArgumentError, fn -> Backfill.run(on_page: :invalid) end + + assert all_enqueued(worker: SearchIndexingWorker) == [] + end + + test "does not duplicate or skip the initial activities when newer rows are inserted" do + activities = insert_list(3, :note_activity) |> Enum.sort_by(& &1.id, :desc) + + assert %{enqueued: 3, exhausted: true} = + Backfill.run( + limit: :infinity, + step: 1, + on_page: fn %{enqueued: enqueued} -> + if enqueued == 1, do: insert(:note_activity) + end + ) + + assert search_indexing_ids() == Enum.map(activities, & &1.id) + end + + defp search_indexing_ids do + all_enqueued(worker: SearchIndexingWorker) + |> Enum.sort_by(& &1.id) + |> Enum.map(& &1.args["activity"]) + end +end From 271c3a3dd8a22fad2fa4cd6d034c98e1ffc91fb4 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sun, 26 Jul 2026 11:58:31 +0400 Subject: [PATCH 174/183] fix(config): keep search startup settings static --- lib/mix/tasks/pleroma/config.ex | 9 ++++--- lib/pleroma/config/transfer_task.ex | 1 + lib/pleroma/config_db.ex | 26 +++++++++++++++++-- .../controllers/config_controller.ex | 5 +++- test/fixtures/config/static_search.secret.exs | 5 ++++ test/mix/tasks/pleroma/config_test.exs | 8 ++++++ test/pleroma/config/transfer_task_test.exs | 25 ++++++++++++++++++ test/pleroma/config_db_test.exs | 14 ++++++++++ .../controllers/config_controller_test.exs | 20 ++++++++++++-- 9 files changed, 105 insertions(+), 8 deletions(-) create mode 100644 test/fixtures/config/static_search.secret.exs diff --git a/lib/mix/tasks/pleroma/config.ex b/lib/mix/tasks/pleroma/config.ex index a10506236..81c41d967 100644 --- a/lib/mix/tasks/pleroma/config.ex +++ b/lib/mix/tasks/pleroma/config.ex @@ -335,9 +335,12 @@ defmodule Mix.Tasks.Pleroma.Config do group |> Pleroma.Config.Loader.filter_group(settings) |> Enum.each(fn {key, value} -> - {:ok, _} = ConfigDB.update_or_create(%{group: group, key: key, value: value}) - - shell_info("Settings for key #{key} migrated.") + if ConfigDB.static?(group, key) do + shell_info("Settings for key #{key} must remain in static configuration; skipping.") + else + {:ok, _} = ConfigDB.update_or_create(%{group: group, key: key, value: value}) + shell_info("Settings for key #{key} migrated.") + end end) shell_info("Settings for group #{inspect(group)} migrated.") diff --git a/lib/pleroma/config/transfer_task.ex b/lib/pleroma/config/transfer_task.ex index 3e59dc4e0..48cb158ca 100644 --- a/lib/pleroma/config/transfer_task.ex +++ b/lib/pleroma/config/transfer_task.ex @@ -47,6 +47,7 @@ defmodule Pleroma.Config.TransferTask do settings = (Repo.all(ConfigDB) ++ deleted_settings) + |> Enum.reject(&ConfigDB.static?/1) |> Enum.map(&merge_with_default/1) started_applications = Application.started_applications() diff --git a/lib/pleroma/config_db.ex b/lib/pleroma/config_db.ex index 2c3df773b..9aa52f7d6 100644 --- a/lib/pleroma/config_db.ex +++ b/lib/pleroma/config_db.ex @@ -23,6 +23,12 @@ defmodule Pleroma.ConfigDB do {:pleroma, :mrf_keyword, :replace} ] + @static_config [ + {:pleroma, Pleroma.Search}, + {:pleroma, Pleroma.Search.ParadeDB}, + {:pleroma, Pleroma.Search.ParadeDB.Repo} + ] + schema "config" do field(:key, Pleroma.EctoType.Config.Atom) field(:group, Pleroma.EctoType.Config.Atom) @@ -32,16 +38,32 @@ defmodule Pleroma.ConfigDB do timestamps() end - @spec get_all_as_keyword() :: keyword() - def get_all_as_keyword do + @spec get_all_as_keyword(keyword()) :: keyword() + def get_all_as_keyword(options \\ []) do ConfigDB |> select([c], {c.group, c.key, c.value}) |> Repo.all() + |> maybe_reject_static(options) |> Enum.reduce([], fn {group, key, value}, acc -> Keyword.update(acc, group, [{key, value}], &Keyword.merge(&1, [{key, value}])) end) end + def static?(%{group: group, key: key}), do: static?(group, key) + def static?(_config), do: false + + def static?(group, key) do + Enum.any?(@static_config, fn {static_group, static_key} -> + group in [static_group, inspect(static_group)] and key in [static_key, inspect(static_key)] + end) + end + + defp maybe_reject_static(configs, exclude_static: true) do + Enum.reject(configs, fn {group, key, _value} -> static?(group, key) end) + end + + defp maybe_reject_static(configs, _options), do: configs + @spec get_all_by_group(atom() | String.t()) :: [t()] def get_all_by_group(group) do from(c in ConfigDB, where: c.group == ^group) |> Repo.all() diff --git a/lib/pleroma/web/admin_api/controllers/config_controller.ex b/lib/pleroma/web/admin_api/controllers/config_controller.ex index 753690d45..4c76d808d 100644 --- a/lib/pleroma/web/admin_api/controllers/config_controller.ex +++ b/lib/pleroma/web/admin_api/controllers/config_controller.ex @@ -75,6 +75,7 @@ defmodule Pleroma.Web.AdminAPI.ConfigController do Pleroma.Docs.JSON.compiled_descriptions() |> Enum.filter(&allowed_config?(&1, :whitelist)) |> Enum.reject(&allowed_config?(&1, :blacklist)) + |> Enum.reject(&ConfigDB.static?/1) json(conn, translate_descriptions(descriptions)) end @@ -92,7 +93,7 @@ defmodule Pleroma.Web.AdminAPI.ConfigController do def show(conn, _params) do with :ok <- configurable_from_database() do - configs = ConfigDB.get_all_as_keyword() + configs = ConfigDB.get_all_as_keyword(exclude_static: true) merged = Config.Holder.default_config() @@ -123,6 +124,7 @@ defmodule Pleroma.Web.AdminAPI.ConfigController do end) end) |> List.flatten() + |> Enum.reject(&ConfigDB.static?/1) render(conn, "index.json", %{ configs: merged, @@ -137,6 +139,7 @@ defmodule Pleroma.Web.AdminAPI.ConfigController do configs |> Enum.filter(&allowed_config?(&1, :whitelist)) |> Enum.reject(&allowed_config?(&1, :blacklist)) + |> Enum.reject(&(ConfigDB.static?(&1) and not &1[:delete])) |> Enum.map(fn %{group: group, key: key, delete: true} = params -> ConfigDB.delete(%{group: group, key: key, subkeys: params[:subkeys]}) diff --git a/test/fixtures/config/static_search.secret.exs b/test/fixtures/config/static_search.secret.exs new file mode 100644 index 000000000..13f744184 --- /dev/null +++ b/test/fixtures/config/static_search.secret.exs @@ -0,0 +1,5 @@ +import Config + +config :pleroma, Pleroma.Search, module: Pleroma.Search.ParadeDB +config :pleroma, Pleroma.Search.ParadeDB, url: "postgres://localhost/paradedb" +config :pleroma, Pleroma.Search.ParadeDB.Repo, pool_size: 2 diff --git a/test/mix/tasks/pleroma/config_test.exs b/test/mix/tasks/pleroma/config_test.exs index 7513e958e..620bf8662 100644 --- a/test/mix/tasks/pleroma/config_test.exs +++ b/test/mix/tasks/pleroma/config_test.exs @@ -87,6 +87,14 @@ defmodule Mix.Tasks.Pleroma.ConfigTest do config = ConfigDB.get_by_params(%{group: ":pleroma", key: ":first_setting"}) assert config.value == [key: "value", key2: [Repo]] end + + test "static search settings remain in the config file" do + MixTask.migrate_to_db("test/fixtures/config/static_search.secret.exs") + + refute ConfigDB.get_by_group_and_key(:pleroma, Pleroma.Search) + refute ConfigDB.get_by_group_and_key(:pleroma, Pleroma.Search.ParadeDB) + refute ConfigDB.get_by_group_and_key(:pleroma, Pleroma.Search.ParadeDB.Repo) + end end describe "with deletion of temp file" do diff --git a/test/pleroma/config/transfer_task_test.exs b/test/pleroma/config/transfer_task_test.exs index 6295fa888..ea5378ffd 100644 --- a/test/pleroma/config/transfer_task_test.exs +++ b/test/pleroma/config/transfer_task_test.exs @@ -71,6 +71,31 @@ defmodule Pleroma.Config.TransferTaskTest do assert assets_env[:mascots] == [a: 1, b: 2] end + test "does not transfer static search configuration" do + search_config = Application.get_env(:pleroma, Pleroma.Search) + parade_config = Application.get_env(:pleroma, Pleroma.Search.ParadeDB) + + clear_config(:database_config_blacklist, []) + + insert(:config, key: Pleroma.Search, value: [module: Pleroma.Search.ParadeDB]) + + insert(:config, + key: Pleroma.Search.ParadeDB, + value: [url: "postgres://localhost/paradedb"] + ) + + insert(:config, + key: Pleroma.Search.ParadeDB.Repo, + value: [pool_size: 50] + ) + + TransferTask.start_link([]) + + assert Application.get_env(:pleroma, Pleroma.Search) == search_config + assert Application.get_env(:pleroma, Pleroma.Search.ParadeDB) == parade_config + assert Application.get_env(:pleroma, Pleroma.Search.ParadeDB.Repo)[:pool_size] == 2 + end + describe "pleroma restart" do setup do on_exit(fn -> diff --git a/test/pleroma/config_db_test.exs b/test/pleroma/config_db_test.exs index 9d70e6d54..ae17a9d34 100644 --- a/test/pleroma/config_db_test.exs +++ b/test/pleroma/config_db_test.exs @@ -36,6 +36,20 @@ defmodule Pleroma.ConfigDBTest do assert config[:goose][:webhook_url] == "https://gander.com/" end + test "get_all_as_keyword/1 excludes static search configuration" do + insert(:config, key: Pleroma.Search, value: [module: Pleroma.Search.ParadeDB]) + insert(:config, key: Pleroma.Search.ParadeDB, value: [url: "postgres://example/db"]) + insert(:config, key: Pleroma.Search.ParadeDB.Repo, value: [pool_size: 50]) + insert(:config, key: :instance, value: [name: "Pleroma"]) + + config = ConfigDB.get_all_as_keyword(exclude_static: true) + + refute config[:pleroma][Pleroma.Search] + refute config[:pleroma][Pleroma.Search.ParadeDB] + refute config[:pleroma][Pleroma.Search.ParadeDB.Repo] + assert config[:pleroma][:instance] == [name: "Pleroma"] + end + describe "update_or_create/1" do test "common" do config = insert(:config) diff --git a/test/pleroma/web/admin_api/controllers/config_controller_test.exs b/test/pleroma/web/admin_api/controllers/config_controller_test.exs index c11ece10d..e4c8d866f 100644 --- a/test/pleroma/web/admin_api/controllers/config_controller_test.exs +++ b/test/pleroma/web/admin_api/controllers/config_controller_test.exs @@ -121,6 +121,19 @@ defmodule Pleroma.Web.AdminAPI.ConfigControllerTest do end) end + test "effective configuration omits static search settings", %{conn: conn} do + insert(:config, key: Pleroma.Search, value: [module: Pleroma.Search.ParadeDB]) + insert(:config, key: Pleroma.Search.ParadeDB, value: [url: "postgres://example/db"]) + + %{"configs" => configs} = + conn + |> get("/api/pleroma/admin/config") + |> json_response_and_validate_schema(200) + + refute Enum.any?(configs, &(&1["key"] == "Pleroma.Search")) + refute Enum.any?(configs, &(&1["key"] == "Pleroma.Search.ParadeDB")) + end + test "subkeys with full update right merge", %{conn: conn} do insert(:config, key: ":emoji", @@ -1552,12 +1565,15 @@ defmodule Pleroma.Web.AdminAPI.ConfigControllerTest do assert web_endpoint["children"] end - test "all keys from description are whitelisted", %{conn: conn} do + test "descriptions exclude blacklisted keys", %{conn: conn} do conn = get(conn, "/api/pleroma/admin/config/descriptions") assert response = json_response_and_validate_schema(conn, 200) - assert length(response) == length(Pleroma.Docs.JSON.compiled_descriptions()) + refute Enum.any?(response, &(&1["key"] == "Pleroma.Search")) + refute Enum.any?(response, &(&1["key"] == "Pleroma.Search.ParadeDB")) + + assert length(response) == length(Pleroma.Docs.JSON.compiled_descriptions()) - 2 end test "filters by database configuration blacklist and whitelist", %{conn: conn} do From 13c944a04b217a534cd9ef0d0f9d8dfd65609faf Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sun, 26 Jul 2026 12:03:24 +0400 Subject: [PATCH 175/183] fix(search): index remote attachment descriptions --- lib/pleroma/search.ex | 1 + test/pleroma/search_test.exs | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/lib/pleroma/search.ex b/lib/pleroma/search.ex index cd4907026..0d9c7c6e3 100644 --- a/lib/pleroma/search.ex +++ b/lib/pleroma/search.ex @@ -78,6 +78,7 @@ defmodule Pleroma.Search do defp attachment_names(attachments) when is_list(attachments) do Enum.map(attachments, fn + %{"summary" => summary} when is_binary(summary) -> summary %{"name" => name} -> name _ -> nil end) diff --git a/test/pleroma/search_test.exs b/test/pleroma/search_test.exs index 00d8b10ed..91521b1a5 100644 --- a/test/pleroma/search_test.exs +++ b/test/pleroma/search_test.exs @@ -83,6 +83,11 @@ defmodule Pleroma.SearchTest do "summary" => "subject text", "attachment" => [ %{"type" => "Document", "name" => "image description"}, + %{ + "type" => "Document", + "name" => "remote-image.jpg", + "summary" => "remote image description" + }, %{"type" => "Document", "name" => ""} ], "published" => DateTime.utc_now() |> DateTime.to_iso8601() @@ -93,6 +98,8 @@ defmodule Pleroma.SearchTest do assert content =~ "body text" assert content =~ "subject text" assert content =~ "image description" + assert content =~ "remote image description" + refute content =~ "remote-image.jpg" end test "object_to_search_data ignores objects with invalid published dates" do From f90a52398ba8825d70db9f4594904eca02e248b5 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sun, 26 Jul 2026 12:16:04 +0400 Subject: [PATCH 176/183] ci(search): gate pinned ParadeDB integration --- .woodpecker/paradedb-integration.yaml | 64 ++++++++ .../search/parade_db_integration_test.exs | 145 +++++++++++++----- 2 files changed, 171 insertions(+), 38 deletions(-) create mode 100644 .woodpecker/paradedb-integration.yaml diff --git a/.woodpecker/paradedb-integration.yaml b/.woodpecker/paradedb-integration.yaml new file mode 100644 index 000000000..b91222837 --- /dev/null +++ b/.woodpecker/paradedb-integration.yaml @@ -0,0 +1,64 @@ +when: + - event: pull_request + path: + - ".woodpecker/**" + - "lib/pleroma/search/**" + - "test/pleroma/search/**" + - "mix.exs" + - "mix.lock" + evaluate: 'CI_COMMIT_SOURCE_BRANCH != "weblate"' + - event: push + branch: ${CI_REPO_DEFAULT_BRANCH} + path: + - ".woodpecker/**" + - "lib/pleroma/search/**" + - "test/pleroma/search/**" + - "mix.exs" + - "mix.lock" + +depends_on: + - lint + +labels: + platform: linux/amd64 + +variables: + script_file_entrypoint: &script_file_entrypoint + - /bin/sh + - -c + - 'printf "%s" "$CI_SCRIPT" | base64 -d > /tmp/ci-script.sh && /bin/sh -xe /tmp/ci-script.sh' + +steps: + paradedb-integration: + image: docker.io/elixir:1.19-otp-28-alpine + entrypoint: *script_file_entrypoint + environment: + MIX_ENV: test + VIX_COMPILATION_MODE: PLATFORM_PROVIDED_LIBVIPS + DB_HOST: postgres + DB_PORT: 5432 + PARADEDB_TEST_URL: postgres://postgres:postgres@paradedb:5432/paradedb + commands: + - apk add --no-cache build-base cmake exiftool ffmpeg vips-dev file-dev git openssl postgresql-client + - mix local.hex --force + - mix local.rebar --force + - mix deps.get + - timeout 60 sh -c 'until pg_isready -h postgres -U postgres -d pleroma_test; do sleep 1; done' || { echo "PostgreSQL did not become ready" >&2; exit 1; } + - timeout 60 sh -c 'until pg_isready -h paradedb -U postgres -d paradedb; do sleep 1; done' || { echo "ParadeDB did not become ready" >&2; exit 1; } + - mix ecto.create + - mix ecto.migrate + - mix test --include integration test/pleroma/search/parade_db_integration_test.exs + +services: + postgres: + image: docker.io/postgres:13-alpine + environment: + POSTGRES_DB: pleroma_test + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + paradedb: + image: docker.io/paradedb/paradedb:0.24.3-pg15@sha256:997d62f6c2605becb7c45878af3f8c8c698f355006b675b3ce212d4eb2d14f2e + environment: + POSTGRES_DB: paradedb + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres diff --git a/test/pleroma/search/parade_db_integration_test.exs b/test/pleroma/search/parade_db_integration_test.exs index 2687260cf..4336ecfc3 100644 --- a/test/pleroma/search/parade_db_integration_test.exs +++ b/test/pleroma/search/parade_db_integration_test.exs @@ -14,7 +14,10 @@ defmodule Pleroma.Search.ParadeDBIntegrationTest do alias Pleroma.Search.ParadeDB alias Pleroma.Web.CommonAPI - @image System.get_env("PARADEDB_TEST_IMAGE", "paradedb/paradedb:latest") + @image System.get_env( + "PARADEDB_TEST_IMAGE", + "paradedb/paradedb:0.24.3-pg15@sha256:997d62f6c2605becb7c45878af3f8c8c698f355006b675b3ce212d4eb2d14f2e" + ) @db_user "postgres" @db_password "postgres" @db_name "paradedb" @@ -26,6 +29,13 @@ defmodule Pleroma.Search.ParadeDBIntegrationTest do end setup_all do + case System.get_env("PARADEDB_TEST_URL") do + nil -> setup_docker_paradedb() + url -> setup_external_paradedb(url) + end + end + + defp setup_docker_paradedb do with :ok <- ensure_docker_available() do container = "pleroma-paradedb-it-#{System.unique_integer([:positive])}" @@ -58,49 +68,67 @@ defmodule Pleroma.Search.ParadeDBIntegrationTest do |> wait_for_port!(@docker_startup_timeout_ms) wait_for_pg_ready!(container, @docker_startup_timeout_ms) - wait_for_tcp_ready!(host_port, @docker_startup_timeout_ms) url = "postgres://#{@db_user}:#{@db_password}@127.0.0.1:#{host_port}/#{@db_name}" - - prev_env = System.get_env("PARADEDB_DATABASE_URL") - System.put_env("PARADEDB_DATABASE_URL", url) - - on_exit(fn -> - if is_nil(prev_env) do - System.delete_env("PARADEDB_DATABASE_URL") - else - System.put_env("PARADEDB_DATABASE_URL", prev_env) - end - end) - - {:ok, _pid} = start_supervised({Pleroma.Search.ParadeDB.Repo, pool_size: 5}) - - previous_table = Pleroma.Config.get([Pleroma.Search.ParadeDB, :table], nil) - table = "pleroma_search_documents_it_#{System.unique_integer([:positive])}" - Pleroma.Config.put([Pleroma.Search.ParadeDB, :table], table) - - on_exit(fn -> - if is_nil(previous_table) do - Pleroma.Config.delete([Pleroma.Search.ParadeDB, :table]) - else - Pleroma.Config.put([Pleroma.Search.ParadeDB, :table], previous_table) - end - end) - - {:ok, - %{ - container: container, - container_id: container_id, - url: url, - table: table - }} + setup_repo(url, %{container: container, container_id: container_id}) end end - test "indexes and searches against real ParadeDB" do + defp setup_external_paradedb(url) do + uri = URI.parse(url) + wait_for_tcp_ready!(uri.host, uri.port || 5432, @docker_startup_timeout_ms) + setup_repo(url, %{}) + end + + defp setup_repo(url, context) do + prev_env = System.get_env("PARADEDB_DATABASE_URL") + System.put_env("PARADEDB_DATABASE_URL", url) + + on_exit(fn -> + if is_nil(prev_env) do + System.delete_env("PARADEDB_DATABASE_URL") + else + System.put_env("PARADEDB_DATABASE_URL", prev_env) + end + end) + + {:ok, _pid} = start_supervised({Pleroma.Search.ParadeDB.Repo, pool_size: 5}) + wait_for_repo_ready!(@docker_startup_timeout_ms) + + previous_table = Pleroma.Config.get([Pleroma.Search.ParadeDB, :table], nil) + suffix = Ecto.UUID.generate() |> String.replace("-", "") + table = "pleroma_search_documents_it_#{suffix}" + Pleroma.Config.put([Pleroma.Search.ParadeDB, :table], table) + + on_exit(fn -> + if is_nil(previous_table) do + Pleroma.Config.delete([Pleroma.Search.ParadeDB, :table]) + else + Pleroma.Config.put([Pleroma.Search.ParadeDB, :table], previous_table) + end + end) + + {:ok, Map.merge(context, %{url: url, table: table})} + end + + test "indexes and searches against real ParadeDB", %{table: table} do + on_exit(fn -> + _ = + Ecto.Adapters.SQL.query( + Pleroma.Search.ParadeDB.Repo, + "DROP TABLE IF EXISTS #{table}" + ) + end) + :ok = ParadeDB.drop_index() assert :ok = ParadeDB.create_index() + assert {:ok, %{rows: [["0.24.3"]]}} = + Ecto.Adapters.SQL.query( + Pleroma.Search.ParadeDB.Repo, + "SELECT extversion FROM pg_extension WHERE extname = 'pg_search'" + ) + user = insert(:user) {:ok, a1} = @@ -118,6 +146,23 @@ defmodule Pleroma.Search.ParadeDBIntegrationTest do a2 = Activity.get_by_id_with_object(a2.id) private = Activity.get_by_id_with_object(private.id) + a1 = %{ + a1 + | object: %{ + a1.object + | data: + Map.merge(a1.object.data, %{ + "summary" => "paradedb subject phrase", + "attachment" => [ + %{ + "name" => "remote-image.jpg", + "summary" => "paradedb attachment description" + } + ] + }) + } + } + assert :ok = ParadeDB.add_to_index(a1) assert :ok = ParadeDB.add_to_index(a2) assert :ok = ParadeDB.add_to_index(private) @@ -130,6 +175,9 @@ defmodule Pleroma.Search.ParadeDBIntegrationTest do refute private.id in ids assert List.first(ids) == a2.id + assert Enum.any?(ParadeDB.search(nil, "subject phrase"), &(&1.id == a1.id)) + assert Enum.any?(ParadeDB.search(nil, "attachment description"), &(&1.id == a1.id)) + previous_fuzzy_distance = Pleroma.Config.get([Pleroma.Search.ParadeDB, :fuzzy_distance], 0) Pleroma.Config.put([Pleroma.Search.ParadeDB, :fuzzy_distance], 1) @@ -226,11 +274,11 @@ defmodule Pleroma.Search.ParadeDBIntegrationTest do wait.(wait) end - defp wait_for_tcp_ready!(port, timeout_ms) do + defp wait_for_tcp_ready!(host, port, timeout_ms) do deadline = System.monotonic_time(:millisecond) + timeout_ms wait = fn wait -> - case :gen_tcp.connect(~c"127.0.0.1", port, [:binary, active: false], 500) do + case :gen_tcp.connect(String.to_charlist(host), port, [:binary, active: false], 500) do {:ok, socket} -> :gen_tcp.close(socket) :ok @@ -248,6 +296,27 @@ defmodule Pleroma.Search.ParadeDBIntegrationTest do wait.(wait) end + defp wait_for_repo_ready!(timeout_ms) do + deadline = System.monotonic_time(:millisecond) + timeout_ms + + wait = fn wait -> + case Ecto.Adapters.SQL.query(Pleroma.Search.ParadeDB.Repo, "SELECT 1") do + {:ok, _} -> + :ok + + _ -> + if System.monotonic_time(:millisecond) > deadline do + raise "Timed out waiting for ParadeDB Repo" + else + :timer.sleep(250) + wait.(wait) + end + end + end + + wait.(wait) + end + defp docker(args) do System.cmd("docker", args, stderr_to_stdout: true) end From 9df50eea82e4141bca673750448c5e795d31fa49 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sun, 26 Jul 2026 12:26:35 +0400 Subject: [PATCH 177/183] fix: reject duplicate Signature headers --- .../activitypub-bot-rfc9421-fallback.fix | 2 +- .../activity_pub/activity_pub_controller.ex | 42 ++++++++---- .../activity_pub_controller_test.exs | 67 +++++++++++++++++++ 3 files changed, 97 insertions(+), 14 deletions(-) diff --git a/changelog.d/activitypub-bot-rfc9421-fallback.fix b/changelog.d/activitypub-bot-rfc9421-fallback.fix index 22642fee1..a04940bfe 100644 --- a/changelog.d/activitypub-bot-rfc9421-fallback.fix +++ b/changelog.d/activitypub-bot-rfc9421-fallback.fix @@ -1 +1 @@ -Reject unsupported RFC 9421 inbox signatures instead of queueing a doomed failed-signature retry, allowing clients to fall back to legacy HTTP Signatures. +Reject unsupported or ambiguous inbox signatures instead of queueing a doomed failed-signature retry, allowing clients to fall back to legacy HTTP Signatures. diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index dd644c132..38b8b8bfc 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -309,7 +309,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do } = conn, %{"nickname" => nickname} = params ) do - with {:recipient_exists, %User{} = recipient} <- + with false <- unsupported_http_message_signature?(conn), + {:recipient_exists, %User{} = recipient} <- {:recipient_exists, User.get_cached_by_nickname(nickname)}, {:sender_exists, {:ok, %User{} = actor}} <- {:sender_exists, User.get_or_fetch_by_ap_id(params["actor"])}, @@ -320,6 +321,9 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do Federator.incoming_ap_doc(params) json(conn, "ok") else + true -> + reject_unsupported_http_message_signature(conn) + {:recipient_exists, _} -> conn |> put_status(:not_found) @@ -348,15 +352,17 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do end def inbox(%{assigns: %{valid_signature: true, valid_host_header: true}} = conn, params) do - Federator.incoming_ap_doc(params) - json(conn, "ok") + if unsupported_http_message_signature?(conn) do + reject_unsupported_http_message_signature(conn) + else + Federator.incoming_ap_doc(params) + json(conn, "ok") + end end def inbox(%{assigns: %{valid_signature: false}} = conn, params) do if unsupported_http_message_signature?(conn) do - conn - |> put_status(:unauthorized) - |> json("error, unsupported HTTP Message Signature") + reject_unsupported_http_message_signature(conn) else Federator.incoming_failed_signature_ap_doc(%{ method: conn.method, @@ -387,15 +393,25 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do |> json("error, missing HTTP Signature") end - defp unsupported_http_message_signature?(conn) do - Plug.Conn.get_req_header(conn, "signature-input") != [] and - not cavage_signature?(conn) + defp reject_unsupported_http_message_signature(conn) do + conn + |> put_status(:unauthorized) + |> json("error, unsupported HTTP Message Signature") end - defp cavage_signature?(conn) do - conn - |> Plug.Conn.get_req_header("signature") - |> Enum.any?(&Regex.match?(~r/(^|,)\s*keyId\s*=/, &1)) + defp unsupported_http_message_signature?(conn) do + signature_headers = Plug.Conn.get_req_header(conn, "signature") + + multiple_signature_headers?(signature_headers) or + (Plug.Conn.get_req_header(conn, "signature-input") != [] and + not cavage_signature?(signature_headers)) + end + + defp multiple_signature_headers?([_, _ | _]), do: true + defp multiple_signature_headers?(_), do: false + + defp cavage_signature?(signature_headers) do + Enum.any?(signature_headers, &Regex.match?(~r/(^|,)\s*keyId\s*=/, &1)) end defp post_inbox_relayed_create(conn, params) do diff --git a/test/pleroma/web/activity_pub/activity_pub_controller_test.exs b/test/pleroma/web/activity_pub/activity_pub_controller_test.exs index e9a777758..0de9618d4 100644 --- a/test/pleroma/web/activity_pub/activity_pub_controller_test.exs +++ b/test/pleroma/web/activity_pub/activity_pub_controller_test.exs @@ -729,6 +729,22 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do assert Activity.get_by_ap_id(data["id"]) end + test "rejects duplicate Signature headers before processing signed inbox requests", %{ + conn: conn + } do + data = File.read!("test/fixtures/mastodon-post-activity.json") |> Jason.decode!() + + conn = + conn + |> assign_valid_signature_for_actor(data["actor"]) + |> put_req_header("content-type", "application/activity+json") + |> Map.update!(:req_headers, &[{"signature", "sig1=:fake-signature:"} | &1]) + |> post("/inbox", data) + + assert "error, unsupported HTTP Message Signature" == json_response(conn, 401) + assert [] == all_enqueued(worker: ReceiverWorker) + end + test "it inserts an incoming activity into the database" <> "even if we can't fetch the user but have it in our db", %{conn: conn} do @@ -794,6 +810,43 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do assert [] == all_enqueued(worker: SignatureRetryWorker) end + test "rejects duplicate Signature headers so clients can fall back", %{conn: conn} do + data = %{ + "type" => "Create", + "actor" => "https://activitypubbot.example/user/ok", + "id" => "https://activitypubbot.example/activities/duplicate-signature-create", + "to" => ["https://www.w3.org/ns/activitystreams#Public"], + "cc" => [], + "object" => %{ + "type" => "Note", + "id" => "https://activitypubbot.example/objects/duplicate-signature-note", + "actor" => "https://activitypubbot.example/user/ok", + "attributedTo" => "https://activitypubbot.example/user/ok", + "content" => "hello from activitypub-bot", + "published" => "2026-05-29T13:33:31Z", + "to" => ["https://www.w3.org/ns/activitystreams#Public"], + "cc" => [] + } + } + + legacy_signature = + "keyId=\"https://activitypubbot.example/user/ok#main-key\",algorithm=\"rsa-sha256\",headers=\"(request-target) host date digest content-type\",signature=\"fake-signature\"" + + Mox.expect(Pleroma.StubbedHTTPSignaturesMock, :validate_conn, fn _conn -> false end) + + conn = + conn + |> put_req_header("content-type", "application/activity+json") + |> put_req_header("date", "Fri, 29 May 2026 13:33:31 GMT") + |> put_req_header("digest", "SHA-256=fake-digest") + |> put_req_header("signature", "sig1=:fake-signature:") + |> Map.update!(:req_headers, &[{"signature", legacy_signature} | &1]) + |> post("/inbox", data) + + assert "error, unsupported HTTP Message Signature" == json_response(conn, 401) + assert [] == all_enqueued(worker: SignatureRetryWorker) + end + test "keeps failed-signature retry for legacy signatures with Signature-Input", %{conn: conn} do data = %{ "type" => "Create", @@ -1397,6 +1450,20 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do assert Activity.get_by_ap_id(data["id"]) end + test "rejects duplicate Signature headers", %{conn: conn, data: data} do + user = insert(:user) + + conn = + conn + |> assign_valid_signature_for_actor(data["actor"]) + |> put_req_header("content-type", "application/activity+json") + |> Map.update!(:req_headers, &[{"signature", "sig1=:fake-signature:"} | &1]) + |> post("/users/#{user.nickname}/inbox", data) + + assert "error, unsupported HTTP Message Signature" == json_response(conn, 401) + assert [] == all_enqueued(worker: ReceiverWorker) + end + test "it accepts messages with to as string instead of array", %{conn: conn, data: data} do user = insert(:user) From cece76e4cbce194a56847057f120550794c4f4fe Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sun, 26 Jul 2026 12:32:24 +0400 Subject: [PATCH 178/183] docs(search): document ParadeDB operations --- docs/configuration/search.md | 125 ++++++++++++++++++++++++++++++++--- 1 file changed, 117 insertions(+), 8 deletions(-) diff --git a/docs/configuration/search.md b/docs/configuration/search.md index 1d5056ddf..5820cdbd9 100644 --- a/docs/configuration/search.md +++ b/docs/configuration/search.md @@ -150,18 +150,22 @@ depends on the amount of text in posts. [ParadeDB](https://www.paradedb.com/) is a Postgres extension that provides BM25 full-text search. In Pleroma, it can be used as an external search backend by pointing Pleroma at a separate Postgres instance with the `pg_search` extension installed. +Pleroma's integration suite tests `pg_search` 0.24.3 on PostgreSQL 15. Pleroma will maintain a small `pleroma_search_documents` table in that database (via the existing search indexing queue) and run search queries against it. The indexed text includes status content, content warnings, and attachment descriptions. -To use it, set the search module to `Pleroma.Search.ParadeDB` and configure the ParadeDB database URL: -These settings must be in static/runtime config (or `PARADEDB_DATABASE_URL`) so the dedicated ParadeDB Repo is started with the application. +### Configuration + +Set the search module to `Pleroma.Search.ParadeDB` and configure the ParadeDB database URL. These settings must be in static/runtime config +(or `PARADEDB_DATABASE_URL`) so the dedicated ParadeDB Repo is started with the application. They cannot be migrated to or changed through +ConfigDB. > config :pleroma, Pleroma.Search, module: Pleroma.Search.ParadeDB > > config :pleroma, Pleroma.Search.ParadeDB, -> url: System.get_env("PARADEDB_DATABASE_URL") || "postgres://postgres:postgres@127.0.0.1:5432/paradedb", +> url: System.get_env("PARADEDB_DATABASE_URL"), > table: "pleroma_search_documents", > fuzzy_distance: 0 > @@ -169,22 +173,127 @@ These settings must be in static/runtime config (or `PARADEDB_DATABASE_URL`) so > pool_size: 2, > prepare: :unnamed -Then, create the table and BM25 index once: +When connecting over an untrusted network, enable certificate verification in the Repo configuration. For example: + +> config :pleroma, Pleroma.Search.ParadeDB.Repo, +> ssl: [ +> verify: :verify_peer, +> cacertfile: "/etc/pleroma/paradedb-ca.pem", +> server_name_indication: ~c"paradedb.example.com" +> ] + +### Initial rollout + +The `pg_search` extension must already be available in the database. Creating the extension normally requires an administrative database +role. The Pleroma runtime role requires `CONNECT` on the database, `USAGE` on the table's schema and the `pdb` schema, and `SELECT`, `INSERT`, `UPDATE`, +and `DELETE` on the generated table. For example, after creating the table as its owner: + +```sql +GRANT CONNECT ON DATABASE paradedb TO pleroma_search; +GRANT USAGE ON SCHEMA public, pdb TO pleroma_search; +GRANT SELECT, INSERT, UPDATE, DELETE ON TABLE pleroma_search_documents TO pleroma_search; +``` + +Use this rollout order to prevent jobs from being handled by the previous backend: + +1. Install `pg_search` 0.24.3 on PostgreSQL 15 or another deliberately tested combination. +2. Stop Pleroma so no search indexing jobs are running. +3. Put the ParadeDB backend in static configuration. +4. Create the table and BM25 index in a one-shot offline process using `PARADEDB_DATABASE_URL` with the table owner/setup credential. +5. Apply the runtime grants shown above, set the service's `PARADEDB_DATABASE_URL` to the restricted runtime credential, and start Pleroma. +6. Start the initial backfill and monitor the `search_indexing` queue. + +Create the table and BM25 index once with: + +=== "OTP" + ```sh + PARADEDB_DATABASE_URL='OWNER_URL' PLEROMA_CTL_RPC_DISABLED=true \ + ./bin/pleroma_ctl search.indexer create_index + ``` === "From Source" ```sh - mix pleroma.search.indexer create_index + PARADEDB_DATABASE_URL='OWNER_URL' mix pleroma.search.indexer create_index ``` -After that, run an initial indexing pass to backfill existing posts: +The task currently prints an error without returning a failing process status. Verify that it prints `Index created` before continuing. + +### Backfill + +Run an initial indexing pass to enqueue existing posts: + +=== "OTP" + ```sh + ./bin/pleroma_ctl search.indexer index + ``` === "From Source" ```sh - mix pleroma.search.indexer index + PARADEDB_DATABASE_URL='RUNTIME_URL' mix pleroma.search.indexer index ``` +The command enqueues up to 100,000 Create activities by default. It uses keyset pagination, so new activities arriving during the backfill +do not shift the remaining pages. The defaults and accepted options are: + +* `--limit`: maximum activities to enqueue in this run; defaults to `100000` and must be positive +* `--step`: activities fetched from Pleroma's database per page; defaults to `1000` and must be positive +* `--chunk`: jobs inserted into Oban per batch; defaults to `100` and must be positive +* `--before`: resume before the checkpoint activity ID printed by an earlier run; the cursor is exclusive + +The checkpoint is the last activity enqueued, not confirmation that its job succeeded. Repeat the command with its last `--before` +checkpoint only when it reports that more activities remain. Inspect failed or discarded Oban jobs as well as waiting jobs before declaring +the backfill complete. + +The same operation can be run from an attached console while Pleroma is online: + +```elixir +Pleroma.Search.Backfill.run(limit: 10_000) +``` + +The returned `next_cursor` can be supplied as `before` when `exhausted` is `false`. The attached API also accepts `limit: :infinity`, but this +can create an unbounded number of Oban rows; bounded runs are safer on large instances. An `on_page` callback may be supplied for progress +reporting. + Note: Like the other external search backends, only public/unlisted Notes are indexed. Set `fuzzy_distance` to `1` or `2` to allow typo-tolerant matching (`2` is the maximum ParadeDB supports). -Indexing runs via the Oban `search_indexing` queue, so ensure that queue is enabled (not paused) in your Oban configuration. +Indexing runs via the Oban `search_indexing` queue. Its stock configuration starts paused so the search health monitor can control it; verify +that the queue is running before backfilling. A manually managed queue can be enabled with `search_indexing: 10` in the Oban `queues` +configuration instead of a `paused: true` entry. +Monitor that queue until the backfill jobs have drained before evaluating search completeness. + +### Rebuilding + +Stop Pleroma, or otherwise ensure no search indexing jobs are running, before rebuilding. While ParadeDB is still the configured backend, +run `drop_index`, then `create_index` with the table owner/setup credential. Reapply runtime grants after recreating the table, restart +Pleroma, and begin a new backfill. The DDL tasks print failures without returning a failing process status, so verify the exact `Index +dropped` and `Index created` success messages. Dropping the table removes only derived search data; Pleroma activities remain in the primary +database. + +### Rolling back + +To retain the ParadeDB table for a later retry, configure `Pleroma.Search.DatabaseSearch` (or the previous external backend) in static config +and restart Pleroma. + +Before switching backends, inspect the `search_indexing` queue. Drain or cancel pending backfill jobs, or explicitly accept that they will run +against the newly configured backend after restart. + +To remove the table during rollback, first stop Pleroma and run `drop_index` with the table owner/setup credential while ParadeDB is still +configured: + +=== "OTP" + ```sh + PARADEDB_DATABASE_URL='OWNER_URL' PLEROMA_CTL_RPC_DISABLED=true \ + ./bin/pleroma_ctl search.indexer drop_index + ``` + +=== "From Source" + ```sh + PARADEDB_DATABASE_URL='OWNER_URL' mix pleroma.search.indexer drop_index + ``` + +Verify that the task prints `Index dropped`. Then change the backend in static config and restart. The ParadeDB environment variable may be +removed after removing the ParadeDB stanza or switching to a backend that does not start its Repo. After switching away from ParadeDB, the +generic `drop_index` task targets the newly configured backend and will not remove the ParadeDB table; remove it directly with the database +owner instead. From ce5c1b4c15ef89b077f62cde6f75379af68ac601 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sun, 26 Jul 2026 12:35:09 +0400 Subject: [PATCH 179/183] fix(activitypub): harden object normalization --- changelog.d/fedidev-federation-compat.add | 1 + lib/pleroma/object/containment.ex | 6 +- .../object_validators/tag_validator.ex | 32 +++- .../web/activity_pub/transmogrifier.ex | 146 ++++++++++++------ test/pleroma/object/containment_test.exs | 5 + .../fedidev_fun_attachments_test.exs | 33 +++- .../transmogrifier/fedidev_fun_emoji_test.exs | 42 ++++- .../fedidev_fun_object_type_list_test.exs | 11 ++ .../fedidev_fun_tag_inference_test.exs | 47 ++++++ .../transmogrifier/note_handling_test.exs | 38 ++++- .../user_update_handling_test.exs | 19 +++ 11 files changed, 315 insertions(+), 65 deletions(-) create mode 100644 changelog.d/fedidev-federation-compat.add diff --git a/changelog.d/fedidev-federation-compat.add b/changelog.d/fedidev-federation-compat.add new file mode 100644 index 000000000..bb29925f9 --- /dev/null +++ b/changelog.d/fedidev-federation-compat.add @@ -0,0 +1 @@ +Accept more real-world ActivityPub object shapes collected by fedidev.fun diff --git a/lib/pleroma/object/containment.ex b/lib/pleroma/object/containment.ex index b90d68636..c4cc1f014 100644 --- a/lib/pleroma/object/containment.ex +++ b/lib/pleroma/object/containment.ex @@ -79,9 +79,11 @@ defmodule Pleroma.Object.Containment do def contain_origin(id, %{"actor" => _actor} = params) do id_uri = URI.parse(id) - actor_uri = URI.parse(get_actor(params)) - compare_uris(actor_uri, id_uri) + case get_actor(params) do + actor when is_binary(actor) -> compare_uris(URI.parse(actor), id_uri) + _ -> :error + end end def contain_origin(id, %{"attributedTo" => actor} = params), diff --git a/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex b/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex index cdab8d78d..0659c93d7 100644 --- a/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex @@ -12,6 +12,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.TagValidator do require Pleroma.Constants @primary_key false + @tag_types ~w[Mention Hashtag Emoji Link] + embedded_schema do # Common field(:type, :string) @@ -49,7 +51,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.TagValidator do |> validate_required([:type, :href]) end - def changeset(struct, %{"type" => "Hashtag", "name" => name} = data) do + def changeset(struct, %{"type" => "Hashtag", "name" => name} = data) when is_binary(name) do name = String.downcase(name) data = Map.put(data, "name", name) @@ -58,7 +60,13 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.TagValidator do |> validate_required([:type, :name]) end - def changeset(struct, %{"type" => "Emoji"} = data) do + def changeset(struct, %{"type" => "Hashtag"} = data) do + struct + |> cast(data, []) + |> Map.put(:action, :ignore) + end + + def changeset(struct, %{"type" => "Emoji", "name" => name} = data) when is_binary(name) do data = data |> Map.put("name", String.trim(data["name"], ":")) @@ -84,6 +92,12 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.TagValidator do end end + def changeset(struct, %{"type" => "Emoji"} = data) do + struct + |> cast(data, []) + |> Map.put(:action, :ignore) + end + def changeset(struct, %{"type" => "Link"} = data) do struct |> cast(data, [:type, :name, :mediaType, :href]) @@ -91,6 +105,10 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.TagValidator do |> validate_required([:type, :href, :mediaType]) end + def changeset(struct, %{"type" => types} = data) when is_list(types) do + changeset(struct, infer_type(data)) + end + def changeset(struct, %{"type" => _} = data) do struct |> cast(data, []) @@ -98,7 +116,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.TagValidator do end def changeset(struct, data) when is_map(data) do - data = infer_type(data) + data = normalize(data) if Map.has_key?(data, "type") do changeset(struct, data) @@ -109,6 +127,14 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.TagValidator do end end + def normalize(data) when is_map(data), do: infer_type(data) + + defp infer_type(%{"type" => types} = data) when is_list(types) do + type = Enum.find(types, &(&1 in @tag_types)) || Enum.find(types, &is_binary/1) + + if type, do: Map.put(data, "type", type), else: Map.delete(data, "type") + end + defp infer_type(%{"type" => _} = data), do: data defp infer_type(%{"name" => "#" <> _} = data), do: Map.put(data, "type", "Hashtag") defp infer_type(%{"name" => "@" <> _} = data), do: Map.put(data, "type", "Mention") diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 61c585daa..33ab62ca3 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -19,6 +19,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do alias Pleroma.Web.ActivityPub.Builder alias Pleroma.Web.ActivityPub.ObjectValidator alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes + alias Pleroma.Web.ActivityPub.ObjectValidators.TagValidator alias Pleroma.Web.ActivityPub.Pipeline alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.ActivityPub.Visibility @@ -28,6 +29,9 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do require Pleroma.Constants + @supported_types Pleroma.Constants.activity_types() ++ + Pleroma.Constants.object_types() ++ Pleroma.Constants.actor_types() + @doc """ Modifies an incoming AP object (mastodon format) to our internal format. """ @@ -301,58 +305,66 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def fix_attachments(%{"attachment" => attachment} = object) when is_list(attachment) do attachments = - Enum.map(attachment, fn data -> - url = - cond do - is_list(data["url"]) -> List.first(data["url"]) - is_map(data["url"]) -> data["url"] - true -> nil - end + Enum.map(attachment, fn + data when is_map(data) -> + url = + cond do + is_list(data["url"]) -> List.first(data["url"]) + is_map(data["url"]) -> data["url"] + true -> nil + end - media_type = - cond do - is_map(url) && is_bitstring(url["mediaType"]) && - url["mediaType"] =~ Pleroma.Constants.mime_regex() -> - url["mediaType"] + url_data = if is_map(url), do: url, else: %{} - is_bitstring(data["mediaType"]) && data["mediaType"] =~ Pleroma.Constants.mime_regex() -> - data["mediaType"] + media_type = + cond do + is_bitstring(url_data["mediaType"]) && + url_data["mediaType"] =~ Pleroma.Constants.mime_regex() -> + url_data["mediaType"] - is_bitstring(data["mimeType"]) && data["mimeType"] =~ Pleroma.Constants.mime_regex() -> - data["mimeType"] + is_bitstring(data["mediaType"]) && + data["mediaType"] =~ Pleroma.Constants.mime_regex() -> + data["mediaType"] - true -> - nil - end + is_bitstring(data["mimeType"]) && data["mimeType"] =~ Pleroma.Constants.mime_regex() -> + data["mimeType"] - href = - cond do - is_map(url) && is_binary(url["href"]) -> url["href"] - is_binary(data["url"]) -> data["url"] - is_binary(data["href"]) -> data["href"] - true -> nil - end + true -> + nil + end + + href = + cond do + is_binary(url) -> url + is_binary(url_data["href"]) -> url_data["href"] + is_binary(data["url"]) -> data["url"] + is_binary(data["href"]) -> data["href"] + true -> nil + end + + if href && valid_http_url?(href) do + attachment_url = + %{ + "href" => href, + "type" => Map.get(url_data, "type", "Link") + } + |> Maps.put_if_present("mediaType", media_type) + |> Maps.put_if_present("width", url_data["width"] || data["width"]) + |> Maps.put_if_present("height", url_data["height"] || data["height"]) - if href && valid_http_url?(href) do - attachment_url = %{ - "href" => href, - "type" => Map.get(url || %{}, "type", "Link") + "url" => [attachment_url], + "type" => data["type"] || "Document" } |> Maps.put_if_present("mediaType", media_type) - |> Maps.put_if_present("width", (url || %{})["width"] || data["width"]) - |> Maps.put_if_present("height", (url || %{})["height"] || data["height"]) + |> Maps.put_if_present("name", data["name"]) + |> Maps.put_if_present("blurhash", data["blurhash"]) + else + nil + end - %{ - "url" => [attachment_url], - "type" => data["type"] || "Document" - } - |> Maps.put_if_present("mediaType", media_type) - |> Maps.put_if_present("name", data["name"]) - |> Maps.put_if_present("blurhash", data["blurhash"]) - else + _ -> nil - end end) |> Enum.filter(& &1) @@ -404,14 +416,23 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do defp emoji_icon_url(_), do: nil + defp valid_emoji_icon_url(icon) do + case emoji_icon_url(icon) do + url when is_binary(url) -> if valid_http_url?(url), do: url + _ -> nil + end + end + def fix_emoji(%{"tag" => tags} = object) when is_list(tags) do emoji = tags - |> Enum.filter(fn data -> is_map(data) and data["type"] == "Emoji" and data["icon"] end) + |> Enum.filter(fn data -> + is_map(data) and data["type"] == "Emoji" and is_binary(data["name"]) and data["icon"] + end) |> Enum.reduce(%{}, fn data, mapping -> name = String.trim(data["name"], ":") - case emoji_icon_url(data["icon"]) do + case valid_emoji_icon_url(data["icon"]) do url when is_binary(url) -> Map.put(mapping, name, url) _ -> mapping end @@ -420,10 +441,11 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do Map.put(object, "emoji", emoji) end - def fix_emoji(%{"tag" => %{"type" => "Emoji"} = tag} = object) do - name = String.trim(tag["name"], ":") + def fix_emoji(%{"tag" => %{"type" => "Emoji", "name" => name} = tag} = object) + when is_binary(name) do + name = String.trim(name, ":") - with url when is_binary(url) <- emoji_icon_url(tag["icon"]) do + with url when is_binary(url) <- valid_emoji_icon_url(tag["icon"]) do Map.put(object, "emoji", %{name => url}) else _ -> object @@ -435,7 +457,9 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def fix_tag(%{"tag" => tag} = object) when is_list(tag) do tags = tag - |> Enum.filter(fn data -> data["type"] == "Hashtag" and data["name"] end) + |> Enum.filter(fn data -> + is_map(data) and data["type"] == "Hashtag" and is_binary(data["name"]) + end) |> Enum.map(fn %{"name" => "#" <> hashtag} -> String.downcase(hashtag) %{"name" => hashtag} -> String.downcase(hashtag) @@ -502,10 +526,32 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do |> handle_incoming_normalized(options) end - defp fix_type_as_string(%{"type" => [type | _]} = data) when is_binary(type), - do: Map.put(data, "type", type) + defp fix_type_as_string(data) do + data + |> normalize_type() + |> normalize_tag_types() + end - defp fix_type_as_string(data), do: data + defp normalize_type(%{"type" => types} = data) when is_list(types) do + type = Enum.find(types, &(&1 in @supported_types)) || Enum.find(types, &is_binary/1) + + if type, do: Map.put(data, "type", type), else: data + end + + defp normalize_type(data), do: data + + defp normalize_tag_types(%{"tag" => tags} = data) when is_list(tags) do + Map.put(data, "tag", Enum.map(tags, &normalize_tag_type/1)) + end + + defp normalize_tag_types(%{"tag" => tag} = data) when is_map(tag) do + Map.put(data, "tag", normalize_tag_type(tag)) + end + + defp normalize_tag_types(data), do: data + + defp normalize_tag_type(tag) when is_map(tag), do: TagValidator.normalize(tag) + defp normalize_tag_type(tag), do: tag # Flag objects are placed ahead of the ID check because Mastodon 2.8 and earlier send them # with nil ID. diff --git a/test/pleroma/object/containment_test.exs b/test/pleroma/object/containment_test.exs index cf906ef47..5560dfce5 100644 --- a/test/pleroma/object/containment_test.exs +++ b/test/pleroma/object/containment_test.exs @@ -27,6 +27,11 @@ defmodule Pleroma.Object.ContainmentTest do }) end + test "rejects an empty actor list without crashing" do + assert :error == + Containment.contain_origin("https://example.com/objects/1", %{"actor" => []}) + end + test "contain_origin_from_id() catches obvious spoofing attempts" do data = %{ "id" => "http://example.com/~alyssa/activities/1234.json" diff --git a/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_attachments_test.exs b/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_attachments_test.exs index a13912b08..9d15974fc 100644 --- a/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_attachments_test.exs +++ b/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_attachments_test.exs @@ -36,8 +36,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.FedidevFunAttachmentsTest do end end - defp ingest_fixture!(rel_path) do - data = rel_path |> File.read!() |> Jason.decode!() + defp ingest_data!(data) do ensure_remote_actor!(data["actor"]) assert {:ok, activity} = Transmogrifier.handle_incoming(data) @@ -46,6 +45,13 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.FedidevFunAttachmentsTest do {activity, object} end + defp ingest_fixture!(rel_path) do + rel_path + |> File.read!() + |> Jason.decode!() + |> ingest_data!() + end + test "drops non-http(s) attachment href" do {_activity, object} = ingest_fixture!("test/fixtures/fedidev.fun/attachment_payto_link.json") assert object.data["attachment"] == [] @@ -66,4 +72,27 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.FedidevFunAttachmentsTest do assert href == "https://fedidev.fun/images/007.png" end + + test "drops malformed attachment list entries" do + data = File.read!("test/fixtures/fedidev.fun/attachment_url_list.json") |> Jason.decode!() + data = update_in(data, ["object", "attachment"], &["malformed" | List.wrap(&1)]) + + {_activity, object} = ingest_data!(data) + + assert length(object.data["attachment"]) == 1 + end + + test "accepts a URI string in an attachment url list" do + data = File.read!("test/fixtures/fedidev.fun/attachment_url_list.json") |> Jason.decode!() + + data = + put_in(data, ["object", "attachment", "url"], [ + "https://fedidev.fun/images/007.png" + ]) + + {_activity, object} = ingest_data!(data) + + assert [%{"url" => [%{"href" => "https://fedidev.fun/images/007.png"}]}] = + object.data["attachment"] + end end diff --git a/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_emoji_test.exs b/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_emoji_test.exs index 56c305352..62e30f5af 100644 --- a/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_emoji_test.exs +++ b/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_emoji_test.exs @@ -36,8 +36,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.FedidevFunEmojiTest do end end - defp ingest_fixture!(rel_path) do - data = rel_path |> File.read!() |> Jason.decode!() + defp ingest_data!(data) do ensure_remote_actor!(data["actor"]) assert {:ok, activity} = Transmogrifier.handle_incoming(data) @@ -46,6 +45,13 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.FedidevFunEmojiTest do {activity, object} end + defp ingest_fixture!(rel_path) do + rel_path + |> File.read!() + |> Jason.decode!() + |> ingest_data!() + end + test "ingests Emoji tag with string icon" do {_activity, object} = ingest_fixture!("test/fixtures/fedidev.fun/emoji_icon_string.json") assert object.data["emoji"]["cow"] == "https://fedidev.fun/static/cow.png" @@ -62,4 +68,36 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.FedidevFunEmojiTest do {_activity, object} = ingest_fixture!("test/fixtures/fedidev.fun/emoji_missing_icon.json") assert object.data["emoji"] == %{} end + + test "ignores Emoji tag missing a string name" do + data = File.read!("test/fixtures/fedidev.fun/emoji_icon_string.json") |> Jason.decode!() + data = put_in(data, ["object", "tag", Access.at(0), "name"], nil) + + {_activity, object} = ingest_data!(data) + + assert object.data["emoji"] == %{} + end + + test "ignores Emoji tag with a non-http icon" do + data = File.read!("test/fixtures/fedidev.fun/emoji_icon_string.json") |> Jason.decode!() + data = put_in(data, ["object", "tag", Access.at(0), "icon"], "file:///tmp/cow.png") + + {_activity, object} = ingest_data!(data) + + assert object.data["emoji"] == %{} + end + + test "normalizes a list-typed Emoji tag before building the emoji map" do + data = File.read!("test/fixtures/fedidev.fun/emoji_icon_string.json") |> Jason.decode!() + + data = + put_in(data, ["object", "tag", Access.at(0), "type"], [ + "https://example.com/ns#CustomTag", + "Emoji" + ]) + + {_activity, object} = ingest_data!(data) + + assert object.data["emoji"]["cow"] == "https://fedidev.fun/static/cow.png" + end end diff --git a/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_object_type_list_test.exs b/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_object_type_list_test.exs index 619e6f477..b169dc181 100644 --- a/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_object_type_list_test.exs +++ b/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_object_type_list_test.exs @@ -50,4 +50,15 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.FedidevFunObjectTypeListTest do {_activity, object} = ingest_fixture!("test/fixtures/fedidev.fun/object_type_list.json") assert object.data["type"] == "Note" end + + test "selects a supported type after an extension type" do + data = File.read!("test/fixtures/fedidev.fun/object_type_list.json") |> Jason.decode!() + data = put_in(data, ["object", "type"], ["https://example.com/ns#CustomType", "Note"]) + ensure_remote_actor!(data["actor"]) + + assert {:ok, activity} = Transmogrifier.handle_incoming(data) + object = Object.normalize(activity.data["object"], fetch: false) + + assert object.data["type"] == "Note" + end end diff --git a/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_tag_inference_test.exs b/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_tag_inference_test.exs index ef7feb8ad..0650dd38c 100644 --- a/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_tag_inference_test.exs +++ b/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_tag_inference_test.exs @@ -62,4 +62,51 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.FedidevFunTagInferenceTest do _ -> false end) end + + test "normalizes a list-typed tag type" do + data = + File.read!("test/fixtures/fedidev.fun/tag_hashtag_missing_type.json") |> Jason.decode!() + + data = put_in(data, ["object", "tag", Access.at(0), "type"], ["Hashtag"]) + ensure_remote_actor!(data["actor"]) + + assert {:ok, activity} = Transmogrifier.handle_incoming(data) + object = Object.normalize(activity.data["object"], fetch: false) + + assert "test" in Object.hashtags(object) + end + + test "ignores a list-typed Hashtag without a string name" do + data = + File.read!("test/fixtures/fedidev.fun/tag_hashtag_missing_type.json") |> Jason.decode!() + + data = put_in(data, ["object", "tag", Access.at(0), "type"], ["Hashtag"]) + data = put_in(data, ["object", "tag", Access.at(0), "name"], nil) + ensure_remote_actor!(data["actor"]) + + assert {:ok, activity} = Transmogrifier.handle_incoming(data) + object = Object.normalize(activity.data["object"], fetch: false) + + assert Object.hashtags(object) == [] + end + + test "infers a Link before deriving quoteUrl" do + data = + File.read!("test/fixtures/fedidev.fun/tag_hashtag_missing_type.json") |> Jason.decode!() + + data = + put_in(data, ["object", "tag"], [ + %{ + "href" => "https://fedidev.fun/assets/note1.jsonap", + "mediaType" => "application/activity+json" + } + ]) + + ensure_remote_actor!(data["actor"]) + + assert {:ok, activity} = Transmogrifier.handle_incoming(data) + object = Object.normalize(activity.data["object"], fetch: false) + + assert object.data["quoteUrl"] == "https://fedidev.fun/assets/note1.jsonap" + end end diff --git a/test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs b/test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs index 4cc199550..793293abd 100644 --- a/test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs +++ b/test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs @@ -671,13 +671,21 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.NoteHandlingTest do test "returns object with emoji when object contains list tags" do assert Transmogrifier.fix_emoji(%{ "tag" => [ - %{"type" => "Emoji", "name" => ":bib:", "icon" => %{"url" => "/test"}}, + %{ + "type" => "Emoji", + "name" => ":bib:", + "icon" => %{"url" => "https://example.com/test"} + }, %{"type" => "Hashtag"} ] }) == %{ - "emoji" => %{"bib" => "/test"}, + "emoji" => %{"bib" => "https://example.com/test"}, "tag" => [ - %{"icon" => %{"url" => "/test"}, "name" => ":bib:", "type" => "Emoji"}, + %{ + "icon" => %{"url" => "https://example.com/test"}, + "name" => ":bib:", + "type" => "Emoji" + }, %{"type" => "Hashtag"} ] } @@ -685,10 +693,28 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.NoteHandlingTest do test "returns object with emoji when object contains map tag" do assert Transmogrifier.fix_emoji(%{ - "tag" => %{"type" => "Emoji", "name" => ":bib:", "icon" => %{"url" => "/test"}} + "tag" => %{ + "type" => "Emoji", + "name" => ":bib:", + "icon" => %{"url" => "https://example.com/test"} + } }) == %{ - "emoji" => %{"bib" => "/test"}, - "tag" => %{"icon" => %{"url" => "/test"}, "name" => ":bib:", "type" => "Emoji"} + "emoji" => %{"bib" => "https://example.com/test"}, + "tag" => %{ + "icon" => %{"url" => "https://example.com/test"}, + "name" => ":bib:", + "type" => "Emoji" + } + } + end + end + + describe "fix_tag/1" do + test "ignores malformed Hashtag names and tag entries" do + tag = %{"type" => "Hashtag", "name" => 123} + + assert Transmogrifier.fix_tag(%{"tag" => [tag, "malformed"]}) == %{ + "tag" => [tag, "malformed"] } end end diff --git a/test/pleroma/web/activity_pub/transmogrifier/user_update_handling_test.exs b/test/pleroma/web/activity_pub/transmogrifier/user_update_handling_test.exs index 851c60850..cf995e143 100644 --- a/test/pleroma/web/activity_pub/transmogrifier/user_update_handling_test.exs +++ b/test/pleroma/web/activity_pub/transmogrifier/user_update_handling_test.exs @@ -50,6 +50,25 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.UserUpdateHandlingTest do assert user.bio == "

Some bio

" end + test "it selects an actor type after an extension type" do + user = insert(:user, local: false) + update_data = File.read!("test/fixtures/mastodon-update.json") |> Jason.decode!() + + object = + update_data["object"] + |> Map.put("actor", user.ap_id) + |> Map.put("id", user.ap_id) + |> Map.put("type", ["https://example.com/ns#CustomActor", "Person"]) + + update_data = + update_data + |> Map.put("actor", user.ap_id) + |> Map.put("object", object) + + assert {:ok, %Activity{local: false}} = Transmogrifier.handle_incoming(update_data) + assert User.get_cached_by_ap_id(user.ap_id).name == "gargle" + end + test "it works with alsoKnownAs" do %{ap_id: actor} = insert(:user, local: false) From 6e73d043ef87c4d0383b0e8d509a4a256f44f3eb Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sun, 26 Jul 2026 12:40:38 +0400 Subject: [PATCH 180/183] test(config): exclude static fixture from test loader --- .../config/{static_search.secret.exs => static_search.config} | 0 test/mix/tasks/pleroma/config_test.exs | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename test/fixtures/config/{static_search.secret.exs => static_search.config} (100%) diff --git a/test/fixtures/config/static_search.secret.exs b/test/fixtures/config/static_search.config similarity index 100% rename from test/fixtures/config/static_search.secret.exs rename to test/fixtures/config/static_search.config diff --git a/test/mix/tasks/pleroma/config_test.exs b/test/mix/tasks/pleroma/config_test.exs index 620bf8662..786d4aac2 100644 --- a/test/mix/tasks/pleroma/config_test.exs +++ b/test/mix/tasks/pleroma/config_test.exs @@ -89,7 +89,7 @@ defmodule Mix.Tasks.Pleroma.ConfigTest do end test "static search settings remain in the config file" do - MixTask.migrate_to_db("test/fixtures/config/static_search.secret.exs") + MixTask.migrate_to_db("test/fixtures/config/static_search.config") refute ConfigDB.get_by_group_and_key(:pleroma, Pleroma.Search) refute ConfigDB.get_by_group_and_key(:pleroma, Pleroma.Search.ParadeDB) From 458a15bf9f7d6eaaf44ea6cbe6c9f2a58b8502d1 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sun, 26 Jul 2026 13:24:03 +0400 Subject: [PATCH 181/183] fix(mastodon-api): harden admin moderation actions --- lib/pleroma/web/activity_pub/activity_pub.ex | 9 ++ .../mastodon_admin/account_operation.ex | 10 +- .../mastodon_admin/report_operation.ex | 43 ++++++- lib/pleroma/web/common_api.ex | 4 +- .../admin/controllers/account_controller.ex | 53 +++++++-- .../admin/controllers/report_controller.ex | 33 +++++- .../mastodon_api/admin/views/report_view.ex | 29 +++-- lib/pleroma/web/router.ex | 15 ++- .../controllers/account_controller_test.exs | 68 ++++++++++- .../controllers/report_controller_test.exs | 111 ++++++++++++++++++ 10 files changed, 344 insertions(+), 31 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 10d3fe063..cdad6a49b 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -1016,6 +1016,14 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp restrict_state(query, _), do: query + defp restrict_report_target(query, %{report_target_id: actor_id}) do + from(activity in query, + where: fragment("?->'object'->>0 = ?", activity.data, ^actor_id) + ) + end + + defp restrict_report_target(query, _), do: query + defp restrict_assigned_account(query, %{assigned_account: assigned_account}) do from(activity in query, where: fragment("?->>'assigned_account' = ?", activity.data, ^assigned_account) @@ -1514,6 +1522,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do |> restrict_actor(opts) |> restrict_type(opts) |> restrict_state(opts) + |> restrict_report_target(opts) |> restrict_assigned_account(opts) |> restrict_favorited_by(opts) |> restrict_blocked(restrict_blocked_opts) diff --git a/lib/pleroma/web/api_spec/operations/mastodon_admin/account_operation.ex b/lib/pleroma/web/api_spec/operations/mastodon_admin/account_operation.ex index ad79d43d7..3b39cbe2a 100644 --- a/lib/pleroma/web/api_spec/operations/mastodon_admin/account_operation.ex +++ b/lib/pleroma/web/api_spec/operations/mastodon_admin/account_operation.ex @@ -216,20 +216,24 @@ defmodule Pleroma.Web.ApiSpec.MastodonAdmin.AccountOperation do properties: %{ type: %Schema{ type: :string, - enum: ["none", "disable", "sensitive", "silence", "suspend"] + enum: ["none", "disable"] }, report_id: %Schema{ type: :string, nullable: true, description: "ID of an associated report that caused this action to be taken" } - } + }, + required: [:type] }, required: true ), responses: %{ 204 => no_content_response(), - 401 => Operation.response("Error", "application/json", ApiError) + 400 => Operation.response("Error", "application/json", ApiError), + 401 => Operation.response("Error", "application/json", ApiError), + 403 => Operation.response("Error", "application/json", ApiError), + 404 => Operation.response("Error", "application/json", ApiError) } } end diff --git a/lib/pleroma/web/api_spec/operations/mastodon_admin/report_operation.ex b/lib/pleroma/web/api_spec/operations/mastodon_admin/report_operation.ex index f59db9966..7e0bcc24b 100644 --- a/lib/pleroma/web/api_spec/operations/mastodon_admin/report_operation.ex +++ b/lib/pleroma/web/api_spec/operations/mastodon_admin/report_operation.ex @@ -34,7 +34,7 @@ defmodule Pleroma.Web.ApiSpec.MastodonAdmin.ReportOperation do :target_account_id, :query, :string, - "Filter by report target account id (not implemented)" + "Filter by report target account id" ) ] ++ pagination_params(), @@ -81,7 +81,8 @@ defmodule Pleroma.Web.ApiSpec.MastodonAdmin.ReportOperation do responses: %{ 200 => Operation.response("Report", "application/json", report()), 400 => Operation.response("Error", "application/json", ApiError), - 401 => Operation.response("Error", "application/json", ApiError) + 401 => Operation.response("Error", "application/json", ApiError), + 404 => Operation.response("Error", "application/json", ApiError) } } end @@ -99,11 +100,43 @@ defmodule Pleroma.Web.ApiSpec.MastodonAdmin.ReportOperation do responses: %{ 200 => Operation.response("Report", "application/json", report()), 400 => Operation.response("Error", "application/json", ApiError), - 401 => Operation.response("Error", "application/json", ApiError) + 401 => Operation.response("Error", "application/json", ApiError), + 404 => Operation.response("Error", "application/json", ApiError) } } end + def assign_to_self_operation do + %Operation{ + tags: ["Report management (Mastodon API)"], + summary: "Assign report to self", + operationId: "MastodonAdmin.ReportController.assign_to_self", + security: [%{"oAuth" => ["admin:write:reports"]}], + parameters: [Operation.parameter(:id, :path, :string, "ID of the report")], + responses: report_update_responses() + } + end + + def unassign_operation do + %Operation{ + tags: ["Report management (Mastodon API)"], + summary: "Unassign report", + operationId: "MastodonAdmin.ReportController.unassign", + security: [%{"oAuth" => ["admin:write:reports"]}], + parameters: [Operation.parameter(:id, :path, :string, "ID of the report")], + responses: report_update_responses() + } + end + + defp report_update_responses do + %{ + 200 => Operation.response("Report", "application/json", report()), + 400 => Operation.response("Error", "application/json", ApiError), + 401 => Operation.response("Error", "application/json", ApiError), + 404 => Operation.response("Error", "application/json", ApiError) + } + end + defp report do %Schema{ title: "Admin::Report", @@ -112,7 +145,7 @@ defmodule Pleroma.Web.ApiSpec.MastodonAdmin.ReportOperation do id: FlakeID, action_taken: %Schema{type: :boolean}, action_taken_at: %Schema{type: :string, format: "date-time", nullable: true}, - category: %Schema{type: :string, enum: ["spam", "violation", "other"]}, + category: %Schema{type: :string, enum: ["spam", "legal", "violation", "other"]}, comment: %Schema{type: :string, nullable: true}, forwarded: %Schema{type: :boolean, nullable: true}, created_at: %Schema{type: :string, format: "date-time"}, @@ -136,7 +169,7 @@ defmodule Pleroma.Web.ApiSpec.MastodonAdmin.ReportOperation do items: %Schema{ type: :object, properties: %{ - id: %Schema{type: :integer}, + id: %Schema{type: :string}, text: %Schema{type: :string} } } diff --git a/lib/pleroma/web/common_api.ex b/lib/pleroma/web/common_api.ex index cb9d521b3..220a8a228 100644 --- a/lib/pleroma/web/common_api.ex +++ b/lib/pleroma/web/common_api.ex @@ -700,7 +700,7 @@ defmodule Pleroma.Web.CommonAPI do end def update_report_state(activity_id, state) do - with %Activity{} = activity <- Activity.get_by_id(activity_id, filter: []) do + with %Activity{} = activity <- Activity.get_report(activity_id) do Utils.update_report_state(activity, state) else nil -> {:error, :not_found} @@ -715,7 +715,7 @@ defmodule Pleroma.Web.CommonAPI do end def assign_report_to_account(activity_id, user) do - with %Activity{} = activity <- Activity.get_by_id(activity_id) do + with %Activity{} = activity <- Activity.get_report(activity_id) do Utils.assign_report_to_account(activity, user) else nil -> {:error, :not_found} diff --git a/lib/pleroma/web/mastodon_api/admin/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/admin/controllers/account_controller.ex index e96566774..b4e038f0e 100644 --- a/lib/pleroma/web/mastodon_api/admin/controllers/account_controller.ex +++ b/lib/pleroma/web/mastodon_api/admin/controllers/account_controller.ex @@ -33,6 +33,8 @@ defmodule Pleroma.Web.MastodonAPI.Admin.AccountController do when action in [:index, :index2, :show] ) + action_fallback(Pleroma.Web.MastodonAPI.FallbackController) + plug( OAuthScopesPlug, %{scopes: ["admin:write:accounts"]} @@ -92,11 +94,18 @@ defmodule Pleroma.Web.MastodonAPI.Admin.AccountController do conn, _params ) do - {:ok, _user} = handle_account_action(user, admin, type) + with :ok <- validate_account_action(user, type), + {:ok, report} <- associated_report(user, body_params), + :ok <- resolve_report(admin, report), + {:ok, _user} <- handle_account_action(user, admin, type) do + json_response(conn, :no_content, "") + else + {:error, :not_found} -> + {:error, :not_found} - resolve_report(admin, body_params) - - json_response(conn, :no_content, "") + {:error, error} -> + json_response(conn, :bad_request, %{error: error}) + end end def delete(%{assigns: %{user: admin, account: user}} = conn, _params) do @@ -161,10 +170,21 @@ defmodule Pleroma.Web.MastodonAPI.Admin.AccountController do User.set_activation(user, false) end - defp handle_account_action(user, _admin, _type) do + defp handle_account_action(%User{}, _admin, "disable") do + {:error, "Only local accounts can be disabled"} + end + + defp handle_account_action(user, _admin, "none") do {:ok, user} end + defp validate_account_action(%User{local: true}, "disable"), do: :ok + + defp validate_account_action(%User{}, "disable"), + do: {:error, "Only local accounts can be disabled"} + + defp validate_account_action(%User{}, "none"), do: :ok + defp build_criteria(params) do %{} |> maybe_filter_local(params) @@ -251,7 +271,23 @@ defmodule Pleroma.Web.MastodonAPI.Admin.AccountController do defp unquote(:"maybe_filter_#{filter_param}")(criteria, _params), do: criteria end - defp resolve_report(admin, %{report_id: id}) do + defp associated_report(user, %{report_id: id}) do + case Activity.get_report(id) do + %Activity{data: %{"object" => [target_ap_id | _]}} = report + when target_ap_id == user.ap_id -> + {:ok, report} + + %Activity{} -> + {:error, "Report does not target this account"} + + nil -> + {:error, :not_found} + end + end + + defp associated_report(_user, _params), do: {:ok, nil} + + defp resolve_report(admin, %Activity{id: id}) do with {:ok, activity} <- CommonAPI.update_report_state(id, "resolved"), report <- Activity.get_by_id_with_user_actor(activity.id) do ModerationLog.insert_log(%{ @@ -260,9 +296,10 @@ defmodule Pleroma.Web.MastodonAPI.Admin.AccountController do subject: activity, subject_actor: report.user_actor }) + + :ok end end - defp resolve_report(_admin, _params) do - end + defp resolve_report(_admin, nil), do: :ok end diff --git a/lib/pleroma/web/mastodon_api/admin/controllers/report_controller.ex b/lib/pleroma/web/mastodon_api/admin/controllers/report_controller.ex index 5dbf19aba..062b90fc6 100644 --- a/lib/pleroma/web/mastodon_api/admin/controllers/report_controller.ex +++ b/lib/pleroma/web/mastodon_api/admin/controllers/report_controller.ex @@ -26,6 +26,13 @@ defmodule Pleroma.Web.MastodonAPI.Admin.ReportController do plug(OAuthScopesPlug, %{scopes: ["admin:write:reports"]} when action in [:resolve, :reopen]) + plug( + OAuthScopesPlug, + %{scopes: ["admin:write:reports"]} when action in [:assign_to_self, :unassign] + ) + + action_fallback(Pleroma.Web.MastodonAPI.FallbackController) + defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.MastodonAdmin.ReportOperation def index(conn, params) do @@ -37,6 +44,7 @@ defmodule Pleroma.Web.MastodonAPI.Admin.ReportController do |> Map.put(:total, true) |> restrict_state(params) |> restrict_actor(params) + |> restrict_target(params) reports = ActivityPub.fetch_activities_query([], opts) @@ -67,6 +75,9 @@ defmodule Pleroma.Web.MastodonAPI.Admin.ReportController do render(conn, "show.json", Report.extract_report_info(report)) else + {:error, :not_found} -> + {:error, :not_found} + {:error, error} -> json_response(conn, :bad_request, %{error: error}) end @@ -84,6 +95,9 @@ defmodule Pleroma.Web.MastodonAPI.Admin.ReportController do render(conn, "show.json", Report.extract_report_info(report)) else + {:error, :not_found} -> + {:error, :not_found} + {:error, error} -> json_response(conn, :bad_request, %{error: error}) end @@ -102,6 +116,9 @@ defmodule Pleroma.Web.MastodonAPI.Admin.ReportController do render(conn, "show.json", Report.extract_report_info(report)) else + {:error, :not_found} -> + {:error, :not_found} + {:error, error} -> json_response(conn, :bad_request, %{error: error}) end @@ -119,14 +136,17 @@ defmodule Pleroma.Web.MastodonAPI.Admin.ReportController do render(conn, "show.json", Report.extract_report_info(report)) else + {:error, :not_found} -> + {:error, :not_found} + {:error, error} -> json_response(conn, :bad_request, %{error: error}) end end - defp restrict_state(opts, %{resolved: true}), do: Map.put(opts, :state, "resolved") + defp restrict_state(opts, %{resolved: true, unresolved: true}), do: opts - defp restrict_state(opts, %{resolved: _}), do: Map.put(opts, :state, "resolved") + defp restrict_state(opts, %{resolved: true}), do: Map.put(opts, :state, "resolved") defp restrict_state(opts, _params), do: Map.put(opts, :state, "open") @@ -139,4 +159,13 @@ defmodule Pleroma.Web.MastodonAPI.Admin.ReportController do end defp restrict_actor(opts, _params), do: opts + + defp restrict_target(opts, %{target_account_id: actor}) do + case User.get_by_id(actor) do + %User{ap_id: ap_id} -> Map.put(opts, :report_target_id, ap_id) + _ -> Map.put(opts, :report_target_id, actor) + end + end + + defp restrict_target(opts, _params), do: opts end diff --git a/lib/pleroma/web/mastodon_api/admin/views/report_view.ex b/lib/pleroma/web/mastodon_api/admin/views/report_view.ex index 664f58b03..088824d03 100644 --- a/lib/pleroma/web/mastodon_api/admin/views/report_view.ex +++ b/lib/pleroma/web/mastodon_api/admin/views/report_view.ex @@ -6,6 +6,7 @@ defmodule Pleroma.Web.MastodonAPI.Admin.ReportView do use Pleroma.Web, :view alias Pleroma.HTML + alias Pleroma.Rule alias Pleroma.Web.AdminAPI.Report alias Pleroma.Web.CommonAPI.Utils alias Pleroma.Web.MastodonAPI.Admin.AccountView @@ -21,9 +22,12 @@ defmodule Pleroma.Web.MastodonAPI.Admin.ReportView do report: report, user: account, account: target_account, - statuses: statuses + statuses: statuses, + assigned_account: assigned_account }) do created_at = Utils.to_masto_date(report.data["published"]) + updated_at = Utils.to_masto_date(report.updated_at) + action_taken = report.data["state"] != "open" content = unless is_nil(report.data["content"]) do @@ -34,23 +38,34 @@ defmodule Pleroma.Web.MastodonAPI.Admin.ReportView do %{ id: report.id, - action_taken: report.data["state"] != "open", - action_taken_at: nil, + action_taken: action_taken, + action_taken_at: if(action_taken, do: updated_at), category: "other", comment: content, - forwarded: nil, + forwarded: false, created_at: created_at, - updated_at: created_at, + updated_at: updated_at, account: AccountView.render("show.json", %{user: account}), target_account: AccountView.render("show.json", %{user: target_account}), - assigned_account: nil, + assigned_account: render_account(assigned_account), action_taken_by_account: nil, statuses: StatusView.render("index.json", %{ activities: statuses, as: :activity }), - rules: [] + rules: rules(report.data["rules"]) } end + + defp render_account(nil), do: nil + defp render_account(account), do: AccountView.render("show.json", %{user: account}) + + defp rules(nil), do: [] + + defp rules(rule_ids) do + rule_ids + |> Rule.get() + |> Enum.map(&%{id: to_string(&1.id), text: &1.text}) + end end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index aa0ec4817..27229ed41 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -137,6 +137,12 @@ defmodule Pleroma.Web.Router do plug(Pleroma.Web.Plugs.EnsurePrivilegedPlug, :users_manage_activation_state) end + pipeline :require_privileged_role_account_actions do + plug(:admin_api) + plug(Pleroma.Web.Plugs.EnsurePrivilegedPlug, :users_manage_activation_state) + plug(Pleroma.Web.Plugs.EnsurePrivilegedPlug, :reports_manage_reports) + end + pipeline :require_privileged_role_users_manage_invites do plug(:admin_api) plug(Pleroma.Web.Plugs.EnsurePrivilegedPlug, :users_manage_invites) @@ -443,10 +449,15 @@ defmodule Pleroma.Web.Router do scope "/api/v1/admin", Pleroma.Web.MastodonAPI.Admin do pipe_through([:require_privileged_role_users_manage_activation_state]) - post("/accounts/:id/action", AccountController, :account_action) post("/accounts/:id/enable", AccountController, :enable) end + scope "/api/v1/admin", Pleroma.Web.MastodonAPI.Admin do + pipe_through([:require_privileged_role_account_actions]) + + post("/accounts/:id/action", AccountController, :account_action) + end + # Mastodon AdminAPI: admins and mods (staff) can perform these actions (if privileged by role) scope "/api/v1/admin", Pleroma.Web.MastodonAPI.Admin do pipe_through([:require_privileged_role_users_manage_invites]) @@ -462,6 +473,8 @@ defmodule Pleroma.Web.Router do get("/reports/:id", ReportController, :show) post("/reports/:id/resolve", ReportController, :resolve) post("/reports/:id/reopen", ReportController, :reopen) + post("/reports/:id/assign_to_self", ReportController, :assign_to_self) + post("/reports/:id/unassign", ReportController, :unassign) end # Mastodon AdminAPI: admins and mods (staff) can perform these actions (if privileged by role) diff --git a/test/pleroma/web/mastodon_api/admin/controllers/account_controller_test.exs b/test/pleroma/web/mastodon_api/admin/controllers/account_controller_test.exs index 113b242dd..38f1189cd 100644 --- a/test/pleroma/web/mastodon_api/admin/controllers/account_controller_test.exs +++ b/test/pleroma/web/mastodon_api/admin/controllers/account_controller_test.exs @@ -183,6 +183,20 @@ defmodule Pleroma.Web.MastodonAPI.Admin.AccountControllerTest do assert %{is_active: false} = user end + test "does not disable a remote account", %{conn: conn} do + %{id: id} = user = insert(:user, local: false) + + assert %{"error" => "Only local accounts can be disabled"} = + conn + |> put_req_header("content-type", "application/json") + |> post("/api/v1/admin/accounts/#{id}/action", %{ + "type" => "disable" + }) + |> json_response_and_validate_schema(400) + + assert %{is_active: true} = Repo.reload!(user) + end + test "perform action with assigned report", %{conn: conn} do [reporter, target_user] = insert_pair(:user) @@ -191,11 +205,9 @@ defmodule Pleroma.Web.MastodonAPI.Admin.AccountControllerTest do account_id: target_user.id }) - %{id: id} = insert(:user) - conn |> put_req_header("content-type", "application/json") - |> post("/api/v1/admin/accounts/#{id}/action", %{ + |> post("/api/v1/admin/accounts/#{target_user.id}/action", %{ "type" => "none", "report_id" => report_id }) @@ -205,6 +217,56 @@ defmodule Pleroma.Web.MastodonAPI.Admin.AccountControllerTest do assert %{data: %{"state" => "resolved"}} = report end + + test "requires the report management privilege", %{conn: conn} do + clear_config([:instance, :admin_privileges], [:users_manage_activation_state]) + + %{id: id} = insert(:user) + + assert %{"error" => _} = + conn + |> put_req_header("content-type", "application/json") + |> post("/api/v1/admin/accounts/#{id}/action", %{"type" => "none"}) + |> json_response(403) + end + + test "does not resolve a report for another account", %{conn: conn} do + [reporter, target_user, other_user] = insert_list(3, :user) + + {:ok, report} = + CommonAPI.report(reporter, %{ + account_id: target_user.id + }) + + assert %{"error" => _} = + conn + |> put_req_header("content-type", "application/json") + |> post("/api/v1/admin/accounts/#{other_user.id}/action", %{ + "type" => "none", + "report_id" => report.id + }) + |> json_response_and_validate_schema(400) + + assert %{data: %{"state" => "open"}} = Repo.reload!(report) + end + + test "requires an action type", %{conn: conn} do + %{id: id} = insert(:user) + + conn + |> put_req_header("content-type", "application/json") + |> post("/api/v1/admin/accounts/#{id}/action", %{}) + |> json_response_and_validate_schema(400) + end + + test "rejects unsupported action types", %{conn: conn} do + %{id: id} = insert(:user) + + conn + |> put_req_header("content-type", "application/json") + |> post("/api/v1/admin/accounts/#{id}/action", %{"type" => "sensitive"}) + |> json_response_and_validate_schema(400) + end end describe "POST /api/v1/admin/accounts/:id/enable" do diff --git a/test/pleroma/web/mastodon_api/admin/controllers/report_controller_test.exs b/test/pleroma/web/mastodon_api/admin/controllers/report_controller_test.exs index 702dcf1db..2ea5e9d92 100644 --- a/test/pleroma/web/mastodon_api/admin/controllers/report_controller_test.exs +++ b/test/pleroma/web/mastodon_api/admin/controllers/report_controller_test.exs @@ -8,6 +8,9 @@ defmodule Pleroma.Web.MastodonAPI.Admin.ReportControllerTest do import Pleroma.Factory + alias Pleroma.Activity + alias Pleroma.Repo + alias Pleroma.Rule alias Pleroma.Web.CommonAPI setup_all do @@ -58,6 +61,11 @@ defmodule Pleroma.Web.MastodonAPI.Admin.ReportControllerTest do |> get("/api/v1/admin/reports") |> json_response_and_validate_schema(200) + assert [%{"id" => ^report_id1}] = + conn + |> get("/api/v1/admin/reports?resolved=false") + |> json_response_and_validate_schema(200) + assert [%{"id" => ^report_id2}, %{"id" => ^report_id1}] = conn |> get("/api/v1/admin/reports?resolved=true&unresolved=true") @@ -68,6 +76,25 @@ defmodule Pleroma.Web.MastodonAPI.Admin.ReportControllerTest do |> get("/api/v1/admin/reports?resolved=true") |> json_response_and_validate_schema(200) end + + test "get reports by target account", %{conn: conn} do + [reporter, target_user, other_target] = insert_list(3, :user) + + {:ok, %{id: report_id}} = + CommonAPI.report(reporter, %{ + account_id: target_user.id + }) + + {:ok, _report} = + CommonAPI.report(reporter, %{ + account_id: other_target.id + }) + + assert [%{"id" => ^report_id}] = + conn + |> get("/api/v1/admin/reports?target_account_id=#{target_user.id}") + |> json_response_and_validate_schema(200) + end end describe "GET /api/v1/admin/reports/:id" do @@ -86,6 +113,34 @@ defmodule Pleroma.Web.MastodonAPI.Admin.ReportControllerTest do |> get("/api/v1/admin/reports/#{report_id}") |> json_response_and_validate_schema(200) end + + test "returns report rules", %{conn: conn} do + [reporter, target_user] = insert_pair(:user) + rule = Rule.create(%{text: "Be excellent to each other"}) + + {:ok, %{id: report_id}} = + CommonAPI.report(reporter, %{ + account_id: target_user.id, + rule_ids: [rule.id] + }) + + assert %{ + "rules" => [ + %{"id" => rule_id, "text" => "Be excellent to each other"} + ] + } = + conn + |> get("/api/v1/admin/reports/#{report_id}") + |> json_response_and_validate_schema(200) + + assert rule_id == to_string(rule.id) + end + + test "returns 404 for a missing report", %{conn: conn} do + conn + |> get("/api/v1/admin/reports/0") + |> json_response_and_validate_schema(404) + end end describe "POST /api/v1/admin/reports/:id/resolve" do @@ -104,6 +159,16 @@ defmodule Pleroma.Web.MastodonAPI.Admin.ReportControllerTest do |> post("/api/v1/admin/reports/#{report_id}/resolve") |> json_response_and_validate_schema(200) end + + test "does not mutate a non-report activity", %{conn: conn} do + activity = insert(:note_activity) + + conn + |> post("/api/v1/admin/reports/#{activity.id}/resolve") + |> json_response_and_validate_schema(404) + + refute Repo.reload!(activity).data["state"] + end end describe "POST /api/v1/admin/reports/:id/reopen" do @@ -124,5 +189,51 @@ defmodule Pleroma.Web.MastodonAPI.Admin.ReportControllerTest do |> post("/api/v1/admin/reports/#{report_id}/reopen") |> json_response_and_validate_schema(200) end + + test "does not mutate a non-report activity", %{conn: conn} do + activity = insert(:note_activity) + + conn + |> post("/api/v1/admin/reports/#{activity.id}/reopen") + |> json_response_and_validate_schema(404) + + refute Repo.reload!(activity).data["state"] + end + end + + describe "POST /api/v1/admin/reports/:id/assign_to_self" do + test "assign and unassign a report", %{admin: admin, conn: conn} do + [reporter, target_user] = insert_pair(:user) + + {:ok, report} = + CommonAPI.report(reporter, %{ + account_id: target_user.id + }) + + assert %{"assigned_account" => %{"id" => admin_id}} = + conn + |> post("/api/v1/admin/reports/#{report.id}/assign_to_self") + |> json_response_and_validate_schema(200) + + assert admin_id == admin.id + assert Repo.reload!(report).data["assigned_account"] == admin.id + + assert %{"assigned_account" => nil} = + conn + |> post("/api/v1/admin/reports/#{report.id}/unassign") + |> json_response_and_validate_schema(200) + + refute Repo.reload!(report).data["assigned_account"] + end + + test "does not assign a non-report activity", %{conn: conn} do + %Activity{id: id} = activity = insert(:note_activity) + + conn + |> post("/api/v1/admin/reports/#{id}/assign_to_self") + |> json_response_and_validate_schema(404) + + refute Repo.reload!(activity).data["assigned_account"] + end end end From 55c20c0fe8381529315740f55ed101e31ece263e Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sun, 26 Jul 2026 13:54:28 +0400 Subject: [PATCH 182/183] test(mastodon-api): avoid relationship expiry races --- .../web/mastodon_api/views/account_view_test.exs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/test/pleroma/web/mastodon_api/views/account_view_test.exs b/test/pleroma/web/mastodon_api/views/account_view_test.exs index f34a801c1..552f95994 100644 --- a/test/pleroma/web/mastodon_api/views/account_view_test.exs +++ b/test/pleroma/web/mastodon_api/views/account_view_test.exs @@ -545,11 +545,9 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do test "represent a relationship for the blocking and blocked user with expiry" do user = insert(:user) other_user = insert(:user) - date = DateTime.utc_now() |> DateTime.add(24 * 60 * 60) |> DateTime.truncate(:second) - {:ok, user, other_user} = User.follow(user, other_user) {:ok, _subscription} = User.subscribe(user, other_user) - {:ok, _user_relationship} = User.block(user, other_user, %{duration: 24 * 60 * 60}) + {:ok, user_relationship} = User.block(user, other_user, %{duration: 24 * 60 * 60}) {:ok, _user_relationship} = User.block(other_user, user) expected = @@ -558,7 +556,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do %{ following: false, blocking: true, - block_expires_at: date, + block_expires_at: user_relationship.expires_at, blocked_by: true, id: to_string(other_user.id) } @@ -570,17 +568,19 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do test "represent a relationship for the muting user with expiry" do user = insert(:user) other_user = insert(:user) - date = DateTime.utc_now() |> DateTime.add(24 * 60 * 60) |> DateTime.truncate(:second) - {:ok, _user_relationship} = + {:ok, user_relationships} = User.mute(user, other_user, %{notifications: true, duration: 24 * 60 * 60}) + user_relationship = + Enum.find(user_relationships, &(&1.relationship_type == :mute)) + expected = Map.merge( @blank_response, %{ muting: true, - mute_expires_at: date, + mute_expires_at: user_relationship.expires_at, muting_notifications: true, id: to_string(other_user.id) } From c591ca518c39eb71c41fb7b956d542bb3c4b793f Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sun, 26 Jul 2026 13:55:57 +0400 Subject: [PATCH 183/183] fix(mastodon-api): finish preview card compatibility --- .../web/api_spec/schemas/chat_message.ex | 33 +----- .../web/api_spec/schemas/preview_card.ex | 109 ++++++++++++++++++ lib/pleroma/web/api_spec/schemas/status.ex | 76 +----------- .../web/mastodon_api/views/status_view.ex | 36 ++++-- .../controllers/status_controller_test.exs | 13 +++ .../mastodon_api/views/status_view_test.exs | 22 +++- 6 files changed, 174 insertions(+), 115 deletions(-) create mode 100644 lib/pleroma/web/api_spec/schemas/preview_card.ex diff --git a/lib/pleroma/web/api_spec/schemas/chat_message.ex b/lib/pleroma/web/api_spec/schemas/chat_message.ex index 57f7890e5..4c18063be 100644 --- a/lib/pleroma/web/api_spec/schemas/chat_message.ex +++ b/lib/pleroma/web/api_spec/schemas/chat_message.ex @@ -5,6 +5,7 @@ defmodule Pleroma.Web.ApiSpec.Schemas.ChatMessage do alias OpenApiSpex.Schema alias Pleroma.Web.ApiSpec.Schemas.Emoji + alias Pleroma.Web.ApiSpec.Schemas.PreviewCard require OpenApiSpex @@ -22,36 +23,8 @@ defmodule Pleroma.Web.ApiSpec.Schemas.ChatMessage do emojis: %Schema{type: :array, items: Emoji}, attachment: %Schema{type: :object, nullable: true}, card: %Schema{ - type: :object, - nullable: true, - description: "Preview card for links included within status content", - required: [:url, :title, :description, :type], - properties: %{ - type: %Schema{ - type: :string, - enum: ["link", "photo", "video", "rich"], - description: "The type of the preview card" - }, - provider_name: %Schema{ - type: :string, - nullable: true, - description: "The provider of the original resource" - }, - provider_url: %Schema{ - type: :string, - format: :uri, - description: "A link to the provider of the original resource" - }, - url: %Schema{type: :string, format: :uri, description: "Location of linked resource"}, - image: %Schema{ - type: :string, - nullable: true, - format: :uri, - description: "Preview thumbnail" - }, - title: %Schema{type: :string, description: "Title of linked resource"}, - description: %Schema{type: :string, description: "Description of preview"} - } + allOf: [PreviewCard], + nullable: true }, unread: %Schema{type: :boolean, description: "Whether a message has been marked as read."} }, diff --git a/lib/pleroma/web/api_spec/schemas/preview_card.ex b/lib/pleroma/web/api_spec/schemas/preview_card.ex new file mode 100644 index 000000000..bb4432e18 --- /dev/null +++ b/lib/pleroma/web/api_spec/schemas/preview_card.ex @@ -0,0 +1,109 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ApiSpec.Schemas.PreviewCard do + alias OpenApiSpex.Schema + alias Pleroma.Web.ApiSpec.Schemas.Account + + require OpenApiSpex + + OpenApiSpex.schema(%{ + title: "PreviewCard", + description: "Preview card for a link included within status or chat message content", + type: :object, + required: [ + :url, + :title, + :description, + :language, + :type, + :authors, + :author_name, + :author_url, + :provider_name, + :provider_url, + :html, + :width, + :height, + :image, + :image_description, + :embed_url, + :blurhash, + :published_at + ], + properties: %{ + url: %Schema{type: :string, format: :uri, description: "Location of linked resource"}, + title: %Schema{type: :string, description: "Title of linked resource"}, + description: %Schema{type: :string, description: "Description of preview"}, + language: %Schema{ + type: :string, + nullable: true, + description: "Language of the linked resource" + }, + type: %Schema{ + type: :string, + enum: ["link", "photo", "video", "rich"], + description: "The type of the preview card" + }, + authors: %Schema{ + type: :array, + description: "Fediverse accounts of the authors of the original resource", + items: %Schema{ + type: :object, + required: [:name, :url, :account], + properties: %{ + name: %Schema{type: :string}, + url: %Schema{type: :string}, + account: %Schema{allOf: [Account], nullable: true} + } + } + }, + author_name: %Schema{type: :string, description: "Author of the original resource"}, + author_url: %Schema{ + type: :string, + description: "Link to the author of the original resource" + }, + provider_name: %Schema{ + type: :string, + description: "The provider of the original resource" + }, + provider_url: %Schema{ + type: :string, + description: "A link to the provider of the original resource" + }, + html: %Schema{ + type: :string, + format: :html, + description: "HTML to be used for generating the preview card" + }, + width: %Schema{type: :integer, description: "Width of preview, in pixels"}, + height: %Schema{type: :integer, description: "Height of preview, in pixels"}, + image: %Schema{ + type: :string, + nullable: true, + format: :uri, + description: "Preview thumbnail" + }, + image_description: %Schema{ + type: :string, + description: "Alternate text that describes what is in the thumbnail" + }, + embed_url: %Schema{ + type: :string, + description: "Used for photo embeds, instead of custom `html`" + }, + blurhash: %Schema{ + type: :string, + nullable: true, + description: + "A hash computed by the [BlurHash algorithm](https://github.com/woltapp/blurhash), for generating colorful preview thumbnails when media has not been downloaded yet." + }, + published_at: %Schema{ + type: :string, + nullable: true, + description: "Publication date of the linked resource" + } + } + }) +end diff --git a/lib/pleroma/web/api_spec/schemas/status.ex b/lib/pleroma/web/api_spec/schemas/status.ex index 893988d18..63e9eebf0 100644 --- a/lib/pleroma/web/api_spec/schemas/status.ex +++ b/lib/pleroma/web/api_spec/schemas/status.ex @@ -9,6 +9,7 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Status do alias Pleroma.Web.ApiSpec.Schemas.Emoji alias Pleroma.Web.ApiSpec.Schemas.FlakeID alias Pleroma.Web.ApiSpec.Schemas.Poll + alias Pleroma.Web.ApiSpec.Schemas.PreviewCard alias Pleroma.Web.ApiSpec.Schemas.Tag alias Pleroma.Web.ApiSpec.Schemas.VisibilityScope @@ -31,79 +32,8 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Status do }, bookmarked: %Schema{type: :boolean, description: "Have you bookmarked this status?"}, card: %Schema{ - type: :object, - nullable: true, - description: "Preview card for links included within status content", - # Yeah, it's effectivelly all required in MastoAPI so far - required: [ - :url, - :title, - :description, - :type, - :author_name, - :author_url, - :provider_name, - :provider_url, - :html, - :width, - :height, - :image, - :embed_url, - :blurhash - ], - properties: %{ - url: %Schema{type: :string, format: :uri, description: "Location of linked resource"}, - title: %Schema{type: :string, description: "Title of linked resource"}, - description: %Schema{type: :string, description: "Description of preview"}, - type: %Schema{ - type: :string, - enum: ["link", "photo", "video", "rich"], - description: "The type of the preview card" - }, - author_name: %Schema{type: :string, description: "author of the original resource"}, - author_url: %Schema{ - type: :string, - format: :uri, - description: "link to the author of the original resource" - }, - provider_name: %Schema{ - type: :string, - description: "The provider of the original resource" - }, - provider_url: %Schema{ - type: :string, - format: :uri, - description: "A link to the provider of the original resource" - }, - html: %Schema{ - type: :string, - format: :html, - description: "HTML to be used for generating the preview card" - }, - width: %Schema{type: :integer, description: "Width of preview, in pixels"}, - height: %Schema{type: :integer, description: "Height of preview, in pixels"}, - image: %Schema{ - type: :string, - nullable: true, - format: :uri, - description: "Preview thumbnail" - }, - image_description: %Schema{ - type: :string, - description: "Alternate text that describes what is in the thumbnail" - }, - embed_url: %Schema{ - type: :string, - format: :uri, - description: "Used for photo embeds, instead of custom `html`" - }, - blurhash: %Schema{ - type: :string, - nullable: true, - description: - "A hash computed by the (BlurHash algorithm)[https://github.com/woltapp/blurhash], for generating colorful preview thumbnails when media has not been downloaded yet." - } - } + allOf: [PreviewCard], + nullable: true }, content: %Schema{type: :string, format: :html, description: "HTML-encoded status content"}, text: %Schema{ diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index b78494c28..356e8fce5 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -587,21 +587,25 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do video_url = proxied_url(rich_media["video"], page_url_data) %{ - type: "link", - provider_name: page_url_data.host, - provider_url: page_url_data.scheme <> "://" <> page_url_data.host, + type: card_type(rich_media["type"]), + provider_name: rich_media["provider_name"] || page_url_data.host, + provider_url: + rich_media["provider_url"] || page_url_data.scheme <> "://" <> page_url_data.host, url: page_url, image: image_url, image_description: rich_media["image:alt"] || "", title: rich_media["title"] || "", description: rich_media["description"] || "", - author_name: "", - author_url: "", - html: "", - width: 0, - height: 0, - embed_url: "", + language: nil, + authors: [], + author_name: rich_media["author_name"] || "", + author_url: rich_media["author_url"] || "", + html: rich_media["html"] || "", + width: card_dimension(rich_media["width"]), + height: card_dimension(rich_media["height"]), + embed_url: rich_media["embed_url"] || "", blurhash: nil, + published_at: nil, pleroma: %{ opengraph: rich_media @@ -706,6 +710,20 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do %{content: content, detected_source_language: detected_source_language, provider: provider} end + defp card_type(type) when type in ["link", "photo", "video", "rich"], do: type + defp card_type(_), do: "link" + + defp card_dimension(value) when is_integer(value) and value >= 0, do: value + + defp card_dimension(value) when is_binary(value) do + case Integer.parse(value) do + {dimension, ""} when dimension >= 0 -> dimension + _ -> 0 + end + end + + defp card_dimension(_), do: 0 + def get_reply_to(activity, %{replied_to_activities: replied_to_activities}) do object = Object.normalize(activity, fetch: false) diff --git a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs index c70ec29c4..6ee79689d 100644 --- a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs @@ -2430,6 +2430,9 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do describe "cards" do setup do + clear_config([:rich_media, :enabled], true) + Mox.stub_with(Pleroma.CachexMock, Pleroma.NullCache) + Pleroma.StaticStubbedConfigMock |> stub(:get, fn [:rich_media, :enabled] -> true @@ -2443,6 +2446,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) {:ok, activity} = CommonAPI.post(user, %{status: "https://example.com/ogp"}) + ObanHelpers.perform_all() card_data = %{ "image" => "http://ia.media-imdb.com/images/rock.jpg", @@ -2465,10 +2469,14 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do }, "author_name" => "", "author_url" => "", + "authors" => [], "blurhash" => nil, "embed_url" => "", "height" => 0, "html" => "", + "image_description" => "", + "language" => nil, + "published_at" => nil, "width" => 0 } @@ -2495,6 +2503,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) {:ok, activity} = CommonAPI.post(user, %{status: "https://example.com/ogp-missing-data"}) + ObanHelpers.perform_all() response = conn @@ -2518,10 +2527,14 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do }, "author_name" => "", "author_url" => "", + "authors" => [], "blurhash" => nil, "embed_url" => "", "height" => 0, "html" => "", + "image_description" => "", + "language" => nil, + "published_at" => nil, "width" => 0 } end diff --git a/test/pleroma/web/mastodon_api/views/status_view_test.exs b/test/pleroma/web/mastodon_api/views/status_view_test.exs index faeca050e..73adb196a 100644 --- a/test/pleroma/web/mastodon_api/views/status_view_test.exs +++ b/test/pleroma/web/mastodon_api/views/status_view_test.exs @@ -845,15 +845,31 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do fields = %{ "url" => page_url, - "site_name" => "Example site name", + "type" => "photo", "title" => "Example website", "image" => page_url <> "/example.jpg", - "description" => "Example description" + "description" => "Example description", + "author_name" => "Example author", + "author_url" => page_url <> "/author", + "provider_name" => "Example provider", + "provider_url" => page_url <> "/provider", + "html" => "", + "width" => "1024", + "height" => 768 } {:ok, card} = Card.create(page_url, fields) - assert match?(%{provider_name: "example.com"}, StatusView.render("card.json", card)) + assert %{ + type: "photo", + author_name: "Example author", + author_url: "https://example.com/author", + provider_name: "Example provider", + provider_url: "https://example.com/provider", + html: "", + width: 1024, + height: 768 + } = StatusView.render("card.json", card) end test "a rich media card has all media proxied" do