From c44cf7d74f513397a9b25e7f7e0fe6092d679840 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Wed, 29 May 2024 06:24:24 +0200 Subject: [PATCH 01/90] StatusOperation: Remove obsolete card_operation --- .../api_spec/operations/status_operation.ex | 28 ------------------- 1 file changed, 28 deletions(-) diff --git a/lib/pleroma/web/api_spec/operations/status_operation.ex b/lib/pleroma/web/api_spec/operations/status_operation.ex index 1717c68c8..f7914406a 100644 --- a/lib/pleroma/web/api_spec/operations/status_operation.ex +++ b/lib/pleroma/web/api_spec/operations/status_operation.ex @@ -339,34 +339,6 @@ defmodule Pleroma.Web.ApiSpec.StatusOperation do } end - def card_operation do - %Operation{ - tags: ["Retrieve status information"], - deprecated: true, - summary: "Preview card", - description: "Deprecated in favor of card property inlined on Status entity", - operationId: "StatusController.card", - parameters: [id_param()], - security: [%{"oAuth" => ["read:statuses"]}], - responses: %{ - 200 => - Operation.response("Card", "application/json", %Schema{ - type: :object, - nullable: true, - properties: %{ - type: %Schema{type: :string, enum: ["link", "photo", "video", "rich"]}, - provider_name: %Schema{type: :string, nullable: true}, - provider_url: %Schema{type: :string, format: :uri}, - url: %Schema{type: :string, format: :uri}, - image: %Schema{type: :string, nullable: true, format: :uri}, - title: %Schema{type: :string}, - description: %Schema{type: :string} - } - }) - } - } - end - def favourited_by_operation do %Operation{ tags: ["Retrieve status information"], From d860d5059703bfbcd3550a8ff8361d83ec06d00d Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Wed, 29 May 2024 06:27:11 +0200 Subject: [PATCH 02/90] StatusController: Restore preview cards tests This is a partial revert of 19002fd6c11760898daf0b5ed648d6ba58d84b97 Subsequent commit updating it to cards embedded in statuses required to pass tests. --- .../controllers/status_controller_test.exs | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs index f34911e5b..0386a5b7a 100644 --- a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs @@ -1653,6 +1653,63 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do end end + describe "cards" do + setup do + Pleroma.StaticStubbedConfigMock + |> stub(:get, fn + [:rich_media, :enabled] -> true + path -> Pleroma.Test.StaticConfig.get(path) + end) + + oauth_access(["read:statuses"]) + end + + test "returns rich-media card", %{conn: conn, user: user} do + Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) + + {:ok, activity} = CommonAPI.post(user, %{status: "https://example.com/ogp"}) + + card_data = %{ + "image" => "http://ia.media-imdb.com/images/rock.jpg", + "provider_name" => "example.com", + "provider_url" => "https://example.com", + "title" => "The Rock", + "type" => "link", + "url" => "https://example.com/ogp", + "description" => + "Directed by Michael Bay. With Sean Connery, Nicolas Cage, Ed Harris, John Spencer.", + "pleroma" => %{ + "opengraph" => %{ + "image" => "http://ia.media-imdb.com/images/rock.jpg", + "title" => "The Rock", + "type" => "video.movie", + "url" => "https://example.com/ogp", + "description" => + "Directed by Michael Bay. With Sean Connery, Nicolas Cage, Ed Harris, John Spencer." + } + } + } + + response = + conn + |> get("/api/v1/statuses/#{activity.id}/card") + |> json_response_and_validate_schema(200) + + assert response == card_data + + # works with private posts + {:ok, activity} = + CommonAPI.post(user, %{status: "https://example.com/ogp", visibility: "direct"}) + + response_two = + conn + |> get("/api/v1/statuses/#{activity.id}/card") + |> json_response_and_validate_schema(200) + + assert response_two == card_data + end + end + test "bookmarks" do bookmarks_uri = "/api/v1/bookmarks" From c7119d17d874ad15291decc29ce1147bbdf78488 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Wed, 29 May 2024 06:24:50 +0200 Subject: [PATCH 03/90] StatusControllerTest: Update preview cards tests to current API --- .../mastodon_api/controllers/status_controller_test.exs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs index 0386a5b7a..08589f6ae 100644 --- a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs @@ -1692,10 +1692,10 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do response = conn - |> get("/api/v1/statuses/#{activity.id}/card") + |> get("/api/v1/statuses/#{activity.id}") |> json_response_and_validate_schema(200) - assert response == card_data + assert response["card"] == card_data # works with private posts {:ok, activity} = @@ -1703,10 +1703,10 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do response_two = conn - |> get("/api/v1/statuses/#{activity.id}/card") + |> get("/api/v1/statuses/#{activity.id}") |> json_response_and_validate_schema(200) - assert response_two == card_data + assert response_two["card"] == card_data end end From 656dce96dc3d4ceb2ac2fed1badc0f2baa290c7f Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Thu, 15 Feb 2024 00:57:34 +0100 Subject: [PATCH 04/90] Status: Fill all MastoAPI required values in preview cards --- changelog.d/previewcard-void-fill.fix | 1 + lib/pleroma/web/api_spec/schemas/status.ex | 49 +++++++++++++++++-- .../web/mastodon_api/views/status_view.ex | 7 +++ .../controllers/status_controller_test.exs | 44 ++++++++++++++++- 4 files changed, 95 insertions(+), 6 deletions(-) create mode 100644 changelog.d/previewcard-void-fill.fix diff --git a/changelog.d/previewcard-void-fill.fix b/changelog.d/previewcard-void-fill.fix new file mode 100644 index 000000000..04e571347 --- /dev/null +++ b/changelog.d/previewcard-void-fill.fix @@ -0,0 +1 @@ +Status: Fill all MastoAPI required values in preview cards diff --git a/lib/pleroma/web/api_spec/schemas/status.ex b/lib/pleroma/web/api_spec/schemas/status.ex index 6e537b5da..1432be264 100644 --- a/lib/pleroma/web/api_spec/schemas/status.ex +++ b/lib/pleroma/web/api_spec/schemas/status.ex @@ -34,16 +34,40 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Status do type: :object, nullable: true, description: "Preview card for links included within status content", - required: [:url, :title, :description, :type], + # Yeah, it's effectivelly all required in MastoAPI so far + required: [ + :url, + :title, + :description, + :type, + :author_name, + :author_url, + :provider_name, + :provider_url, + :html, + :width, + :height, + :image, + :embed_url, + :blurhash + ], properties: %{ + url: %Schema{type: :string, format: :uri, description: "Location of linked resource"}, + title: %Schema{type: :string, description: "Title of linked resource"}, + description: %Schema{type: :string, description: "Description of preview"}, type: %Schema{ type: :string, enum: ["link", "photo", "video", "rich"], description: "The type of the preview card" }, + author_name: %Schema{type: :string, description: "author of the original resource"}, + author_url: %Schema{ + type: :string, + format: :uri, + description: "link to the author of the original resource" + }, provider_name: %Schema{ type: :string, - nullable: true, description: "The provider of the original resource" }, provider_url: %Schema{ @@ -51,7 +75,13 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Status do format: :uri, description: "A link to the provider of the original resource" }, - url: %Schema{type: :string, format: :uri, description: "Location of linked resource"}, + html: %Schema{ + type: :string, + format: :html, + description: "HTML to be used for generating the preview card" + }, + width: %Schema{type: :integer, description: "Width of preview, in pixels"}, + height: %Schema{type: :integer, description: "Height of preview, in pixels"}, image: %Schema{ type: :string, nullable: true, @@ -62,8 +92,17 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Status do type: :string, description: "Alternate text that describes what is in the thumbnail" }, - title: %Schema{type: :string, description: "Title of linked resource"}, - description: %Schema{type: :string, description: "Description of preview"} + embed_url: %Schema{ + type: :string, + format: :uri, + description: "Used for photo embeds, instead of custom `html`" + }, + blurhash: %Schema{ + type: :string, + nullable: true, + description: + "A hash computed by the (BlurHash algorithm)[https://github.com/woltapp/blurhash], for generating colorful preview thumbnails when media has not been downloaded yet." + } } }, content: %Schema{type: :string, format: :html, description: "HTML-encoded status content"}, diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 0c16749a4..7a654381d 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -586,6 +586,13 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do image_description: rich_media["image:alt"] || "", title: rich_media["title"] || "", description: rich_media["description"] || "", + author_name: "", + author_url: "", + html: "", + width: 0, + height: 0, + embed_url: "", + blurhash: nil, pleroma: %{ opengraph: rich_media diff --git a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs index 08589f6ae..725356edc 100644 --- a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs @@ -1687,7 +1687,14 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do "description" => "Directed by Michael Bay. With Sean Connery, Nicolas Cage, Ed Harris, John Spencer." } - } + }, + "author_name" => "", + "author_url" => "", + "blurhash" => nil, + "embed_url" => "", + "height" => 0, + "html" => "", + "width" => 0 } response = @@ -1708,6 +1715,41 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do assert response_two["card"] == card_data end + + test "replaces missing description with an empty string", %{conn: conn, user: user} do + Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) + + {:ok, activity} = CommonAPI.post(user, %{status: "https://example.com/ogp-missing-data"}) + + response = + conn + |> get("/api/v1/statuses/#{activity.id}") + |> json_response_and_validate_schema(:ok) + + assert response["card"] == %{ + "type" => "link", + "title" => "Pleroma", + "description" => "", + "image" => nil, + "provider_name" => "example.com", + "provider_url" => "https://example.com", + "url" => "https://example.com/ogp-missing-data", + "pleroma" => %{ + "opengraph" => %{ + "title" => "Pleroma", + "type" => "website", + "url" => "https://example.com/ogp-missing-data" + } + }, + "author_name" => "", + "author_url" => "", + "blurhash" => nil, + "embed_url" => "", + "height" => 0, + "html" => "", + "width" => 0 + } + end end test "bookmarks" do From 9ce615d3d58eebe59dd7279ae5c7ccf9dcc73f82 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Thu, 19 Dec 2024 21:48:48 +0100 Subject: [PATCH 05/90] =?UTF-8?q?rel/=E2=80=A6/init.d/pleroma:=20tab=20ind?= =?UTF-8?q?ent?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rel/files/installation/init.d/pleroma | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rel/files/installation/init.d/pleroma b/rel/files/installation/init.d/pleroma index ca5b842e1..ed8a27830 100755 --- a/rel/files/installation/init.d/pleroma +++ b/rel/files/installation/init.d/pleroma @@ -17,6 +17,6 @@ retry="SIGTERM/30/SIGKILL/5" pidfile="/var/run/pleroma.pid" depend() { - want nginx - need postgresql + want nginx + need postgresql } From a1247220f35b5d42d2c33dfeacb3c0c0ebc8c279 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Thu, 19 Dec 2024 21:49:12 +0100 Subject: [PATCH 06/90] =?UTF-8?q?rel/=E2=80=A6/init.d/pleroma:=20set=20res?= =?UTF-8?q?pawn=5Fmax=20&=20respawn=5Fperiod?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rel/files/installation/init.d/pleroma | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/rel/files/installation/init.d/pleroma b/rel/files/installation/init.d/pleroma index ed8a27830..4e3c20575 100755 --- a/rel/files/installation/init.d/pleroma +++ b/rel/files/installation/init.d/pleroma @@ -16,6 +16,10 @@ retry="SIGTERM/30/SIGKILL/5" pidfile="/var/run/pleroma.pid" +# Needs OpenRC >= 0.42 +respawn_max=200 +respawn_period=86400 # 1*day + depend() { want nginx need postgresql From 9bbeeeb428272873063d0eacd33112be09f4d6bd Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Thu, 19 Dec 2024 21:49:32 +0100 Subject: [PATCH 07/90] */init.d/pleroma: run migrations before starting --- changelog.d/opt-init.d-migrations.change | 1 + installation/init.d/pleroma | 6 ++++++ rel/files/installation/init.d/pleroma | 6 ++++++ 3 files changed, 13 insertions(+) create mode 100644 changelog.d/opt-init.d-migrations.change diff --git a/changelog.d/opt-init.d-migrations.change b/changelog.d/opt-init.d-migrations.change new file mode 100644 index 000000000..c53f2baa0 --- /dev/null +++ b/changelog.d/opt-init.d-migrations.change @@ -0,0 +1 @@ +- OpenRC: Automatically run migrations before starting pleroma diff --git a/installation/init.d/pleroma b/installation/init.d/pleroma index cb6635a0b..2cd366140 100755 --- a/installation/init.d/pleroma +++ b/installation/init.d/pleroma @@ -37,6 +37,12 @@ depend() { need nginx postgresql } +start_pre() { + ebegin "running migrations if any" + su -s /bin/sh -c '/usr/bin/mix ecto.migrate' -l "${command_user}" + eend "$?" +} + healthcheck() { # put pleroma_health=YES in /etc/conf.d/pleroma if you want healthchecking # and make sure you have curl installed diff --git a/rel/files/installation/init.d/pleroma b/rel/files/installation/init.d/pleroma index 4e3c20575..0e117e9b8 100755 --- a/rel/files/installation/init.d/pleroma +++ b/rel/files/installation/init.d/pleroma @@ -24,3 +24,9 @@ depend() { want nginx need postgresql } + +start_pre() { + ebegin "running migrations if any" + su -s /bin/sh -c '/opt/pleroma/bin/pleroma_ctl migrate' -l "${command_user}" + eend "$?" +} From c3c14f5860051d0311ae34900ef0181ada5c630d Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 21 Jan 2025 14:36:50 +0400 Subject: [PATCH 08/90] OAuth App: Fix test, optimize code. --- .../optimize-orphaned-apps-cleanup.change | 1 + lib/pleroma/web/o_auth/app.ex | 14 ++++--- test/pleroma/web/o_auth/app_test.exs | 42 ++++++++++++++++--- 3 files changed, 46 insertions(+), 11 deletions(-) create mode 100644 changelog.d/optimize-orphaned-apps-cleanup.change diff --git a/changelog.d/optimize-orphaned-apps-cleanup.change b/changelog.d/optimize-orphaned-apps-cleanup.change new file mode 100644 index 000000000..e86b75524 --- /dev/null +++ b/changelog.d/optimize-orphaned-apps-cleanup.change @@ -0,0 +1 @@ +- Optimize orphaned OAuth apps cleanup by using a single delete query instead of deleting records one by one. diff --git a/lib/pleroma/web/o_auth/app.ex b/lib/pleroma/web/o_auth/app.ex index 7661c2566..30a9d9134 100644 --- a/lib/pleroma/web/o_auth/app.ex +++ b/lib/pleroma/web/o_auth/app.ex @@ -168,16 +168,20 @@ defmodule Pleroma.Web.OAuth.App do @spec remove_orphans(pos_integer()) :: :ok def remove_orphans(limit \\ 100) do - fifteen_mins_ago = DateTime.add(DateTime.utc_now(), -900, :second) + fifteen_mins_ago = NaiveDateTime.add(NaiveDateTime.utc_now(), -900) - Repo.transaction(fn -> + # First get the IDs of apps to delete + app_ids = from(a in __MODULE__, where: is_nil(a.user_id) and a.inserted_at < ^fifteen_mins_ago, - limit: ^limit + limit: ^limit, + select: a.id ) |> Repo.all() - |> Enum.each(&Repo.delete(&1)) - end) + + # Then delete those specific apps + from(a in __MODULE__, where: a.id in ^app_ids) + |> Repo.delete_all() :ok end diff --git a/test/pleroma/web/o_auth/app_test.exs b/test/pleroma/web/o_auth/app_test.exs index 44219cf90..871b10ac3 100644 --- a/test/pleroma/web/o_auth/app_test.exs +++ b/test/pleroma/web/o_auth/app_test.exs @@ -55,19 +55,49 @@ defmodule Pleroma.Web.OAuth.AppTest do end test "removes orphaned apps" do + # Create an orphaned app (no user_id) attrs = %{client_name: "Mastodon-Local", redirect_uris: "."} {:ok, %App{} = old_app} = App.get_or_make(attrs, ["write"]) - attrs = %{client_name: "PleromaFE", redirect_uris: "."} - {:ok, %App{} = app} = App.get_or_make(attrs, ["write"]) + # Create a non-orphaned app with a user + user = insert(:user) + attrs = %{client_name: "PleromaFE", redirect_uris: ".", user_id: user.id} + {:ok, %App{} = kept_app} = App.get_or_make(attrs, ["write"]) + + # Create an old but non-orphaned app + attrs = %{client_name: "OldButValid", redirect_uris: ".", user_id: user.id} + {:ok, %App{} = old_kept_app} = App.get_or_make(attrs, ["write"]) + + # backdate both old apps so they're within the threshold for being cleaned up + # 1 hour ago + past_time = NaiveDateTime.add(NaiveDateTime.utc_now(), -3600) - # backdate the old app so it's within the threshold for being cleaned up {:ok, _} = - "UPDATE apps SET inserted_at = now() - interval '1 hour' WHERE id = #{old_app.id}" - |> Pleroma.Repo.query() + Repo.query( + "UPDATE apps SET inserted_at = $1 WHERE id IN ($2, $3)", + [past_time, old_app.id, old_kept_app.id] + ) + + # Ensure the updates were applied + updated_app = Repo.get(App, old_app.id) + updated_kept_app = Repo.get(App, old_kept_app.id) + + assert NaiveDateTime.compare( + updated_app.inserted_at, + NaiveDateTime.add(NaiveDateTime.utc_now(), -900) + ) == :lt + + assert NaiveDateTime.compare( + updated_kept_app.inserted_at, + NaiveDateTime.add(NaiveDateTime.utc_now(), -900) + ) == :lt App.remove_orphans() - assert [app] == Pleroma.Repo.all(App) + # Verify the orphaned app was removed + assert is_nil(Repo.get(App, old_app.id)) + # Verify both non-orphaned apps still exist, regardless of age + assert Repo.get(App, kept_app.id) + assert Repo.get(App, old_kept_app.id) end end From 9d7bdab7a2d9808e8c6dc77531f99d653364d61a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?nicole=20miko=C5=82ajczyk?= Date: Fri, 19 Sep 2025 17:07:09 +0200 Subject: [PATCH 09/90] Mastodon-compatible webhooks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: nicole mikołajczyk --- changelog.d/webhooks.add | 1 + config/config.exs | 3 +- config/description.exs | 20 ++ docs/development/API/admin_api.md | 89 ++++++++ lib/pleroma/application.ex | 3 +- lib/pleroma/user.ex | 2 + lib/pleroma/web/activity_pub/activity_pub.ex | 5 +- .../controllers/webhook_controller.ex | 88 ++++++++ .../web/admin_api/views/webhook_view.ex | 33 +++ lib/pleroma/web/api_spec.ex | 3 +- .../operations/admin/webhook_operation.ex | 193 ++++++++++++++++++ .../operations/notification_operation.ex | 1 - .../mastodon_api/admin/views/account_view.ex | 65 ++++++ .../mastodon_api/admin/views/report_view.ex | 54 +++++ lib/pleroma/web/router.ex | 9 + lib/pleroma/webhook.ex | 100 +++++++++ lib/pleroma/webhook/notify.ex | 83 ++++++++ lib/pleroma/workers/webhook_worker.ex | 41 ++++ .../20220624104914_create_webhooks.exs | 20 ++ test/mix/tasks/pleroma/database_test.exs | 3 +- test/pleroma/user_test.exs | 10 + .../web/activity_pub/activity_pub_test.exs | 24 +++ .../controllers/webhook_controller_test.exs | 84 ++++++++ test/pleroma/webhook/notify_test.exs | 75 +++++++ test/pleroma/webhook_test.exs | 56 +++++ 25 files changed, 1059 insertions(+), 6 deletions(-) create mode 100644 changelog.d/webhooks.add create mode 100644 lib/pleroma/web/admin_api/controllers/webhook_controller.ex create mode 100644 lib/pleroma/web/admin_api/views/webhook_view.ex create mode 100644 lib/pleroma/web/api_spec/operations/admin/webhook_operation.ex create mode 100644 lib/pleroma/web/mastodon_api/admin/views/account_view.ex create mode 100644 lib/pleroma/web/mastodon_api/admin/views/report_view.ex create mode 100644 lib/pleroma/webhook.ex create mode 100644 lib/pleroma/webhook/notify.ex create mode 100644 lib/pleroma/workers/webhook_worker.ex create mode 100644 priv/repo/migrations/20220624104914_create_webhooks.exs create mode 100644 test/pleroma/web/admin_api/controllers/webhook_controller_test.exs create mode 100644 test/pleroma/webhook/notify_test.exs create mode 100644 test/pleroma/webhook_test.exs diff --git a/changelog.d/webhooks.add b/changelog.d/webhooks.add new file mode 100644 index 000000000..323428f9f --- /dev/null +++ b/changelog.d/webhooks.add @@ -0,0 +1 @@ +Add support for Mastodon-compatible webhooks \ No newline at end of file diff --git a/config/config.exs b/config/config.exs index 683805fe3..b01f1ffcc 100644 --- a/config/config.exs +++ b/config/config.exs @@ -926,7 +926,8 @@ config :pleroma, Pleroma.User.Backup, timeout: :timer.minutes(30) config :pleroma, ConcurrentLimiter, [ - {Pleroma.Search, [max_running: 30, max_waiting: 50]} + {Pleroma.Search, [max_running: 30, max_waiting: 50]}, + {Pleroma.Webhook.Notify, [max_running: 5, max_waiting: 200]} ] config :pleroma, Pleroma.Web.WebFinger, domain: nil, update_nickname_on_user_fetch: true diff --git a/config/description.exs b/config/description.exs index c61a344e8..764137674 100644 --- a/config/description.exs +++ b/config/description.exs @@ -3479,6 +3479,26 @@ config :pleroma, :config_description, [ suggestion: [5] } ] + }, + %{ + key: Pleroma.Webhook.Notify, + type: :keyword, + description: "Concurrent limits configuration for webhooks.", + suggestions: [max_running: 5, max_waiting: 5], + children: [ + %{ + key: :max_running, + type: :integer, + description: "Max running concurrently jobs.", + suggestion: [5] + }, + %{ + key: :max_waiting, + type: :integer, + description: "Max waiting jobs.", + suggestion: [5] + } + ] } ] }, diff --git a/docs/development/API/admin_api.md b/docs/development/API/admin_api.md index 64c06ca2b..ee82c077d 100644 --- a/docs/development/API/admin_api.md +++ b/docs/development/API/admin_api.md @@ -1806,3 +1806,92 @@ Note that this differs from the Mastodon API variant: Mastodon API only returns ```json {} ``` + +## `GET /api/v1/pleroma/admin/webhooks` + +### List webhooks + +- Method: `GET` +- Response: + +```json +[ + { + "enabled": true, + "id": "2", + "events": ["account.created"], + "url": "https://webhook.example/", + "secret": "eb85d4ccd8510e78f912743949dc354e8146987d", + "updated_at": "2022-10-29T17:44:16.000Z", + "created_at": "2022-10-29T17:44:13.000Z" + } +] +``` + +## `GET /api/v1/pleroma/admin/webhooks/:id` + +### Get an individual webhook + +- Method: `GET` +- Params: + - `id`: **string** Webhook ID +- Response: A webhook + +## `POST /api/v1/pleroma/admin/webhooks` + +### Create a webhook + +- Method: `POST` +- Params: + - `url`: **string** Webhook URL + - *optional* `events`: **[string]** Types of events to trigger on (`account.created`, `report.created`) + - *optional* `enabled`: **boolean** Whether webhook is enabled +- Response: A webhook + +## `PATCH /api/v1/pleroma/admin/webhooks/:id` + +### Update a webhook + +- Method: `PATCH` +- Params: + - `id`: **string** Webhook ID + - *optional* `url`: **string** Webhook URL + - *optional* `events`: **[string]** Types of events to trigger on (`account.created`, `report.created`) + - *optional* `enabled`: **boolean** Whether webhook is enabled +- Response: A webhook + +## `DELETE /api/v1/pleroma/admin/webhooks/:id` + +### Delete a webhook + +- Method: `DELETE` +- Params: + - `id`: **string** Webhook ID +- Response: A webhook + +## `POST /api/v1/pleroma/admin/webhooks/:id/enable` + +### Activate a webhook + +- Method: `POST` +- Params: + - `id`: **string** Webhook ID +- Response: A webhook + +## `POST /api/v1/pleroma/admin/webhooks/:id/disable` + +### Deactivate a webhook + +- Method: `POST` +- Params: + - `id`: **string** Webhook ID +- Response: A webhook + +## `POST /api/v1/pleroma/admin/webhooks/:id/rotate_secret` + +### Rotate webhook signing secret + +- Method: `POST` +- Params: + - `id`: **string** Webhook ID +- Response: A webhook diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 8e1c5de0d..171884639 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -278,7 +278,8 @@ defmodule Pleroma.Application do config = Config.get(ConcurrentLimiter, []) [ - Pleroma.Search + Pleroma.Search, + Pleroma.Webhook.Notify ] |> Enum.each(fn module -> mod_config = Keyword.get(config, module, []) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 468e124b5..f9f5c15f7 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -39,6 +39,7 @@ defmodule Pleroma.User do alias Pleroma.Web.Endpoint alias Pleroma.Web.OAuth alias Pleroma.Web.RelMe + alias Pleroma.Webhook.Notify alias Pleroma.Workers.BackgroundWorker alias Pleroma.Workers.DeleteWorker alias Pleroma.Workers.UserRefreshWorker @@ -980,6 +981,7 @@ defmodule Pleroma.User do @doc "Inserts provided changeset, performs post-registration actions (confirmation email sending etc.)" def register(%Ecto.Changeset{} = changeset) do with {:ok, user} <- Repo.insert(changeset) do + Notify.trigger_webhooks(user, :"account.created") post_register_action(user) end end diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 3e5239a28..deaf249eb 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -30,6 +30,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do import Ecto.Query import Pleroma.Web.ActivityPub.Utils import Pleroma.Web.ActivityPub.Visibility + import Pleroma.Webhook.Notify, only: [trigger_webhooks: 2] require Logger require Pleroma.Constants @@ -415,7 +416,9 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do with flag_data <- make_flag_data(params, additional), {:ok, activity} <- insert(flag_data, local), _ <- notify_and_stream(activity), - :ok <- maybe_federate(activity) do + _ <- trigger_webhooks(activity, :"report.created"), + :ok <- + maybe_federate(activity) do User.all_users_with_privilege(:reports_manage_reports) |> Enum.filter(fn user -> user.ap_id != actor end) |> Enum.filter(fn user -> not is_nil(user.email) end) diff --git a/lib/pleroma/web/admin_api/controllers/webhook_controller.ex b/lib/pleroma/web/admin_api/controllers/webhook_controller.ex new file mode 100644 index 000000000..8a6b0de7a --- /dev/null +++ b/lib/pleroma/web/admin_api/controllers/webhook_controller.ex @@ -0,0 +1,88 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.AdminAPI.WebhookController do + use Pleroma.Web, :controller + + alias Pleroma.Repo + alias Pleroma.Web.Plugs.OAuthScopesPlug + alias Pleroma.Webhook + + plug(Pleroma.Web.ApiSpec.CastAndValidate) + + plug( + OAuthScopesPlug, + %{scopes: ["admin:write"]} + when action in [:update, :create, :delete, :enable, :disable, :rotate_secret] + ) + + plug(OAuthScopesPlug, %{scopes: ["admin:read"]} when action in [:index, :show]) + + action_fallback(Pleroma.Web.AdminAPI.FallbackController) + + defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.Admin.WebhookOperation + + def index(conn, _) do + webhooks = + Webhook + |> Repo.all() + + render(conn, "index.json", webhooks: webhooks) + end + + def show(conn, %{id: id}) do + with %Webhook{} = webhook <- Webhook.get(id) do + render(conn, "show.json", webhook: webhook) + else + nil -> {:error, :not_found} + end + end + + def create(%{body_params: params} = conn, _) do + with webhook <- Webhook.create(params) do + render(conn, "show.json", webhook: webhook) + end + end + + def update(%{body_params: params} = conn, %{id: id}) do + with %Webhook{} = webhook <- Webhook.get(id), + webhook <- Webhook.update(webhook, params) do + render(conn, "show.json", webhook: webhook) + end + end + + def delete(conn, %{id: id}) do + with %Webhook{} = webhook <- Webhook.get(id), + {:ok, webhook} <- Webhook.delete(webhook) do + render(conn, "show.json", webhook: webhook) + end + end + + def enable(conn, %{id: id}) do + with %Webhook{} = webhook <- Webhook.get(id), + {:ok, webhook} <- Webhook.set_enabled(webhook, true) do + render(conn, "show.json", webhook: webhook) + else + nil -> {:error, :not_found} + end + end + + def disable(conn, %{id: id}) do + with %Webhook{} = webhook <- Webhook.get(id), + {:ok, webhook} <- Webhook.set_enabled(webhook, false) do + render(conn, "show.json", webhook: webhook) + else + nil -> {:error, :not_found} + end + end + + def rotate_secret(conn, %{id: id}) do + with %Webhook{} = webhook <- Webhook.get(id), + {:ok, webhook} <- Webhook.rotate_secret(webhook) do + render(conn, "show.json", webhook: webhook) + else + nil -> {:error, :not_found} + end + end +end diff --git a/lib/pleroma/web/admin_api/views/webhook_view.ex b/lib/pleroma/web/admin_api/views/webhook_view.ex new file mode 100644 index 000000000..725183029 --- /dev/null +++ b/lib/pleroma/web/admin_api/views/webhook_view.ex @@ -0,0 +1,33 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.AdminAPI.WebhookView do + use Pleroma.Web, :view + + alias Pleroma.Web.CommonAPI.Utils + + def render("index.json", %{webhooks: webhooks}) do + render_many(webhooks, __MODULE__, "show.json") + end + + def render("show.json", %{webhook: webhook}) do + %{ + id: webhook.id |> to_string(), + url: webhook.url, + events: webhook.events, + secret: webhook.secret, + enabled: webhook.enabled, + created_at: Utils.to_masto_date(webhook.inserted_at), + updated_at: Utils.to_masto_date(webhook.updated_at) + } + end + + def render("event.json", %{type: type, object: object}) do + %{ + type: type, + created_at: Utils.to_masto_date(NaiveDateTime.utc_now()), + object: object + } + end +end diff --git a/lib/pleroma/web/api_spec.ex b/lib/pleroma/web/api_spec.ex index 3e0ac3704..71d9f08f7 100644 --- a/lib/pleroma/web/api_spec.ex +++ b/lib/pleroma/web/api_spec.ex @@ -105,7 +105,8 @@ defmodule Pleroma.Web.ApiSpec do "Report management", "Status administration", "User administration", - "Announcement management" + "Announcement management", + "Webhooks" ] }, %{"name" => "Applications", "tags" => ["Applications", "Push subscriptions"]}, diff --git a/lib/pleroma/web/api_spec/operations/admin/webhook_operation.ex b/lib/pleroma/web/api_spec/operations/admin/webhook_operation.ex new file mode 100644 index 000000000..c83caa555 --- /dev/null +++ b/lib/pleroma/web/api_spec/operations/admin/webhook_operation.ex @@ -0,0 +1,193 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ApiSpec.Admin.WebhookOperation do + alias OpenApiSpex.Operation + alias OpenApiSpex.Schema + + import Pleroma.Web.ApiSpec.Helpers + + def open_api_operation(action) do + operation = String.to_existing_atom("#{action}_operation") + apply(__MODULE__, operation, []) + end + + def index_operation do + %Operation{ + tags: ["Webhooks"], + summary: "Retrieve a list of webhooks", + operationId: "AdminAPI.WebhookController.index", + security: [%{"oAuth" => ["admin:show"]}], + responses: %{ + 200 => + Operation.response("Array of webhooks", "application/json", %Schema{ + type: :array, + items: webhook() + }) + } + } + end + + def show_operation do + %Operation{ + tags: ["Webhooks"], + summary: "Retrieve a webhook", + operationId: "AdminAPI.WebhookController.show", + security: [%{"oAuth" => ["admin:show"]}], + parameters: [id_param()], + responses: %{ + 200 => Operation.response("Webhook", "application/json", webhook()) + } + } + end + + def create_operation do + %Operation{ + tags: ["Webhooks"], + summary: "Create a webhook", + operationId: "AdminAPI.WebhookController.create", + security: [%{"oAuth" => ["admin:write"]}], + requestBody: + request_body( + "Parameters", + %Schema{ + description: "POST body for creating a webhook", + type: :object, + properties: %{ + url: %Schema{type: :string, format: :uri, required: true}, + events: event_type(true), + enabled: %Schema{type: :boolean} + } + } + ), + responses: %{ + 200 => Operation.response("Webhook", "application/json", webhook()) + } + } + end + + def update_operation do + %Operation{ + tags: ["Webhooks"], + summary: "Update a webhook", + operationId: "AdminAPI.WebhookController.update", + security: [%{"oAuth" => ["admin:write"]}], + parameters: [id_param()], + requestBody: + request_body( + "Parameters", + %Schema{ + description: "POST body for updating a webhook", + type: :object, + properties: %{ + url: %Schema{type: :string, format: :uri}, + events: event_type(), + enabled: %Schema{type: :boolean} + } + } + ), + responses: %{ + 200 => Operation.response("Webhook", "application/json", webhook()) + } + } + end + + def delete_operation do + %Operation{ + tags: ["Webhooks"], + summary: "Delete a webhook", + operationId: "AdminAPI.WebhookController.delete", + security: [%{"oAuth" => ["admin:write"]}], + parameters: [id_param()], + responses: %{ + 200 => Operation.response("Webhook", "application/json", webhook()) + } + } + end + + def enable_operation do + %Operation{ + tags: ["Webhooks"], + summary: "Enable a webhook", + operationId: "AdminAPI.WebhookController.enable", + security: [%{"oAuth" => ["admin:write"]}], + parameters: [id_param()], + responses: %{ + 200 => Operation.response("Webhook", "application/json", webhook()) + } + } + end + + def disable_operation do + %Operation{ + tags: ["Webhooks"], + summary: "Disable a webhook", + operationId: "AdminAPI.WebhookController.disable", + security: [%{"oAuth" => ["admin:write"]}], + parameters: [id_param()], + responses: %{ + 200 => Operation.response("Webhook", "application/json", webhook()) + } + } + end + + def rotate_secret_operation do + %Operation{ + tags: ["Webhooks"], + summary: "Rotate webhook signing secret", + operationId: "AdminAPI.WebhookController.rotate_secret", + security: [%{"oAuth" => ["admin:write"]}], + parameters: [id_param()], + responses: %{ + 200 => Operation.response("Webhook", "application/json", webhook()) + } + } + end + + defp webhook do + %Schema{ + title: "Webhook", + description: "Schema for a webhook", + type: :object, + properties: %{ + id: %Schema{type: :string}, + url: %Schema{type: :string, format: :uri}, + events: event_type(), + secret: %Schema{type: :string}, + enabled: %Schema{type: :boolean}, + created_at: %Schema{type: :string, format: :"date-time"}, + updated_at: %Schema{type: :string, format: :"date-time"} + }, + example: %{ + "id" => "1", + "url" => "https://example.com/webhook", + "events" => ["report.created"], + "secret" => "d3d8cf4bc11fd9c41fd34dcc38d282e451c8bd34", + "enabled" => true, + "created_at" => "2022-06-24T16:19:38.523Z", + "updated_at" => "2022-06-24T16:19:38.523Z" + } + } + end + + defp event_type(required \\ nil) do + %Schema{ + type: :array, + items: %Schema{ + title: "Event", + description: "Event type", + type: :string, + enum: ["account.created", "report.created"], + required: required + } + } + end + + defp id_param do + Operation.parameter(:id, :path, :string, "Webhook ID", + example: "123", + required: true + ) + end +end diff --git a/lib/pleroma/web/api_spec/operations/notification_operation.ex b/lib/pleroma/web/api_spec/operations/notification_operation.ex index 94d1f6b82..ded5e725d 100644 --- a/lib/pleroma/web/api_spec/operations/notification_operation.ex +++ b/lib/pleroma/web/api_spec/operations/notification_operation.ex @@ -3,7 +3,6 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ApiSpec.NotificationOperation do - alias OpenApiSpex.Operation alias OpenApiSpex.Operation alias OpenApiSpex.Schema alias Pleroma.Web.ApiSpec.Schemas.Account diff --git a/lib/pleroma/web/mastodon_api/admin/views/account_view.ex b/lib/pleroma/web/mastodon_api/admin/views/account_view.ex new file mode 100644 index 000000000..40fac94ca --- /dev/null +++ b/lib/pleroma/web/mastodon_api/admin/views/account_view.ex @@ -0,0 +1,65 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2024 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.MastodonAPI.Admin.AccountView do + use Pleroma.Web, :view + + alias Pleroma.User + alias Pleroma.Web.CommonAPI.Utils + alias Pleroma.Web.MastodonAPI + + def render("index.json", %{users: users}) do + render_many(users, __MODULE__, "show.json", as: :user) + end + + def render("show.json", %{user: user}) do + account = + MastodonAPI.AccountView.render("show.json", %{user: user, skip_visibility_check: true}) + + %{ + id: user.id, + username: username_from_nickname(user.nickname), + domain: domain_from_nickname(user.nickname), + created_at: Utils.to_masto_date(user.inserted_at), + email: user.email, + ip: nil, + role: role(user), + confirmed: user.is_confirmed, + sensitized: nil, + suspened: nil, + silenced: nil, + disabled: !user.is_active, + approved: user.is_approved, + locale: nil, + invite_request: user.registration_reason, + ips: nil, + account: account + } + end + + defp username_from_nickname(string) when is_binary(string) do + hd(String.split(string, "@")) + end + + defp username_from_nickname(_), do: nil + + defp domain_from_nickname(string) when is_binary(string) do + String.split(string, "@") + |> Enum.at(1, nil) + end + + defp domain_from_nickname(_), do: nil + + defp role(%User{is_admin: true}) do + "admin" + end + + defp role(%User{is_moderator: true}) do + "moderator" + end + + defp role(_user) do + nil + end +end diff --git a/lib/pleroma/web/mastodon_api/admin/views/report_view.ex b/lib/pleroma/web/mastodon_api/admin/views/report_view.ex new file mode 100644 index 000000000..dd8dec8e3 --- /dev/null +++ b/lib/pleroma/web/mastodon_api/admin/views/report_view.ex @@ -0,0 +1,54 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2024 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.MastodonAPI.Admin.ReportView do + use Pleroma.Web, :view + + alias Pleroma.HTML + alias Pleroma.Web.AdminAPI.Report + alias Pleroma.Web.CommonAPI.Utils + alias Pleroma.Web.MastodonAPI.Admin.AccountView + alias Pleroma.Web.MastodonAPI.StatusView + + def render("index.json", %{reports: reports}) do + reports + |> Enum.map(&Report.extract_report_info/1) + |> Enum.map(&render(__MODULE__, "show.json", &1)) + end + + def render("show.json", %{ + report: report, + user: account, + account: target_account, + statuses: statuses + }) do + created_at = Utils.to_masto_date(report.data["published"]) + + content = + unless is_nil(report.data["content"]) do + HTML.filter_tags(report.data["content"]) + else + nil + end + + %{ + id: report.id, + action_taken: report.data["state"] != "open", + category: "other", + comment: content, + created_at: created_at, + updated_at: created_at, + account: AccountView.render("show.json", %{user: account}), + target_account: AccountView.render("show.json", %{user: target_account}), + assigned_account: nil, + action_taken_by_account: nil, + statuses: + StatusView.render("index.json", %{ + activities: statuses, + as: :activity + }), + rules: [] + } + end +end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index cd9cfd3ed..e3c2f3ff7 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -320,6 +320,15 @@ defmodule Pleroma.Web.Router do get("/announcements/:id", AnnouncementController, :show) patch("/announcements/:id", AnnouncementController, :change) delete("/announcements/:id", AnnouncementController, :delete) + + get("/webhooks", WebhookController, :index) + get("/webhooks/:id", WebhookController, :show) + post("/webhooks", WebhookController, :create) + patch("/webhooks/:id", WebhookController, :update) + delete("/webhooks/:id", WebhookController, :delete) + post("/webhooks/:id/enable", WebhookController, :enable) + post("/webhooks/:id/disable", WebhookController, :disable) + post("/webhooks/:id/rotate_secret", WebhookController, :rotate_secret) end # AdminAPI: admins and mods (staff) can perform these actions (if privileged by role) diff --git a/lib/pleroma/webhook.ex b/lib/pleroma/webhook.ex new file mode 100644 index 000000000..6cf47fd68 --- /dev/null +++ b/lib/pleroma/webhook.ex @@ -0,0 +1,100 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Webhook do + use Ecto.Schema + + import Ecto.Changeset + import Ecto.Query + + alias Pleroma.EctoType.ActivityPub.ObjectValidators + alias Pleroma.Repo + + @event_types [:"account.created", :"report.created"] + + schema "webhooks" do + field(:url, ObjectValidators.Uri) + field(:events, {:array, Ecto.Enum}, values: @event_types, default: []) + field(:secret, :string, default: "") + field(:enabled, :boolean, default: true) + + timestamps() + end + + def get(id), do: Repo.get(__MODULE__, id) + + def get_by_type(type) do + __MODULE__ + |> where([w], ^type in w.events) + |> Repo.all() + end + + def changeset(%__MODULE__{} = webhook, params) do + webhook + |> cast(params, [:url, :events, :enabled]) + |> validate_required([:url, :events]) + |> unique_constraint(:url) + |> strip_events() + |> put_secret() + end + + def update_changeset(%__MODULE__{} = webhook, params \\ %{}) do + webhook + |> cast(params, [:url, :events, :enabled]) + |> unique_constraint(:url) + |> strip_events() + end + + def create(params) do + {:ok, webhook} = + %__MODULE__{} + |> changeset(params) + |> Repo.insert() + + webhook + end + + def update(%__MODULE__{} = webhook, params) do + {:ok, webhook} = + webhook + |> update_changeset(params) + |> Repo.update() + + webhook + end + + def delete(webhook), do: webhook |> Repo.delete() + + def rotate_secret(%__MODULE__{} = webhook) do + webhook + |> cast(%{}, []) + |> put_secret() + |> Repo.update() + end + + def set_enabled(%__MODULE__{} = webhook, enabled) do + webhook + |> cast(%{enabled: enabled}, [:enabled]) + |> Repo.update() + end + + defp strip_events(params) do + if Map.has_key?(params, :events) do + params + |> Map.put(:events, Enum.filter(params[:events], &Enum.member?(@event_types, &1))) + else + params + end + end + + defp put_secret(changeset) do + changeset + |> put_change(:secret, generate_secret()) + end + + defp generate_secret do + Base.encode16(:crypto.strong_rand_bytes(20)) + |> String.downcase() + end +end diff --git a/lib/pleroma/webhook/notify.ex b/lib/pleroma/webhook/notify.ex new file mode 100644 index 000000000..7dbfc52a0 --- /dev/null +++ b/lib/pleroma/webhook/notify.ex @@ -0,0 +1,83 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Webhook.Notify do + alias Phoenix.View + alias Pleroma.Activity + alias Pleroma.User + alias Pleroma.Web.AdminAPI.Report + alias Pleroma.Webhook + alias Pleroma.Workers.WebhookWorker + + def trigger_webhooks(%Activity{} = activity, :"report.created" = type) do + webhooks = Webhook.get_by_type(type) + + webhooks + |> Enum.map(fn webhook -> + WebhookWorker.new(%{ + "op" => "notify", + "type" => type, + "webhook_id" => webhook.id, + "activity_id" => activity.id + }) + end) + |> Oban.insert_all() + end + + def trigger_webhooks(%User{} = user, :"account.created" = type) do + webhooks = Webhook.get_by_type(type) + + webhooks + |> Enum.map(fn webhook -> + WebhookWorker.new(%{ + "op" => "notify", + "type" => type, + "webhook_id" => webhook.id, + "user_id" => user.id + }) + end) + |> Oban.insert_all() + end + + def report_created(%Webhook{} = webhook, %Activity{} = report) do + object = + View.render( + Pleroma.Web.MastodonAPI.Admin.ReportView, + "show.json", + Report.extract_report_info(report) + ) + + deliver(webhook, object, :"report.created") + end + + def account_created(%Webhook{} = webhook, %User{} = user) do + object = + View.render( + Pleroma.Web.MastodonAPI.Admin.AccountView, + "show.json", + user: user + ) + + deliver(webhook, object, :"account.created") + end + + defp deliver(%Webhook{url: url, secret: secret}, object, type) do + body = + View.render_to_string(Pleroma.Web.AdminAPI.WebhookView, "event.json", + type: type, + object: object + ) + + headers = [ + {"Content-Type", "application/json"}, + {"X-Hub-Signature", "sha256=#{signature(body, secret)}"} + ] + + Pleroma.HTTP.post(url, body, headers) + end + + defp signature(body, secret) do + :crypto.mac(:hmac, :sha256, secret, body) |> Base.encode16() + end +end diff --git a/lib/pleroma/workers/webhook_worker.ex b/lib/pleroma/workers/webhook_worker.ex new file mode 100644 index 000000000..b4329b775 --- /dev/null +++ b/lib/pleroma/workers/webhook_worker.ex @@ -0,0 +1,41 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Workers.WebhookWorker do + alias Pleroma.Activity + + use Oban.Worker, queue: :background + + @impl true + def perform(%Job{ + args: %{ + "op" => "notify", + "type" => "report.created", + "webhook_id" => webhook_id, + "activity_id" => report_id + } + }) do + webhook = Pleroma.Webhook.get(webhook_id) + report = Activity.get_by_id(report_id) + + Pleroma.Webhook.Notify.report_created(webhook, report) + end + + def perform(%Job{ + args: %{ + "op" => "notify", + "type" => "account.created", + "webhook_id" => webhook_id, + "user_id" => user_id + } + }) do + webhook = Pleroma.Webhook.get(webhook_id) + user = Pleroma.User.get_by_id(user_id) + + Pleroma.Webhook.Notify.account_created(webhook, user) + end + + @impl true + def timeout(_job), do: :timer.seconds(10) +end diff --git a/priv/repo/migrations/20220624104914_create_webhooks.exs b/priv/repo/migrations/20220624104914_create_webhooks.exs new file mode 100644 index 000000000..c7836fc0c --- /dev/null +++ b/priv/repo/migrations/20220624104914_create_webhooks.exs @@ -0,0 +1,20 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Repo.Migrations.CreateWebhooks do + use Ecto.Migration + + def change do + create_if_not_exists table(:webhooks) do + add(:url, :string, null: false) + add(:events, {:array, :string}, null: false, default: []) + add(:secret, :string, null: false, default: "") + add(:enabled, :boolean, null: false, default: true) + + timestamps() + end + + create_if_not_exists(unique_index(:webhooks, [:url])) + end +end diff --git a/test/mix/tasks/pleroma/database_test.exs b/test/mix/tasks/pleroma/database_test.exs index 38ed096ae..c7dba7771 100644 --- a/test/mix/tasks/pleroma/database_test.exs +++ b/test/mix/tasks/pleroma/database_test.exs @@ -416,7 +416,8 @@ defmodule Mix.Tasks.Pleroma.DatabaseTest do ["user_invite_tokens"], ["user_notes"], ["user_relationships"], - ["users"] + ["users"], + ["webhooks"] ] end diff --git a/test/pleroma/user_test.exs b/test/pleroma/user_test.exs index 0b4dc9197..04ec3a32e 100644 --- a/test/pleroma/user_test.exs +++ b/test/pleroma/user_test.exs @@ -11,12 +11,14 @@ defmodule Pleroma.UserTest do alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.CommonAPI + alias Pleroma.Webhook.Notify use Pleroma.DataCase, async: false use Oban.Testing, repo: Pleroma.Repo import Pleroma.Factory import ExUnit.CaptureLog + import Mock import Swoosh.TestAssertions setup do @@ -719,6 +721,14 @@ defmodule Pleroma.UserTest do assert user.is_confirmed end + + test_with_mock "triggers webhooks", Notify, trigger_webhooks: fn _, _ -> nil end do + cng = User.register_changeset(%User{}, @full_user_data) + + {:ok, registered_user} = User.register(cng) + + assert_called(Notify.trigger_webhooks(registered_user, :"account.created")) + end end describe "user registration, with :account_activation_required" do diff --git a/test/pleroma/web/activity_pub/activity_pub_test.exs b/test/pleroma/web/activity_pub/activity_pub_test.exs index 7fc4f47cd..771af7639 100644 --- a/test/pleroma/web/activity_pub/activity_pub_test.exs +++ b/test/pleroma/web/activity_pub/activity_pub_test.exs @@ -17,6 +17,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.AdminAPI.AccountView alias Pleroma.Web.CommonAPI + alias Pleroma.Webhook.Notify import ExUnit.CaptureLog import Mock @@ -1715,6 +1716,29 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do assert Repo.aggregate(Object, :count, :id) == 1 assert Repo.aggregate(Notification, :count, :id) == 0 end + + test_with_mock "triggers webhooks", + %{ + reporter: reporter, + context: context, + target_account: target_account, + reported_activity: reported_activity, + content: content + }, + Notify, + [:passthrough], + trigger_webhooks: fn _, _ -> nil end do + {:ok, activity} = + ActivityPub.flag(%{ + actor: reporter, + context: context, + account: target_account, + statuses: [reported_activity], + content: content + }) + + assert_called(Notify.trigger_webhooks(activity, :"report.created")) + end end test "fetch_activities/2 returns activities addressed to a list " do diff --git a/test/pleroma/web/admin_api/controllers/webhook_controller_test.exs b/test/pleroma/web/admin_api/controllers/webhook_controller_test.exs new file mode 100644 index 000000000..6a1586ff1 --- /dev/null +++ b/test/pleroma/web/admin_api/controllers/webhook_controller_test.exs @@ -0,0 +1,84 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.AdminAPI.WebhookControllerTest do + use Pleroma.Web.ConnCase, async: true + + import Pleroma.Factory + + alias Pleroma.Webhook + + setup do + admin = insert(:user, is_admin: true) + token = insert(:oauth_admin_token, user: admin) + + conn = + build_conn() + |> assign(:user, admin) + |> assign(:token, token) + + {:ok, %{admin: admin, token: token, conn: conn}} + end + + describe "GET /api/pleroma/admin/webhook" do + test "lists existing webhooks", %{conn: conn} do + Webhook.create(%{url: "https://example.com/webhook1", events: [:"report.created"]}) + Webhook.create(%{url: "https://example.com/webhook2", events: [:"account.created"]}) + + response = + conn + |> get("/api/pleroma/admin/webhooks") + |> json_response_and_validate_schema(:ok) + + assert length(response) == 2 + end + end + + describe "POST /api/pleroma/admin/webhooks" do + test "creates a webhook", %{conn: conn} do + %{"id" => id} = + conn + |> put_req_header("content-type", "application/json") + |> post("/api/pleroma/admin/webhooks", %{ + url: "http://example.com/webhook", + events: ["account.created"] + }) + |> json_response_and_validate_schema(:ok) + + assert %{url: "http://example.com/webhook", events: [:"account.created"]} = Webhook.get(id) + end + end + + describe "PATCH /api/pleroma/admin/webhooks" do + test "edits a webhook", %{conn: conn} do + %{id: id} = + Webhook.create(%{url: "https://example.com/webhook1", events: [:"report.created"]}) + + conn + |> put_req_header("content-type", "application/json") + |> patch("/api/pleroma/admin/webhooks/#{id}", %{ + events: ["report.created", "account.created"] + }) + |> json_response_and_validate_schema(:ok) + + assert %{events: [:"report.created", :"account.created"]} = Webhook.get(id) + end + end + + describe "DELETE /api/pleroma/admin/webhooks" do + test "deletes a webhook", %{conn: conn} do + %{id: id} = + Webhook.create(%{url: "https://example.com/webhook1", events: [:"report.created"]}) + + conn + |> put_req_header("content-type", "application/json") + |> delete("/api/pleroma/admin/webhooks/#{id}") + |> json_response_and_validate_schema(:ok) + + assert [] = + Webhook + |> Pleroma.Repo.all() + end + end +end diff --git a/test/pleroma/webhook/notify_test.exs b/test/pleroma/webhook/notify_test.exs new file mode 100644 index 000000000..e03890aab --- /dev/null +++ b/test/pleroma/webhook/notify_test.exs @@ -0,0 +1,75 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Webhook.NotifyTest do + use Pleroma.DataCase, async: true + + alias Pleroma.Webhook + alias Pleroma.Webhook.Notify + + import Pleroma.Factory + + test "triggers a webhook for a report" do + %{id: activity_id} = activity = insert(:report_activity) + + url = "https://example.com/webhook" + + Webhook.create(%{url: url, events: [:"report.created"]}) + + Tesla.Mock.mock(fn %{url: ^url, body: body} = _ -> + report = + body + |> Jason.decode!() + |> Map.get("object") + + assert %{"id" => ^activity_id} = report + + %Tesla.Env{status: 200, body: ""} + end) + + [job] = Notify.trigger_webhooks(activity, :"report.created") + + Pleroma.Workers.WebhookWorker.perform(job) + end + + test "triggers a webhook for an account" do + %{id: account_id} = user = insert(:user) + + url = "https://example.com/webhook" + + Webhook.create(%{url: url, events: [:"account.created"]}) + + Tesla.Mock.mock(fn %{url: ^url, body: body} = _ -> + report = + body + |> Jason.decode!() + |> Map.get("object") + + assert %{"id" => ^account_id} = report + + %Tesla.Env{status: 200, body: ""} + end) + + [job] = Notify.trigger_webhooks(user, :"account.created") + + Pleroma.Workers.WebhookWorker.perform(job) + end + + test "notifies have a valid signature" do + activity = insert(:report_activity) + + %{secret: secret} = + webhook = Webhook.create(%{url: "https://example.com/webhook", events: [:"report.created"]}) + + Tesla.Mock.mock(fn %{url: "https://example.com/webhook", body: body, headers: headers} = _ -> + {"X-Hub-Signature", "sha256=" <> signature} = + Enum.find(headers, fn {key, _} -> key == "X-Hub-Signature" end) + + assert signature == :crypto.mac(:hmac, :sha256, secret, body) |> Base.encode16() + %Tesla.Env{status: 200, body: ""} + end) + + Notify.report_created(webhook, activity) + end +end diff --git a/test/pleroma/webhook_test.exs b/test/pleroma/webhook_test.exs new file mode 100644 index 000000000..ea1378730 --- /dev/null +++ b/test/pleroma/webhook_test.exs @@ -0,0 +1,56 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.WebhookTest do + use Pleroma.DataCase, async: true + + alias Pleroma.Webhook + + test "creating a webhook" do + %{id: id} = Webhook.create(%{url: "https://example.com/webhook", events: [:"report.created"]}) + + assert %{url: "https://example.com/webhook"} = Webhook.get(id) + end + + test "editing a webhook" do + %{id: id} = + webhook = Webhook.create(%{url: "https://example.com/webhook", events: [:"report.created"]}) + + Webhook.update(webhook, %{events: [:"account.created"]}) + + assert %{events: [:"account.created"]} = Webhook.get(id) + end + + test "filter webhooks by type" do + %{id: id1} = + Webhook.create(%{url: "https://example.com/webhook1", events: [:"report.created"]}) + + %{id: id2} = + Webhook.create(%{ + url: "https://example.com/webhook2", + events: [:"account.created", :"report.created"] + }) + + Webhook.create(%{url: "https://example.com/webhook3", events: [:"account.created"]}) + + assert [%{id: ^id1}, %{id: ^id2}] = Webhook.get_by_type(:"report.created") + end + + test "change webhook state" do + %{id: id, enabled: true} = + webhook = Webhook.create(%{url: "https://example.com/webhook", events: [:"report.created"]}) + + Webhook.set_enabled(webhook, false) + assert %{enabled: false} = Webhook.get(id) + end + + test "rotate webhook secrets" do + %{id: id, secret: secret} = + webhook = Webhook.create(%{url: "https://example.com/webhook", events: [:"report.created"]}) + + Webhook.rotate_secret(webhook) + %{secret: new_secret} = Webhook.get(id) + assert secret != new_secret + end +end From 6d46aefb7f780ce0410a459d756f846ec05fef56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Sat, 29 Oct 2022 19:07:54 +0200 Subject: [PATCH 10/90] Add internal (not managable from API) webhooks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: nicole mikołajczyk --- .../controllers/fallback_controller.ex | 6 ++++++ .../controllers/webhook_controller.ex | 17 ++++++++++++----- .../web/admin_api/views/webhook_view.ex | 1 + .../operations/admin/webhook_operation.ex | 18 +++++++++++++----- lib/pleroma/webhook.ex | 5 +++-- ...20221029171353_add_internal_to_webhooks.exs | 13 +++++++++++++ .../controllers/webhook_controller_test.exs | 14 ++++++++++++++ 7 files changed, 62 insertions(+), 12 deletions(-) create mode 100644 priv/repo/migrations/20221029171353_add_internal_to_webhooks.exs diff --git a/lib/pleroma/web/admin_api/controllers/fallback_controller.ex b/lib/pleroma/web/admin_api/controllers/fallback_controller.ex index e72f45c21..d8fb54cad 100644 --- a/lib/pleroma/web/admin_api/controllers/fallback_controller.ex +++ b/lib/pleroma/web/admin_api/controllers/fallback_controller.ex @@ -11,6 +11,12 @@ defmodule Pleroma.Web.AdminAPI.FallbackController do |> json(%{error: dgettext("errors", "Not found")}) end + def call(conn, {:error, :forbidden}) do + conn + |> put_status(:forbidden) + |> json(%{error: dgettext("errors", "Forbidden")}) + end + def call(conn, {:error, reason}) do conn |> put_status(:bad_request) diff --git a/lib/pleroma/web/admin_api/controllers/webhook_controller.ex b/lib/pleroma/web/admin_api/controllers/webhook_controller.ex index 8a6b0de7a..dd9712a31 100644 --- a/lib/pleroma/web/admin_api/controllers/webhook_controller.ex +++ b/lib/pleroma/web/admin_api/controllers/webhook_controller.ex @@ -46,42 +46,49 @@ defmodule Pleroma.Web.AdminAPI.WebhookController do end def update(%{body_params: params} = conn, %{id: id}) do - with %Webhook{} = webhook <- Webhook.get(id), + with %Webhook{internal: false} = webhook <- Webhook.get(id), webhook <- Webhook.update(webhook, params) do render(conn, "show.json", webhook: webhook) + else + %Webhook{internal: true} -> {:error, :forbidden} end end def delete(conn, %{id: id}) do - with %Webhook{} = webhook <- Webhook.get(id), + with %Webhook{internal: false} = webhook <- Webhook.get(id), {:ok, webhook} <- Webhook.delete(webhook) do render(conn, "show.json", webhook: webhook) + else + %Webhook{internal: true} -> {:error, :forbidden} end end def enable(conn, %{id: id}) do - with %Webhook{} = webhook <- Webhook.get(id), + with %Webhook{internal: false} = webhook <- Webhook.get(id), {:ok, webhook} <- Webhook.set_enabled(webhook, true) do render(conn, "show.json", webhook: webhook) else + %Webhook{internal: true} -> {:error, :forbidden} nil -> {:error, :not_found} end end def disable(conn, %{id: id}) do - with %Webhook{} = webhook <- Webhook.get(id), + with %Webhook{internal: false} = webhook <- Webhook.get(id), {:ok, webhook} <- Webhook.set_enabled(webhook, false) do render(conn, "show.json", webhook: webhook) else + %Webhook{internal: true} -> {:error, :forbidden} nil -> {:error, :not_found} end end def rotate_secret(conn, %{id: id}) do - with %Webhook{} = webhook <- Webhook.get(id), + with %Webhook{internal: false} = webhook <- Webhook.get(id), {:ok, webhook} <- Webhook.rotate_secret(webhook) do render(conn, "show.json", webhook: webhook) else + %Webhook{internal: true} -> {:error, :forbidden} nil -> {:error, :not_found} end end diff --git a/lib/pleroma/web/admin_api/views/webhook_view.ex b/lib/pleroma/web/admin_api/views/webhook_view.ex index 725183029..552aba4f1 100644 --- a/lib/pleroma/web/admin_api/views/webhook_view.ex +++ b/lib/pleroma/web/admin_api/views/webhook_view.ex @@ -18,6 +18,7 @@ defmodule Pleroma.Web.AdminAPI.WebhookView do events: webhook.events, secret: webhook.secret, enabled: webhook.enabled, + internal: webhook.internal, created_at: Utils.to_masto_date(webhook.inserted_at), updated_at: Utils.to_masto_date(webhook.updated_at) } diff --git a/lib/pleroma/web/api_spec/operations/admin/webhook_operation.ex b/lib/pleroma/web/api_spec/operations/admin/webhook_operation.ex index c83caa555..fd5f3a561 100644 --- a/lib/pleroma/web/api_spec/operations/admin/webhook_operation.ex +++ b/lib/pleroma/web/api_spec/operations/admin/webhook_operation.ex @@ -5,6 +5,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.WebhookOperation do alias OpenApiSpex.Operation alias OpenApiSpex.Schema + alias Pleroma.Web.ApiSpec.Schemas.ApiError import Pleroma.Web.ApiSpec.Helpers @@ -88,7 +89,8 @@ defmodule Pleroma.Web.ApiSpec.Admin.WebhookOperation do } ), responses: %{ - 200 => Operation.response("Webhook", "application/json", webhook()) + 200 => Operation.response("Webhook", "application/json", webhook()), + 403 => Operation.response("Forbidden", "application/json", ApiError) } } end @@ -101,7 +103,8 @@ defmodule Pleroma.Web.ApiSpec.Admin.WebhookOperation do security: [%{"oAuth" => ["admin:write"]}], parameters: [id_param()], responses: %{ - 200 => Operation.response("Webhook", "application/json", webhook()) + 200 => Operation.response("Webhook", "application/json", webhook()), + 403 => Operation.response("Forbidden", "application/json", ApiError) } } end @@ -114,7 +117,8 @@ defmodule Pleroma.Web.ApiSpec.Admin.WebhookOperation do security: [%{"oAuth" => ["admin:write"]}], parameters: [id_param()], responses: %{ - 200 => Operation.response("Webhook", "application/json", webhook()) + 200 => Operation.response("Webhook", "application/json", webhook()), + 403 => Operation.response("Forbidden", "application/json", ApiError) } } end @@ -127,7 +131,8 @@ defmodule Pleroma.Web.ApiSpec.Admin.WebhookOperation do security: [%{"oAuth" => ["admin:write"]}], parameters: [id_param()], responses: %{ - 200 => Operation.response("Webhook", "application/json", webhook()) + 200 => Operation.response("Webhook", "application/json", webhook()), + 403 => Operation.response("Forbidden", "application/json", ApiError) } } end @@ -140,7 +145,8 @@ defmodule Pleroma.Web.ApiSpec.Admin.WebhookOperation do security: [%{"oAuth" => ["admin:write"]}], parameters: [id_param()], responses: %{ - 200 => Operation.response("Webhook", "application/json", webhook()) + 200 => Operation.response("Webhook", "application/json", webhook()), + 403 => Operation.response("Forbidden", "application/json", ApiError) } } end @@ -156,6 +162,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.WebhookOperation do events: event_type(), secret: %Schema{type: :string}, enabled: %Schema{type: :boolean}, + internal: %Schema{type: :boolean}, created_at: %Schema{type: :string, format: :"date-time"}, updated_at: %Schema{type: :string, format: :"date-time"} }, @@ -165,6 +172,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.WebhookOperation do "events" => ["report.created"], "secret" => "d3d8cf4bc11fd9c41fd34dcc38d282e451c8bd34", "enabled" => true, + "internal" => false, "created_at" => "2022-06-24T16:19:38.523Z", "updated_at" => "2022-06-24T16:19:38.523Z" } diff --git a/lib/pleroma/webhook.ex b/lib/pleroma/webhook.ex index 6cf47fd68..f1e1b7678 100644 --- a/lib/pleroma/webhook.ex +++ b/lib/pleroma/webhook.ex @@ -18,6 +18,7 @@ defmodule Pleroma.Webhook do field(:events, {:array, Ecto.Enum}, values: @event_types, default: []) field(:secret, :string, default: "") field(:enabled, :boolean, default: true) + field(:internal, :boolean, default: false) timestamps() end @@ -32,7 +33,7 @@ defmodule Pleroma.Webhook do def changeset(%__MODULE__{} = webhook, params) do webhook - |> cast(params, [:url, :events, :enabled]) + |> cast(params, [:url, :events, :enabled, :internal]) |> validate_required([:url, :events]) |> unique_constraint(:url) |> strip_events() @@ -41,7 +42,7 @@ defmodule Pleroma.Webhook do def update_changeset(%__MODULE__{} = webhook, params \\ %{}) do webhook - |> cast(params, [:url, :events, :enabled]) + |> cast(params, [:url, :events, :enabled, :internal]) |> unique_constraint(:url) |> strip_events() end diff --git a/priv/repo/migrations/20221029171353_add_internal_to_webhooks.exs b/priv/repo/migrations/20221029171353_add_internal_to_webhooks.exs new file mode 100644 index 000000000..8c8ce7fc7 --- /dev/null +++ b/priv/repo/migrations/20221029171353_add_internal_to_webhooks.exs @@ -0,0 +1,13 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Repo.Migrations.AddInternalToWebhooks do + use Ecto.Migration + + def change do + alter table(:webhooks) do + add(:internal, :boolean, default: false, null: false) + end + end +end diff --git a/test/pleroma/web/admin_api/controllers/webhook_controller_test.exs b/test/pleroma/web/admin_api/controllers/webhook_controller_test.exs index 6a1586ff1..0cd00e753 100644 --- a/test/pleroma/web/admin_api/controllers/webhook_controller_test.exs +++ b/test/pleroma/web/admin_api/controllers/webhook_controller_test.exs @@ -64,6 +64,20 @@ defmodule Pleroma.Web.AdminAPI.WebhookControllerTest do assert %{events: [:"report.created", :"account.created"]} = Webhook.get(id) end + + test "can't edit an internal webhook", %{conn: conn} do + %{id: id} = + Webhook.create(%{url: "https://example.com/webhook1", events: [], internal: true}) + + conn + |> put_req_header("content-type", "application/json") + |> patch("/api/pleroma/admin/webhooks/#{id}", %{ + events: ["report.created", "account.created"] + }) + |> json_response_and_validate_schema(:forbidden) + + assert %{events: []} = Webhook.get(id) + end end describe "DELETE /api/pleroma/admin/webhooks" do From 0feb23a3082b65915e0abbe09c2df83f6014d6c3 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Fri, 16 Jan 2026 23:16:51 +0400 Subject: [PATCH 11/90] AP ingest: accept object type as list --- .../web/activity_pub/transmogrifier.ex | 6 +++ .../fedidev.fun/object_type_list.json | 28 ++++++++++ .../fedidev_fun_object_type_list_test.exs | 53 +++++++++++++++++++ 3 files changed, 87 insertions(+) create mode 100644 test/fixtures/fedidev.fun/object_type_list.json create mode 100644 test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_object_type_list_test.exs diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index ba6f4030d..6b3194709 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -433,9 +433,15 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def handle_incoming(data, options \\ []) do data |> fix_recursive(&strip_internal_fields/1) + |> fix_recursive(&fix_type_as_string/1) |> handle_incoming_normalized(options) end + defp fix_type_as_string(%{"type" => [type | _]} = data) when is_binary(type), + do: Map.put(data, "type", type) + + defp fix_type_as_string(data), do: data + # Flag objects are placed ahead of the ID check because Mastodon 2.8 and earlier send them # with nil ID. defp handle_incoming_normalized( diff --git a/test/fixtures/fedidev.fun/object_type_list.json b/test/fixtures/fedidev.fun/object_type_list.json new file mode 100644 index 000000000..f4ab00578 --- /dev/null +++ b/test/fixtures/fedidev.fun/object_type_list.json @@ -0,0 +1,28 @@ +{ + "@context": [ + "https://www.w3.org/ns/activitystreams" + ], + "type": "Create", + "id": "https://fedidev.fun/activity/object-type-list", + "actor": "https://fedidev.fun/users/alice", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [], + "published": "2025-11-21T14:43:32Z", + "object": { + "type": [ + "Note" + ], + "id": "https://fedidev.fun/objects/note-object-type-list", + "attributedTo": "https://fedidev.fun/users/alice", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [], + "published": "2025-11-21T14:43:32Z", + "name": "name", + "summary": "summary", + "content": "content" + } +} diff --git a/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_object_type_list_test.exs b/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_object_type_list_test.exs new file mode 100644 index 000000000..619e6f477 --- /dev/null +++ b/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_object_type_list_test.exs @@ -0,0 +1,53 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.Transmogrifier.FedidevFunObjectTypeListTest do + use Oban.Testing, repo: Pleroma.Repo + use Pleroma.DataCase + + alias Pleroma.Object + alias Pleroma.User + alias Pleroma.Web.ActivityPub.Transmogrifier + + import Pleroma.Factory + + setup_all do + Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) + :ok + end + + defp ensure_remote_actor!(ap_id) when is_binary(ap_id) do + case User.get_cached_by_ap_id(ap_id) do + %User{} -> + :ok + + _ -> + insert(:user, + local: false, + ap_id: ap_id, + follower_address: ap_id <> "/followers", + following_address: ap_id <> "/following", + featured_address: ap_id <> "/collections/featured", + inbox: ap_id <> "/inbox" + ) + + :ok + end + end + + defp ingest_fixture!(rel_path) do + data = rel_path |> File.read!() |> Jason.decode!() + ensure_remote_actor!(data["actor"]) + + assert {:ok, activity} = Transmogrifier.handle_incoming(data) + object = Object.normalize(activity.data["object"], fetch: false) + + {activity, object} + end + + test "accepts object type as a list and stores the first type" do + {_activity, object} = ingest_fixture!("test/fixtures/fedidev.fun/object_type_list.json") + assert object.data["type"] == "Note" + end +end From 39817fa54727204dc997e9211bae57d5fd625583 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Fri, 16 Jan 2026 23:25:07 +0400 Subject: [PATCH 12/90] AP ingest: normalize object url list --- .../article_note_page_validator.ex | 27 +++++++++- test/fixtures/fedidev.fun/url_list.json | 27 ++++++++++ .../fedidev_fun_url_list_test.exs | 53 +++++++++++++++++++ 3 files changed, 106 insertions(+), 1 deletion(-) create mode 100644 test/fixtures/fedidev.fun/url_list.json create mode 100644 test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_url_list_test.exs diff --git a/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex b/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex index c0626ce4d..b0cdee552 100644 --- a/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex @@ -46,7 +46,32 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidator do end defp fix_url(%{"url" => url} = data) when is_bitstring(url), do: data - defp fix_url(%{"url" => url} = data) when is_map(url), do: Map.put(data, "url", url["href"]) + + defp fix_url(%{"url" => url} = data) when is_map(url) do + if is_binary(url["href"]) do + Map.put(data, "url", url["href"]) + else + Map.delete(data, "url") + end + end + + defp fix_url(%{"url" => url} = data) when is_list(url) do + first_element = Enum.at(url, 0) + + url_string = + cond do + is_bitstring(first_element) -> first_element + is_map(first_element) -> first_element["href"] + true -> nil + end + + if is_binary(url_string) do + Map.put(data, "url", url_string) + else + Map.delete(data, "url") + end + end + defp fix_url(data), do: data defp fix_tag(%{"tag" => tag} = data) when is_list(tag) do diff --git a/test/fixtures/fedidev.fun/url_list.json b/test/fixtures/fedidev.fun/url_list.json new file mode 100644 index 000000000..b4cfd4cd6 --- /dev/null +++ b/test/fixtures/fedidev.fun/url_list.json @@ -0,0 +1,27 @@ +{ + "@context": [ + "https://www.w3.org/ns/activitystreams" + ], + "type": "Create", + "id": "https://fedidev.fun/activity/url-list", + "actor": "https://fedidev.fun/users/alice", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [], + "published": "2025-11-21T14:39:06Z", + "object": { + "type": "Note", + "id": "https://fedidev.fun/objects/note-url-list", + "attributedTo": "https://fedidev.fun/users/alice", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [], + "published": "2025-11-21T14:39:06Z", + "content": "text", + "url": [ + "https://fedidev.fun/objects/123" + ] + } +} diff --git a/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_url_list_test.exs b/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_url_list_test.exs new file mode 100644 index 000000000..bc084a622 --- /dev/null +++ b/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_url_list_test.exs @@ -0,0 +1,53 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.Transmogrifier.FedidevFunUrlListTest do + use Oban.Testing, repo: Pleroma.Repo + use Pleroma.DataCase + + alias Pleroma.Object + alias Pleroma.User + alias Pleroma.Web.ActivityPub.Transmogrifier + + import Pleroma.Factory + + setup_all do + Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) + :ok + end + + defp ensure_remote_actor!(ap_id) when is_binary(ap_id) do + case User.get_cached_by_ap_id(ap_id) do + %User{} -> + :ok + + _ -> + insert(:user, + local: false, + ap_id: ap_id, + follower_address: ap_id <> "/followers", + following_address: ap_id <> "/following", + featured_address: ap_id <> "/collections/featured", + inbox: ap_id <> "/inbox" + ) + + :ok + end + end + + defp ingest_fixture!(rel_path) do + data = rel_path |> File.read!() |> Jason.decode!() + ensure_remote_actor!(data["actor"]) + + assert {:ok, activity} = Transmogrifier.handle_incoming(data) + object = Object.normalize(activity.data["object"], fetch: false) + + {activity, object} + end + + test "normalizes object url list to a string" do + {_activity, object} = ingest_fixture!("test/fixtures/fedidev.fun/url_list.json") + assert object.data["url"] == "https://fedidev.fun/objects/123" + end +end From 6701f6199006b52a70288d2c90cd93aa97ce34ff Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Fri, 16 Jan 2026 23:27:06 +0400 Subject: [PATCH 13/90] AP ingest: normalize quoteUrl fields --- .../object_validators/common_fixes.ex | 27 ++++++-- .../fedidev.fun/misskey_quote_list.json | 28 +++++++++ test/fixtures/fedidev.fun/quote_uri_list.json | 28 +++++++++ test/fixtures/fedidev.fun/quote_url_list.json | 28 +++++++++ .../fedidev_fun_quote_url_list_test.exs | 63 +++++++++++++++++++ 5 files changed, 170 insertions(+), 4 deletions(-) create mode 100644 test/fixtures/fedidev.fun/misskey_quote_list.json create mode 100644 test/fixtures/fedidev.fun/quote_uri_list.json create mode 100644 test/fixtures/fedidev.fun/quote_url_list.json create mode 100644 test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_quote_url_list_test.exs diff --git a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex index 2e0f7a8a0..f34a4327f 100644 --- a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex +++ b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex @@ -96,23 +96,37 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes do Map.put(data, "to", to) end - def fix_quote_url(%{"quoteUrl" => _quote_url} = data), do: data + def fix_quote_url(%{"quoteUrl" => quote_url} = data) do + case normalize_object_id(quote_url) do + quote_url when is_binary(quote_url) -> Map.put(data, "quoteUrl", quote_url) + _ -> Map.delete(data, "quoteUrl") + end + end # Fedibird # https://github.com/fedibird/mastodon/commit/dbd7ae6cf58a92ec67c512296b4daaea0d01e6ac def fix_quote_url(%{"quoteUri" => quote_url} = data) do - Map.put(data, "quoteUrl", quote_url) + case normalize_object_id(quote_url) do + quote_url when is_binary(quote_url) -> Map.put(data, "quoteUrl", quote_url) + _ -> data + end end # Old Fedibird (bug) # https://github.com/fedibird/mastodon/issues/9 def fix_quote_url(%{"quoteURL" => quote_url} = data) do - Map.put(data, "quoteUrl", quote_url) + case normalize_object_id(quote_url) do + quote_url when is_binary(quote_url) -> Map.put(data, "quoteUrl", quote_url) + _ -> data + end end # Misskey fallback def fix_quote_url(%{"_misskey_quote" => quote_url} = data) do - Map.put(data, "quoteUrl", quote_url) + case normalize_object_id(quote_url) do + quote_url when is_binary(quote_url) -> Map.put(data, "quoteUrl", quote_url) + _ -> data + end end def fix_quote_url(%{"tag" => [_ | _] = tags} = data) do @@ -128,6 +142,11 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes do def fix_quote_url(data), do: data + defp normalize_object_id(object_id) when is_binary(object_id), do: object_id + defp normalize_object_id(%{"id" => object_id}) when is_binary(object_id), do: object_id + defp normalize_object_id([object_id | _]) when is_binary(object_id), do: object_id + defp normalize_object_id(_), do: nil + # On Mastodon, `"likes"` attribute includes an inlined `Collection` with `totalItems`, # not a list of users. # https://github.com/mastodon/mastodon/pull/32007 diff --git a/test/fixtures/fedidev.fun/misskey_quote_list.json b/test/fixtures/fedidev.fun/misskey_quote_list.json new file mode 100644 index 000000000..b9ea08691 --- /dev/null +++ b/test/fixtures/fedidev.fun/misskey_quote_list.json @@ -0,0 +1,28 @@ +{ + "@context": [ + "https://www.w3.org/ns/activitystreams" + ], + "type": "Create", + "id": "https://fedidev.fun/activity/misskey-quote-list", + "actor": "https://fedidev.fun/users/alice", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [], + "published": "2025-11-21T14:43:53Z", + "object": { + "type": "Note", + "id": "https://fedidev.fun/objects/note-misskey-quote-list", + "attributedTo": "https://fedidev.fun/users/alice", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [], + "published": "2025-11-21T14:43:53Z", + "content": "_misskey_quote attribute as list", + "_misskey_quote": [ + "https://fedidev.fun/assets/note1.jsonap", + "https://fedidev.fun/assets/note2.jsonap" + ] + } +} diff --git a/test/fixtures/fedidev.fun/quote_uri_list.json b/test/fixtures/fedidev.fun/quote_uri_list.json new file mode 100644 index 000000000..9245077e6 --- /dev/null +++ b/test/fixtures/fedidev.fun/quote_uri_list.json @@ -0,0 +1,28 @@ +{ + "@context": [ + "https://www.w3.org/ns/activitystreams" + ], + "type": "Create", + "id": "https://fedidev.fun/activity/quote-uri-list", + "actor": "https://fedidev.fun/users/alice", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [], + "published": "2025-11-21T14:43:52Z", + "object": { + "type": "Note", + "id": "https://fedidev.fun/objects/note-quote-uri-list", + "attributedTo": "https://fedidev.fun/users/alice", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [], + "published": "2025-11-21T14:43:52Z", + "content": "quoteUri attribute as list", + "quoteUri": [ + "https://fedidev.fun/assets/note1.jsonap", + "https://fedidev.fun/assets/note2.jsonap" + ] + } +} diff --git a/test/fixtures/fedidev.fun/quote_url_list.json b/test/fixtures/fedidev.fun/quote_url_list.json new file mode 100644 index 000000000..eecd0dd35 --- /dev/null +++ b/test/fixtures/fedidev.fun/quote_url_list.json @@ -0,0 +1,28 @@ +{ + "@context": [ + "https://www.w3.org/ns/activitystreams" + ], + "type": "Create", + "id": "https://fedidev.fun/activity/quote-url-list", + "actor": "https://fedidev.fun/users/alice", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [], + "published": "2025-11-21T14:43:52Z", + "object": { + "type": "Note", + "id": "https://fedidev.fun/objects/note-quote-url-list", + "attributedTo": "https://fedidev.fun/users/alice", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [], + "published": "2025-11-21T14:43:52Z", + "content": "quoteUrl attribute as list", + "quoteUrl": [ + "https://fedidev.fun/assets/note1.jsonap", + "https://fedidev.fun/assets/note2.jsonap" + ] + } +} diff --git a/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_quote_url_list_test.exs b/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_quote_url_list_test.exs new file mode 100644 index 000000000..9607fabdd --- /dev/null +++ b/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_quote_url_list_test.exs @@ -0,0 +1,63 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.Transmogrifier.FedidevFunQuoteUrlListTest do + use Oban.Testing, repo: Pleroma.Repo + use Pleroma.DataCase + + alias Pleroma.Object + alias Pleroma.User + alias Pleroma.Web.ActivityPub.Transmogrifier + + import Pleroma.Factory + + setup_all do + Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) + :ok + end + + defp ensure_remote_actor!(ap_id) when is_binary(ap_id) do + case User.get_cached_by_ap_id(ap_id) do + %User{} -> + :ok + + _ -> + insert(:user, + local: false, + ap_id: ap_id, + follower_address: ap_id <> "/followers", + following_address: ap_id <> "/following", + featured_address: ap_id <> "/collections/featured", + inbox: ap_id <> "/inbox" + ) + + :ok + end + end + + defp ingest_fixture!(rel_path) do + data = rel_path |> File.read!() |> Jason.decode!() + ensure_remote_actor!(data["actor"]) + + assert {:ok, activity} = Transmogrifier.handle_incoming(data) + object = Object.normalize(activity.data["object"], fetch: false) + + {activity, object} + end + + test "normalizes quoteUri list to the first object ID" do + {_activity, object} = ingest_fixture!("test/fixtures/fedidev.fun/quote_uri_list.json") + assert object.data["quoteUrl"] == "https://fedidev.fun/assets/note1.jsonap" + end + + test "normalizes quoteUrl list to the first object ID" do + {_activity, object} = ingest_fixture!("test/fixtures/fedidev.fun/quote_url_list.json") + assert object.data["quoteUrl"] == "https://fedidev.fun/assets/note1.jsonap" + end + + test "normalizes _misskey_quote list to the first object ID" do + {_activity, object} = ingest_fixture!("test/fixtures/fedidev.fun/misskey_quote_list.json") + assert object.data["quoteUrl"] == "https://fedidev.fun/assets/note1.jsonap" + end +end From bff67650e109b8f2064c4b6570164da06fe90ac4 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Fri, 16 Jan 2026 23:57:01 +0400 Subject: [PATCH 14/90] AP ingest: normalize inReplyTo shapes --- .../object_validators/common_fixes.ex | 30 +++++--- .../web/activity_pub/transmogrifier.ex | 30 +++++++- .../fedidev.fun/in_reply_to_collection.json | 30 ++++++++ .../in_reply_to_embedded_note.json | 35 +++++++++ .../fedidev.fun/in_reply_to_link.json | 28 ++++++++ .../fedidev.fun/in_reply_to_list.json | 28 ++++++++ .../fedidev_fun_in_reply_to_test.exs | 72 +++++++++++++++++++ .../transmogrifier/note_handling_test.exs | 4 +- 8 files changed, 245 insertions(+), 12 deletions(-) create mode 100644 test/fixtures/fedidev.fun/in_reply_to_collection.json create mode 100644 test/fixtures/fedidev.fun/in_reply_to_embedded_note.json create mode 100644 test/fixtures/fedidev.fun/in_reply_to_link.json create mode 100644 test/fixtures/fedidev.fun/in_reply_to_list.json create mode 100644 test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_in_reply_to_test.exs diff --git a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex index f34a4327f..631f301af 100644 --- a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex +++ b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex @@ -41,22 +41,34 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes do def fix_object_defaults(data) do data = Maps.filter_empty_values(data) + in_reply_to_id = in_reply_to_id(data["inReplyTo"]) + context = Utils.maybe_create_context( - data["context"] || data["conversation"] || data["inReplyTo"] || data["id"] + data["context"] || data["conversation"] || in_reply_to_id || data["id"] ) - %User{follower_address: follower_collection} = User.get_cached_by_ap_id(data["attributedTo"]) + data = Map.put(data, "context", context) - data - |> Map.put("context", context) - |> cast_and_filter_recipients("to", follower_collection) - |> cast_and_filter_recipients("cc", follower_collection) - |> cast_and_filter_recipients("bto", follower_collection) - |> cast_and_filter_recipients("bcc", follower_collection) - |> Transmogrifier.fix_implicit_addressing(follower_collection) + with attributed_to when is_binary(attributed_to) <- data["attributedTo"], + %User{follower_address: follower_collection} <- User.get_cached_by_ap_id(attributed_to) do + data + |> cast_and_filter_recipients("to", follower_collection) + |> cast_and_filter_recipients("cc", follower_collection) + |> cast_and_filter_recipients("bto", follower_collection) + |> cast_and_filter_recipients("bcc", follower_collection) + |> Transmogrifier.fix_implicit_addressing(follower_collection) + else + _ -> data + end end + defp in_reply_to_id(in_reply_to) when is_binary(in_reply_to), do: in_reply_to + defp in_reply_to_id(%{"id" => in_reply_to}) when is_binary(in_reply_to), do: in_reply_to + defp in_reply_to_id(%{"href" => in_reply_to}) when is_binary(in_reply_to), do: in_reply_to + defp in_reply_to_id([in_reply_to | _]), do: in_reply_to_id(in_reply_to) + defp in_reply_to_id(_), do: nil + def fix_activity_addressing(activity) do %User{follower_address: follower_collection} = User.get_cached_by_ap_id(activity["actor"]) diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 6b3194709..43d991a22 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -201,9 +201,29 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def fix_in_reply_to(%{"inReplyTo" => in_reply_to} = object, options) when not is_nil(in_reply_to) do in_reply_to_id = prepare_in_reply_to(in_reply_to) + + object = + cond do + is_list(in_reply_to) -> + if not_empty_string(in_reply_to_id) do + Map.put(object, "inReplyTo", in_reply_to_id) + else + Map.delete(object, "inReplyTo") + end + + is_map(in_reply_to) and is_nil(in_reply_to["id"]) and is_binary(in_reply_to["href"]) -> + Map.put(object, "inReplyTo", in_reply_to["href"]) + + is_map(in_reply_to) and is_nil(in_reply_to["id"]) and is_nil(in_reply_to["href"]) -> + Map.delete(object, "inReplyTo") + + true -> + object + end + depth = (options[:depth] || 0) + 1 - if Federator.allowed_thread_distance?(depth) do + if not_empty_string(in_reply_to_id) and Federator.allowed_thread_distance?(depth) do with {:ok, replied_object} <- get_obj_helper(in_reply_to_id, options), %Activity{} <- Activity.get_create_by_object_ap_id(replied_object.data["id"]) do object @@ -249,9 +269,17 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do is_map(in_reply_to) && is_bitstring(in_reply_to["id"]) -> in_reply_to["id"] + is_map(in_reply_to) && is_bitstring(in_reply_to["href"]) -> + in_reply_to["href"] + is_list(in_reply_to) && is_bitstring(Enum.at(in_reply_to, 0)) -> Enum.at(in_reply_to, 0) + is_list(in_reply_to) && is_map(Enum.at(in_reply_to, 0)) -> + in_reply_to + |> Enum.at(0) + |> prepare_in_reply_to() + true -> "" end diff --git a/test/fixtures/fedidev.fun/in_reply_to_collection.json b/test/fixtures/fedidev.fun/in_reply_to_collection.json new file mode 100644 index 000000000..4deb7ed9a --- /dev/null +++ b/test/fixtures/fedidev.fun/in_reply_to_collection.json @@ -0,0 +1,30 @@ +{ + "@context": [ + "https://www.w3.org/ns/activitystreams" + ], + "type": "Create", + "id": "https://fedidev.fun/activity/in-reply-to-collection", + "actor": "https://fedidev.fun/users/alice", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [], + "published": "2025-11-21T14:40:08Z", + "object": { + "type": "Note", + "id": "https://fedidev.fun/objects/note-in-reply-to-collection", + "attributedTo": "https://fedidev.fun/users/alice", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [], + "published": "2025-11-21T14:40:08Z", + "content": "collection", + "inReplyTo": { + "type": "Collection", + "items": [ + "https://fedidev.fun/users/alice" + ] + } + } +} diff --git a/test/fixtures/fedidev.fun/in_reply_to_embedded_note.json b/test/fixtures/fedidev.fun/in_reply_to_embedded_note.json new file mode 100644 index 000000000..de66f144f --- /dev/null +++ b/test/fixtures/fedidev.fun/in_reply_to_embedded_note.json @@ -0,0 +1,35 @@ +{ + "@context": [ + "https://www.w3.org/ns/activitystreams" + ], + "type": "Create", + "id": "https://fedidev.fun/activity/in-reply-to-embedded-note", + "actor": "https://fedidev.fun/users/alice", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [], + "published": "2025-11-21T14:40:21Z", + "object": { + "type": "Note", + "id": "https://fedidev.fun/objects/note-in-reply-to-embedded-note", + "attributedTo": "https://fedidev.fun/users/alice", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [], + "published": "2025-11-21T14:40:21Z", + "content": "embedded note", + "inReplyTo": { + "@context": "https://www.w3.org/ns/activitystreams", + "type": "Note", + "id": "https://fedidev.fun/assets/note1.jsonap", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "attributedTo": "https://fedidev.fun/users/alice", + "content": "One", + "published": "2024-01-06T13:11:45Z" + } + } +} diff --git a/test/fixtures/fedidev.fun/in_reply_to_link.json b/test/fixtures/fedidev.fun/in_reply_to_link.json new file mode 100644 index 000000000..f000ddcac --- /dev/null +++ b/test/fixtures/fedidev.fun/in_reply_to_link.json @@ -0,0 +1,28 @@ +{ + "@context": [ + "https://www.w3.org/ns/activitystreams" + ], + "type": "Create", + "id": "https://fedidev.fun/activity/in-reply-to-link", + "actor": "https://fedidev.fun/users/alice", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [], + "published": "2025-11-21T14:39:56Z", + "object": { + "type": "Note", + "id": "https://fedidev.fun/objects/note-in-reply-to-link", + "attributedTo": "https://fedidev.fun/users/alice", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [], + "published": "2025-11-21T14:39:56Z", + "content": "Link in inReplyTo", + "inReplyTo": { + "type": "Link", + "href": "https://fedidev.fun/assets/note1.jsonap" + } + } +} diff --git a/test/fixtures/fedidev.fun/in_reply_to_list.json b/test/fixtures/fedidev.fun/in_reply_to_list.json new file mode 100644 index 000000000..7242558b3 --- /dev/null +++ b/test/fixtures/fedidev.fun/in_reply_to_list.json @@ -0,0 +1,28 @@ +{ + "@context": [ + "https://www.w3.org/ns/activitystreams" + ], + "type": "Create", + "id": "https://fedidev.fun/activity/in-reply-to-list", + "actor": "https://fedidev.fun/users/alice", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [], + "published": "2025-11-21T14:40:08Z", + "object": { + "type": "Note", + "id": "https://fedidev.fun/objects/note-in-reply-to-list", + "attributedTo": "https://fedidev.fun/users/alice", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [], + "published": "2025-11-21T14:40:08Z", + "content": "two elements in inReplyTo", + "inReplyTo": [ + "https://fedidev.fun/assets/note1.jsonap", + "https://fedidev.fun/assets/note2.jsonap" + ] + } +} diff --git a/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_in_reply_to_test.exs b/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_in_reply_to_test.exs new file mode 100644 index 000000000..b720a5441 --- /dev/null +++ b/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_in_reply_to_test.exs @@ -0,0 +1,72 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.Transmogrifier.FedidevFunInReplyToTest do + use Oban.Testing, repo: Pleroma.Repo + use Pleroma.DataCase + + alias Pleroma.Object + alias Pleroma.User + alias Pleroma.Web.ActivityPub.Transmogrifier + + import Pleroma.Factory + + setup_all do + Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) + :ok + end + + defp ensure_remote_actor!(ap_id) when is_binary(ap_id) do + case User.get_cached_by_ap_id(ap_id) do + %User{} -> + :ok + + _ -> + insert(:user, + local: false, + ap_id: ap_id, + follower_address: ap_id <> "/followers", + following_address: ap_id <> "/following", + featured_address: ap_id <> "/collections/featured", + inbox: ap_id <> "/inbox" + ) + + :ok + end + end + + defp ingest_fixture!(rel_path) do + data = rel_path |> File.read!() |> Jason.decode!() + ensure_remote_actor!(data["actor"]) + + assert {:ok, activity} = Transmogrifier.handle_incoming(data) + object = Object.normalize(activity.data["object"], fetch: false) + + {activity, object} + end + + test "normalizes Link inReplyTo to a string" do + {_activity, object} = ingest_fixture!("test/fixtures/fedidev.fun/in_reply_to_link.json") + assert object.data["inReplyTo"] == "https://fedidev.fun/assets/note1.jsonap" + end + + test "normalizes list inReplyTo to the first object ID" do + {_activity, object} = ingest_fixture!("test/fixtures/fedidev.fun/in_reply_to_list.json") + assert object.data["inReplyTo"] == "https://fedidev.fun/assets/note1.jsonap" + end + + test "drops invalid map shapes in inReplyTo" do + {_activity, object} = + ingest_fixture!("test/fixtures/fedidev.fun/in_reply_to_collection.json") + + assert object.data["inReplyTo"] == nil + end + + test "accepts embedded objects in inReplyTo" do + {_activity, object} = + ingest_fixture!("test/fixtures/fedidev.fun/in_reply_to_embedded_note.json") + + assert object.data["inReplyTo"] == "https://fedidev.fun/assets/note1.jsonap" + end +end diff --git a/test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs b/test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs index 403c98a2d..f28d4f771 100644 --- a/test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs +++ b/test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs @@ -572,11 +572,11 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.NoteHandlingTest do Map.put(data["object"], "inReplyTo", ["https://shitposter.club/notice/2827873"]) modified_object = Transmogrifier.fix_in_reply_to(object_with_reply) - assert modified_object["inReplyTo"] == ["https://shitposter.club/notice/2827873"] + assert modified_object["inReplyTo"] == "https://shitposter.club/notice/2827873" object_with_reply = Map.put(data["object"], "inReplyTo", []) modified_object = Transmogrifier.fix_in_reply_to(object_with_reply) - assert modified_object["inReplyTo"] == [] + assert modified_object["inReplyTo"] == nil end test "returns modified object when allowed incoming reply", %{data: data} do From e781a7ef9b231a62c9c3d93cde20841dca13abea Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sat, 17 Jan 2026 00:08:19 +0400 Subject: [PATCH 15/90] AP ingest: harden attachment parsing --- .../article_note_page_validator.ex | 5 +- .../web/activity_pub/transmogrifier.ex | 11 ++- .../fedidev.fun/attachment_embedded_note.json | 32 +++++++++ .../fedidev.fun/attachment_payto_link.json | 31 +++++++++ .../fedidev.fun/attachment_url_list.json | 40 +++++++++++ .../fedidev_fun_attachments_test.exs | 69 +++++++++++++++++++ 6 files changed, 182 insertions(+), 6 deletions(-) create mode 100644 test/fixtures/fedidev.fun/attachment_embedded_note.json create mode 100644 test/fixtures/fedidev.fun/attachment_payto_link.json create mode 100644 test/fixtures/fedidev.fun/attachment_url_list.json create mode 100644 test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_attachments_test.exs diff --git a/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex b/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex index b0cdee552..f2df2250f 100644 --- a/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex @@ -100,10 +100,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidator do defp fix_replies(data), do: Map.delete(data, "replies") - def fix_attachments(%{"attachment" => attachment} = data) when is_map(attachment), - do: Map.put(data, "attachment", [attachment]) - - def fix_attachments(data), do: data + def fix_attachments(data), do: Transmogrifier.fix_attachments(data) defp fix(data) do data diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 43d991a22..cc84cf8af 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -293,6 +293,12 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do |> Map.drop(["conversation"]) end + defp valid_http_url?(href) when is_binary(href) do + match?({:ok, _}, ObjectValidators.Uri.cast(href)) + end + + defp valid_http_url?(_), do: false + def fix_attachments(%{"attachment" => attachment} = object) when is_list(attachment) do attachments = Enum.map(attachment, fn data -> @@ -305,7 +311,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do media_type = cond do - is_map(url) && url =~ Pleroma.Constants.mime_regex() -> + is_map(url) && is_bitstring(url["mediaType"]) && + url["mediaType"] =~ Pleroma.Constants.mime_regex() -> url["mediaType"] is_bitstring(data["mediaType"]) && data["mediaType"] =~ Pleroma.Constants.mime_regex() -> @@ -326,7 +333,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do true -> nil end - if href do + if href && valid_http_url?(href) do attachment_url = %{ "href" => href, diff --git a/test/fixtures/fedidev.fun/attachment_embedded_note.json b/test/fixtures/fedidev.fun/attachment_embedded_note.json new file mode 100644 index 000000000..8e83762dc --- /dev/null +++ b/test/fixtures/fedidev.fun/attachment_embedded_note.json @@ -0,0 +1,32 @@ +{ + "@context": [ + "https://www.w3.org/ns/activitystreams" + ], + "type": "Create", + "id": "https://fedidev.fun/activity/attachment-embedded-note", + "actor": "https://fedidev.fun/users/alice", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [], + "published": "2025-11-21T14:43:01Z", + "object": { + "type": "Note", + "id": "https://fedidev.fun/objects/note-attachment-embedded-note", + "attributedTo": "https://fedidev.fun/users/alice", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [], + "published": "2025-11-21T14:43:01Z", + "content": "Note with embedded Note attachment", + "attachment": { + "type": "Note", + "attributedTo": "https://fedidev.fun/users/alice", + "name": "attached note", + "content": "This is just a note", + "published": "2024-03-06T07:23:56Z" + } + } +} + diff --git a/test/fixtures/fedidev.fun/attachment_payto_link.json b/test/fixtures/fedidev.fun/attachment_payto_link.json new file mode 100644 index 000000000..1da324cb3 --- /dev/null +++ b/test/fixtures/fedidev.fun/attachment_payto_link.json @@ -0,0 +1,31 @@ +{ + "@context": [ + "https://www.w3.org/ns/activitystreams" + ], + "type": "Create", + "id": "https://fedidev.fun/activity/attachment-payto-link", + "actor": "https://fedidev.fun/users/alice", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [], + "published": "2025-11-21T14:43:00Z", + "object": { + "type": "Note", + "id": "https://fedidev.fun/objects/note-attachment-payto-link", + "attributedTo": "https://fedidev.fun/users/alice", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [], + "published": "2025-11-21T14:43:00Z", + "content": "Payment Link", + "attachment": { + "type": "Link", + "name": "Donate", + "href": "payto://iban/DE75512108001245126199", + "rel": "payment" + } + } +} + diff --git a/test/fixtures/fedidev.fun/attachment_url_list.json b/test/fixtures/fedidev.fun/attachment_url_list.json new file mode 100644 index 000000000..0efaeafdb --- /dev/null +++ b/test/fixtures/fedidev.fun/attachment_url_list.json @@ -0,0 +1,40 @@ +{ + "@context": [ + "https://www.w3.org/ns/activitystreams" + ], + "type": "Create", + "id": "https://fedidev.fun/activity/attachment-url-list", + "actor": "https://fedidev.fun/users/alice", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [], + "published": "2025-11-21T14:43:02Z", + "object": { + "type": "Note", + "id": "https://fedidev.fun/objects/note-attachment-url-list", + "attributedTo": "https://fedidev.fun/users/alice", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [], + "published": "2025-11-21T14:43:02Z", + "content": "url is array of two Link objects", + "attachment": { + "type": "Image", + "url": [ + { + "type": "Link", + "href": "https://fedidev.fun/images/007.png", + "mediaType": "image/png" + }, + { + "type": "Link", + "href": "https://fedidev.fun/images/008.jpg", + "mediaType": "image/jpeg" + } + ] + } + } +} + diff --git a/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_attachments_test.exs b/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_attachments_test.exs new file mode 100644 index 000000000..a13912b08 --- /dev/null +++ b/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_attachments_test.exs @@ -0,0 +1,69 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.Transmogrifier.FedidevFunAttachmentsTest do + use Oban.Testing, repo: Pleroma.Repo + use Pleroma.DataCase + + alias Pleroma.Object + alias Pleroma.User + alias Pleroma.Web.ActivityPub.Transmogrifier + + import Pleroma.Factory + + setup_all do + Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) + :ok + end + + defp ensure_remote_actor!(ap_id) when is_binary(ap_id) do + case User.get_cached_by_ap_id(ap_id) do + %User{} -> + :ok + + _ -> + insert(:user, + local: false, + ap_id: ap_id, + follower_address: ap_id <> "/followers", + following_address: ap_id <> "/following", + featured_address: ap_id <> "/collections/featured", + inbox: ap_id <> "/inbox" + ) + + :ok + end + end + + defp ingest_fixture!(rel_path) do + data = rel_path |> File.read!() |> Jason.decode!() + ensure_remote_actor!(data["actor"]) + + assert {:ok, activity} = Transmogrifier.handle_incoming(data) + object = Object.normalize(activity.data["object"], fetch: false) + + {activity, object} + end + + test "drops non-http(s) attachment href" do + {_activity, object} = ingest_fixture!("test/fixtures/fedidev.fun/attachment_payto_link.json") + assert object.data["attachment"] == [] + end + + test "drops embedded non-attachment objects" do + {_activity, object} = + ingest_fixture!("test/fixtures/fedidev.fun/attachment_embedded_note.json") + + assert object.data["attachment"] == [] + end + + test "accepts attachment url as list and keeps the first link" do + {_activity, object} = ingest_fixture!("test/fixtures/fedidev.fun/attachment_url_list.json") + + [attachment] = object.data["attachment"] + [%{"href" => href} | _] = attachment["url"] + + assert href == "https://fedidev.fun/images/007.png" + end +end From e9e3d255e5349f913ed88a36e6ff38ec93c8544b Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sat, 17 Jan 2026 00:12:48 +0400 Subject: [PATCH 16/90] AP ingest: infer tag type when missing --- .../object_validators/tag_validator.ex | 21 ++++++ .../fedidev.fun/tag_hashtag_missing_type.json | 30 +++++++++ .../fedidev.fun/tag_mention_missing_type.json | 31 +++++++++ .../fedidev_fun_tag_inference_test.exs | 65 +++++++++++++++++++ 4 files changed, 147 insertions(+) create mode 100644 test/fixtures/fedidev.fun/tag_hashtag_missing_type.json create mode 100644 test/fixtures/fedidev.fun/tag_mention_missing_type.json create mode 100644 test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_tag_inference_test.exs diff --git a/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex b/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex index dc2770189..a412c8dea 100644 --- a/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex @@ -80,6 +80,27 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.TagValidator do |> Map.put(:action, :ignore) end + def changeset(struct, data) when is_map(data) do + data = infer_type(data) + + if Map.has_key?(data, "type") do + changeset(struct, data) + else + struct + |> cast(data, []) + |> Map.put(:action, :ignore) + end + end + + defp infer_type(%{"type" => _} = data), do: data + defp infer_type(%{"name" => "#" <> _} = data), do: Map.put(data, "type", "Hashtag") + defp infer_type(%{"name" => "@" <> _} = data), do: Map.put(data, "type", "Mention") + + defp infer_type(%{"mediaType" => _media_type, "href" => _href} = data), + do: Map.put(data, "type", "Link") + + defp infer_type(data), do: data + def icon_changeset(struct, data) do struct |> cast(data, [:type, :url]) diff --git a/test/fixtures/fedidev.fun/tag_hashtag_missing_type.json b/test/fixtures/fedidev.fun/tag_hashtag_missing_type.json new file mode 100644 index 000000000..74f9d91b8 --- /dev/null +++ b/test/fixtures/fedidev.fun/tag_hashtag_missing_type.json @@ -0,0 +1,30 @@ +{ + "@context": [ + "https://www.w3.org/ns/activitystreams" + ], + "type": "Create", + "id": "https://fedidev.fun/activity/tag-hashtag-missing-type", + "actor": "https://fedidev.fun/users/alice", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [], + "published": "2025-11-21T14:41:00Z", + "object": { + "type": "Note", + "id": "https://fedidev.fun/objects/note-tag-hashtag-missing-type", + "attributedTo": "https://fedidev.fun/users/alice", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [], + "published": "2025-11-21T14:41:00Z", + "content": "hashtag without type", + "tag": [ + { + "name": "#Test" + } + ] + } +} + diff --git a/test/fixtures/fedidev.fun/tag_mention_missing_type.json b/test/fixtures/fedidev.fun/tag_mention_missing_type.json new file mode 100644 index 000000000..d290be6c2 --- /dev/null +++ b/test/fixtures/fedidev.fun/tag_mention_missing_type.json @@ -0,0 +1,31 @@ +{ + "@context": [ + "https://www.w3.org/ns/activitystreams" + ], + "type": "Create", + "id": "https://fedidev.fun/activity/tag-mention-missing-type", + "actor": "https://fedidev.fun/users/alice", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [], + "published": "2025-11-21T14:41:01Z", + "object": { + "type": "Note", + "id": "https://fedidev.fun/objects/note-tag-mention-missing-type", + "attributedTo": "https://fedidev.fun/users/alice", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [], + "published": "2025-11-21T14:41:01Z", + "content": "mention without type", + "tag": [ + { + "name": "@bob", + "href": "https://remote.example/users/bob" + } + ] + } +} + diff --git a/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_tag_inference_test.exs b/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_tag_inference_test.exs new file mode 100644 index 000000000..ef7feb8ad --- /dev/null +++ b/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_tag_inference_test.exs @@ -0,0 +1,65 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.Transmogrifier.FedidevFunTagInferenceTest do + use Oban.Testing, repo: Pleroma.Repo + use Pleroma.DataCase + + alias Pleroma.Object + alias Pleroma.User + alias Pleroma.Web.ActivityPub.Transmogrifier + + import Pleroma.Factory + + setup_all do + Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) + :ok + end + + defp ensure_remote_actor!(ap_id) when is_binary(ap_id) do + case User.get_cached_by_ap_id(ap_id) do + %User{} -> + :ok + + _ -> + insert(:user, + local: false, + ap_id: ap_id, + follower_address: ap_id <> "/followers", + following_address: ap_id <> "/following", + featured_address: ap_id <> "/collections/featured", + inbox: ap_id <> "/inbox" + ) + + :ok + end + end + + defp ingest_fixture!(rel_path) do + data = rel_path |> File.read!() |> Jason.decode!() + ensure_remote_actor!(data["actor"]) + + assert {:ok, activity} = Transmogrifier.handle_incoming(data) + object = Object.normalize(activity.data["object"], fetch: false) + + {activity, object} + end + + test "infers Hashtag type when missing" do + {_activity, object} = + ingest_fixture!("test/fixtures/fedidev.fun/tag_hashtag_missing_type.json") + + assert "test" in Object.hashtags(object) + end + + test "infers Mention type when missing" do + {_activity, object} = + ingest_fixture!("test/fixtures/fedidev.fun/tag_mention_missing_type.json") + + assert Enum.any?(object.data["tag"], fn + %{"type" => "Mention", "href" => "https://remote.example/users/bob"} -> true + _ -> false + end) + end +end From 9912a9f603b1d0816f95491b67a9ba12553123ce Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sat, 17 Jan 2026 01:33:21 +0400 Subject: [PATCH 17/90] AP ingest: accept Emoji icon variants --- .../object_validators/tag_validator.ex | 62 ++++++++++++++++-- .../web/activity_pub/transmogrifier.ex | 30 ++++++++- .../fedidev.fun/emoji_icon_string.json | 32 +++++++++ .../emoji_icon_url_missing_type.json | 34 ++++++++++ .../fedidev.fun/emoji_missing_icon.json | 31 +++++++++ .../transmogrifier/fedidev_fun_emoji_test.exs | 65 +++++++++++++++++++ 6 files changed, 246 insertions(+), 8 deletions(-) create mode 100644 test/fixtures/fedidev.fun/emoji_icon_string.json create mode 100644 test/fixtures/fedidev.fun/emoji_icon_url_missing_type.json create mode 100644 test/fixtures/fedidev.fun/emoji_missing_icon.json create mode 100644 test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_emoji_test.exs diff --git a/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex b/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex index a412c8dea..cdab8d78d 100644 --- a/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex @@ -59,12 +59,29 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.TagValidator do end def changeset(struct, %{"type" => "Emoji"} = data) do - data = Map.put(data, "name", String.trim(data["name"], ":")) + data = + data + |> Map.put("name", String.trim(data["name"], ":")) + |> normalize_emoji_icon() - struct - |> cast(data, [:type, :name, :updated, :id]) - |> cast_embed(:icon, with: &icon_changeset/2) - |> validate_required([:type, :name, :icon]) + case data["icon"] do + %{"url" => url} when is_binary(url) -> + if valid_http_url?(url) do + struct + |> cast(data, [:type, :name, :updated, :id]) + |> cast_embed(:icon, with: &icon_changeset/2) + |> validate_required([:type, :name, :icon]) + else + struct + |> cast(data, []) + |> Map.put(:action, :ignore) + end + + _ -> + struct + |> cast(data, []) + |> Map.put(:action, :ignore) + end end def changeset(struct, %{"type" => "Link"} = data) do @@ -101,6 +118,41 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.TagValidator do defp infer_type(data), do: data + defp normalize_emoji_icon(%{"icon" => icon} = data) when is_binary(icon) do + Map.put(data, "icon", %{"type" => "Image", "url" => icon}) + end + + defp normalize_emoji_icon(%{"icon" => icon} = data) when is_map(icon) do + case extract_icon_url(icon) do + url when is_binary(url) -> Map.put(data, "icon", %{"type" => "Image", "url" => url}) + _ -> Map.delete(data, "icon") + end + end + + defp normalize_emoji_icon(data), do: data + + defp extract_icon_url(%{"url" => url}) when is_binary(url), do: url + + defp extract_icon_url(%{"url" => %{"href" => href}}) when is_binary(href), do: href + + defp extract_icon_url(%{"url" => [first | _]}) do + cond do + is_binary(first) -> first + is_map(first) -> first["href"] + true -> nil + end + end + + defp extract_icon_url(%{"href" => href}) when is_binary(href), do: href + + defp extract_icon_url(_), do: nil + + defp valid_http_url?(url) when is_binary(url) do + match?({:ok, _}, ObjectValidators.Uri.cast(url)) + end + + defp valid_http_url?(_), do: false + def icon_changeset(struct, data) do struct |> cast(data, [:type, :url]) diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index cc84cf8af..55b70b881 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -386,6 +386,24 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def fix_url(object), do: object + defp emoji_icon_url(icon) when is_binary(icon), do: icon + + defp emoji_icon_url(%{"url" => url}) when is_binary(url), do: url + + defp emoji_icon_url(%{"url" => %{"href" => href}}) when is_binary(href), do: href + + defp emoji_icon_url(%{"url" => [first | _]}) do + cond do + is_binary(first) -> first + is_map(first) -> first["href"] || first["url"] || first["id"] + true -> nil + end + end + + defp emoji_icon_url(%{"href" => href}) when is_binary(href), do: href + + defp emoji_icon_url(_), do: nil + def fix_emoji(%{"tag" => tags} = object) when is_list(tags) do emoji = tags @@ -393,7 +411,10 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do |> Enum.reduce(%{}, fn data, mapping -> name = String.trim(data["name"], ":") - Map.put(mapping, name, data["icon"]["url"]) + case emoji_icon_url(data["icon"]) do + url when is_binary(url) -> Map.put(mapping, name, url) + _ -> mapping + end end) Map.put(object, "emoji", emoji) @@ -401,9 +422,12 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def fix_emoji(%{"tag" => %{"type" => "Emoji"} = tag} = object) do name = String.trim(tag["name"], ":") - emoji = %{name => tag["icon"]["url"]} - Map.put(object, "emoji", emoji) + with url when is_binary(url) <- emoji_icon_url(tag["icon"]) do + Map.put(object, "emoji", %{name => url}) + else + _ -> object + end end def fix_emoji(object), do: object diff --git a/test/fixtures/fedidev.fun/emoji_icon_string.json b/test/fixtures/fedidev.fun/emoji_icon_string.json new file mode 100644 index 000000000..0be3deb8c --- /dev/null +++ b/test/fixtures/fedidev.fun/emoji_icon_string.json @@ -0,0 +1,32 @@ +{ + "@context": [ + "https://www.w3.org/ns/activitystreams" + ], + "type": "Create", + "id": "https://fedidev.fun/activity/emoji-icon-string", + "actor": "https://fedidev.fun/users/alice", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [], + "published": "2025-11-21T14:40:00Z", + "object": { + "type": "Note", + "id": "https://fedidev.fun/objects/note-emoji-icon-string", + "attributedTo": "https://fedidev.fun/users/alice", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [], + "published": "2025-11-21T14:40:00Z", + "content": "emoji icon as string :cow:", + "tag": [ + { + "type": "Emoji", + "name": ":cow:", + "icon": "https://fedidev.fun/static/cow.png" + } + ] + } +} + diff --git a/test/fixtures/fedidev.fun/emoji_icon_url_missing_type.json b/test/fixtures/fedidev.fun/emoji_icon_url_missing_type.json new file mode 100644 index 000000000..2837872eb --- /dev/null +++ b/test/fixtures/fedidev.fun/emoji_icon_url_missing_type.json @@ -0,0 +1,34 @@ +{ + "@context": [ + "https://www.w3.org/ns/activitystreams" + ], + "type": "Create", + "id": "https://fedidev.fun/activity/emoji-icon-url-missing-type", + "actor": "https://fedidev.fun/users/alice", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [], + "published": "2025-11-21T14:40:01Z", + "object": { + "type": "Note", + "id": "https://fedidev.fun/objects/note-emoji-icon-url-missing-type", + "attributedTo": "https://fedidev.fun/users/alice", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [], + "published": "2025-11-21T14:40:01Z", + "content": "emoji icon url missing type :cow2:", + "tag": [ + { + "type": "Emoji", + "name": ":cow2:", + "icon": { + "url": "https://fedidev.fun/static/cow.png" + } + } + ] + } +} + diff --git a/test/fixtures/fedidev.fun/emoji_missing_icon.json b/test/fixtures/fedidev.fun/emoji_missing_icon.json new file mode 100644 index 000000000..69af01a73 --- /dev/null +++ b/test/fixtures/fedidev.fun/emoji_missing_icon.json @@ -0,0 +1,31 @@ +{ + "@context": [ + "https://www.w3.org/ns/activitystreams" + ], + "type": "Create", + "id": "https://fedidev.fun/activity/emoji-missing-icon", + "actor": "https://fedidev.fun/users/alice", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [], + "published": "2025-11-21T14:40:02Z", + "object": { + "type": "Note", + "id": "https://fedidev.fun/objects/note-emoji-missing-icon", + "attributedTo": "https://fedidev.fun/users/alice", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [], + "published": "2025-11-21T14:40:02Z", + "content": "emoji missing icon :cow3:", + "tag": [ + { + "type": "Emoji", + "name": ":cow3:" + } + ] + } +} + diff --git a/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_emoji_test.exs b/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_emoji_test.exs new file mode 100644 index 000000000..56c305352 --- /dev/null +++ b/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_emoji_test.exs @@ -0,0 +1,65 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.Transmogrifier.FedidevFunEmojiTest do + use Oban.Testing, repo: Pleroma.Repo + use Pleroma.DataCase + + alias Pleroma.Object + alias Pleroma.User + alias Pleroma.Web.ActivityPub.Transmogrifier + + import Pleroma.Factory + + setup_all do + Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) + :ok + end + + defp ensure_remote_actor!(ap_id) when is_binary(ap_id) do + case User.get_cached_by_ap_id(ap_id) do + %User{} -> + :ok + + _ -> + insert(:user, + local: false, + ap_id: ap_id, + follower_address: ap_id <> "/followers", + following_address: ap_id <> "/following", + featured_address: ap_id <> "/collections/featured", + inbox: ap_id <> "/inbox" + ) + + :ok + end + end + + defp ingest_fixture!(rel_path) do + data = rel_path |> File.read!() |> Jason.decode!() + ensure_remote_actor!(data["actor"]) + + assert {:ok, activity} = Transmogrifier.handle_incoming(data) + object = Object.normalize(activity.data["object"], fetch: false) + + {activity, object} + end + + test "ingests Emoji tag with string icon" do + {_activity, object} = ingest_fixture!("test/fixtures/fedidev.fun/emoji_icon_string.json") + assert object.data["emoji"]["cow"] == "https://fedidev.fun/static/cow.png" + end + + test "ingests Emoji tag with icon missing type" do + {_activity, object} = + ingest_fixture!("test/fixtures/fedidev.fun/emoji_icon_url_missing_type.json") + + assert object.data["emoji"]["cow2"] == "https://fedidev.fun/static/cow.png" + end + + test "ignores Emoji tag missing icon" do + {_activity, object} = ingest_fixture!("test/fixtures/fedidev.fun/emoji_missing_icon.json") + assert object.data["emoji"] == %{} + end +end From 3b3e369f83940db06da4030242c9ccc2ba8897cf Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sat, 17 Jan 2026 02:46:59 +0400 Subject: [PATCH 18/90] AP ingest: fill missing attributedTo from activity --- lib/pleroma/object/containment.ex | 30 ++++++++---- .../web/activity_pub/object_validator.ex | 2 + .../object_validators/common_fixes.ex | 11 +++++ .../fedidev.fun/missing_attributed_to.json | 25 ++++++++++ ...fedidev_fun_missing_attributed_to_test.exs | 49 +++++++++++++++++++ 5 files changed, 108 insertions(+), 9 deletions(-) create mode 100644 test/fixtures/fedidev.fun/missing_attributed_to.json create mode 100644 test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_missing_attributed_to_test.exs diff --git a/lib/pleroma/object/containment.ex b/lib/pleroma/object/containment.ex index 77fac12c0..b90d68636 100644 --- a/lib/pleroma/object/containment.ex +++ b/lib/pleroma/object/containment.ex @@ -15,12 +15,28 @@ defmodule Pleroma.Object.Containment do actor end + def get_actor(%{"actor" => nil, "attributedTo" => actor}) when not is_nil(actor) do + get_actor(%{"actor" => actor}) + end + + def get_actor(%{"actor" => nil}), do: nil + + def get_actor(%{"actor" => []}), do: nil + def get_actor(%{"actor" => actor}) when is_list(actor) do - if is_binary(Enum.at(actor, 0)) do - Enum.at(actor, 0) - else - Enum.find(actor, fn %{"type" => type} -> type in ["Person", "Service", "Application"] end) - |> Map.get("id") + case actor do + [first | _] when is_binary(first) -> + first + + _ -> + Enum.find_value(actor, fn + %{"type" => type, "id" => id} + when type in ["Person", "Service", "Application"] and is_binary(id) -> + id + + _ -> + nil + end) end end @@ -28,10 +44,6 @@ defmodule Pleroma.Object.Containment do id end - def get_actor(%{"actor" => nil, "attributedTo" => actor}) when not is_nil(actor) do - get_actor(%{"actor" => actor}) - end - def get_object(%{"object" => id}) when is_binary(id) do id end diff --git a/lib/pleroma/web/activity_pub/object_validator.ex b/lib/pleroma/web/activity_pub/object_validator.ex index 17652a0de..0254d6dc1 100644 --- a/lib/pleroma/web/activity_pub/object_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validator.ex @@ -118,6 +118,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do when objtype in ~w[Question Answer Audio Video Image Event Article Note Page] do with {:ok, object_data} <- object + |> CommonFixes.maybe_set_attributed_to_from_activity(create_activity) |> CommonFixes.maybe_add_language_from_activity(create_activity) |> cast_and_apply_and_stringify_with_history(), meta = Keyword.put(meta, :object_data, object_data), @@ -172,6 +173,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do {_, {:ok, object_data, _}} <- {:object_validation, object + |> CommonFixes.maybe_set_attributed_to_from_activity(update_activity) |> CommonFixes.maybe_add_language_from_activity(update_activity) |> validate(meta)}, meta = Keyword.put(meta, :object_data, object_data), diff --git a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex index 631f301af..c5defd6f4 100644 --- a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex +++ b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex @@ -91,6 +91,17 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes do |> Map.put("attributedTo", actor) end + def maybe_set_attributed_to_from_activity(object, activity) when is_map(object) do + if is_nil(object["attributedTo"]) and is_nil(object["actor"]) do + case Containment.get_actor(activity) do + actor when is_binary(actor) -> Map.put(object, "attributedTo", actor) + _ -> object + end + else + object + end + end + def fix_activity_context(data, %Object{data: %{"context" => object_context}}) do data |> Map.put("context", object_context) diff --git a/test/fixtures/fedidev.fun/missing_attributed_to.json b/test/fixtures/fedidev.fun/missing_attributed_to.json new file mode 100644 index 000000000..c2e644e84 --- /dev/null +++ b/test/fixtures/fedidev.fun/missing_attributed_to.json @@ -0,0 +1,25 @@ +{ + "@context": [ + "https://www.w3.org/ns/activitystreams" + ], + "type": "Create", + "id": "https://fedidev.fun/activity/missing-attributedto", + "actor": "https://fedidev.fun/users/alice", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [], + "published": "2025-11-21T14:42:00Z", + "object": { + "type": "Note", + "id": "https://fedidev.fun/objects/note-missing-attributedto", + "attributedTo": null, + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [], + "published": "2025-11-21T14:42:00Z", + "content": "object without attributedTo" + } +} + diff --git a/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_missing_attributed_to_test.exs b/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_missing_attributed_to_test.exs new file mode 100644 index 000000000..c08e20fc9 --- /dev/null +++ b/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_missing_attributed_to_test.exs @@ -0,0 +1,49 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.Transmogrifier.FedidevFunMissingAttributedToTest do + use Oban.Testing, repo: Pleroma.Repo + use Pleroma.DataCase + + alias Pleroma.Object + alias Pleroma.User + alias Pleroma.Web.ActivityPub.Transmogrifier + + import Pleroma.Factory + + setup_all do + Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) + :ok + end + + defp ensure_remote_actor!(ap_id) when is_binary(ap_id) do + case User.get_cached_by_ap_id(ap_id) do + %User{} -> + :ok + + _ -> + insert(:user, + local: false, + ap_id: ap_id, + follower_address: ap_id <> "/followers", + following_address: ap_id <> "/following", + featured_address: ap_id <> "/collections/featured", + inbox: ap_id <> "/inbox" + ) + + :ok + end + end + + test "inherits attributedTo from surrounding activity" do + data = File.read!("test/fixtures/fedidev.fun/missing_attributed_to.json") |> Jason.decode!() + ensure_remote_actor!(data["actor"]) + + assert {:ok, activity} = Transmogrifier.handle_incoming(data) + object = Object.normalize(activity.data["object"], fetch: false) + + assert object.data["attributedTo"] == data["actor"] + assert object.data["actor"] == data["actor"] + end +end From 10ab71d1577dd54719cdf81172707f4080a95d49 Mon Sep 17 00:00:00 2001 From: Dryusdan Date: Wed, 21 Jan 2026 17:55:31 +0100 Subject: [PATCH 19/90] Improve Debian 13 OTP dependencies and remove Debian 12 in source installation Cause of Elexir ~> 1.15 (mandatory in Pleroma 2.10) that don't support in Debian 12, I fix doc and replace Debian 12 by Debian 13 And replace a dependencies in OTP release + add a new dependencies for Debian 13. --- changelog.d/improve-doc-debian-13.change | 1 + docs/installation/debian_based_en.md | 4 ++-- docs/installation/otp_en.md | 6 +++++- 3 files changed, 8 insertions(+), 3 deletions(-) create mode 100644 changelog.d/improve-doc-debian-13.change diff --git a/changelog.d/improve-doc-debian-13.change b/changelog.d/improve-doc-debian-13.change new file mode 100644 index 000000000..c8f62ef71 --- /dev/null +++ b/changelog.d/improve-doc-debian-13.change @@ -0,0 +1 @@ +Improve Debian 13 OTP dependencies and remove Debian 12 in source installation diff --git a/docs/installation/debian_based_en.md b/docs/installation/debian_based_en.md index 30f48792d..df45e8366 100644 --- a/docs/installation/debian_based_en.md +++ b/docs/installation/debian_based_en.md @@ -4,7 +4,7 @@ ## Installation -This guide will assume you are on Debian 12 (“bookworm”) or later. This guide should also work with Ubuntu 22.04 (“jammy”) and later. It also assumes that you have administrative rights, either as root or a user with [sudo permissions](https://www.digitalocean.com/community/tutorials/how-to-add-delete-and-grant-sudo-privileges-to-users-on-a-debian-vps). If you want to run this guide with root, ignore the `sudo` at the beginning of the lines, unless it calls a user like `sudo -Hu pleroma`; in this case, use `su -s $SHELL -c 'command'` instead. +This guide will assume you are on Debian 13 (“Trixie”) or later. This guide should also work with Ubuntu 24.04 (“noble”) and later. It also assumes that you have administrative rights, either as root or a user with [sudo permissions](https://www.digitalocean.com/community/tutorials/how-to-add-delete-and-grant-sudo-privileges-to-users-on-a-debian-vps). If you want to run this guide with root, ignore the `sudo` at the beginning of the lines, unless it calls a user like `sudo -Hu pleroma`; in this case, use `su -s $SHELL -c 'command'` instead. {! backend/installation/generic_dependencies.include !} @@ -69,7 +69,7 @@ cd /opt/pleroma sudo -Hu pleroma mix deps.get ``` -* Generate the configuration: +* Generate the configuration: ```shell sudo -Hu pleroma MIX_ENV=prod mix pleroma.instance gen diff --git a/docs/installation/otp_en.md b/docs/installation/otp_en.md index 123dccf3a..f3a571510 100644 --- a/docs/installation/otp_en.md +++ b/docs/installation/otp_en.md @@ -2,7 +2,7 @@ {! backend/installation/otp_vs_from_source.include !} -This guide covers a installation using OTP releases as built by the Pleroma project, it is meant as a fallback to distribution packages/recipes which are the preferred installation method. +This guide covers a installation using OTP releases as built by the Pleroma project, it is meant as a fallback to distribution packages/recipes which are the preferred installation method. To install Pleroma from source, please check out the corresponding guide for your distro. ## Pre-requisites @@ -48,6 +48,10 @@ Other than things bundled in the OTP release Pleroma depends on: ``` apt install curl unzip libncurses5 postgresql postgresql-contrib nginx certbot libmagic-dev ``` + On Debian 13, replace `libncurses5` by `libncurses6` and add `libvips42t64`. So + ``` + apt install curl unzip libncurses6 libvips42t64 postgresql postgresql-contrib nginx certbot libmagic-dev + ``` ### Installing optional packages From 145b38b3149acb93bf1cf72693fe881f8b3eaa74 Mon Sep 17 00:00:00 2001 From: Dryusdan Date: Wed, 21 Jan 2026 20:15:55 +0100 Subject: [PATCH 20/90] Reset deleted double space --- docs/installation/debian_based_en.md | 2 +- docs/installation/otp_en.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/installation/debian_based_en.md b/docs/installation/debian_based_en.md index df45e8366..b6730ce19 100644 --- a/docs/installation/debian_based_en.md +++ b/docs/installation/debian_based_en.md @@ -69,7 +69,7 @@ cd /opt/pleroma sudo -Hu pleroma mix deps.get ``` -* Generate the configuration: +* Generate the configuration: ```shell sudo -Hu pleroma MIX_ENV=prod mix pleroma.instance gen diff --git a/docs/installation/otp_en.md b/docs/installation/otp_en.md index f3a571510..4b88b11e0 100644 --- a/docs/installation/otp_en.md +++ b/docs/installation/otp_en.md @@ -2,7 +2,7 @@ {! backend/installation/otp_vs_from_source.include !} -This guide covers a installation using OTP releases as built by the Pleroma project, it is meant as a fallback to distribution packages/recipes which are the preferred installation method. +This guide covers a installation using OTP releases as built by the Pleroma project, it is meant as a fallback to distribution packages/recipes which are the preferred installation method. To install Pleroma from source, please check out the corresponding guide for your distro. ## Pre-requisites From 177edbbb2c7258876e1c204130cd3f47667c7e11 Mon Sep 17 00:00:00 2001 From: Dryusdan Date: Thu, 22 Jan 2026 08:50:50 +0100 Subject: [PATCH 21/90] Add more RUM versions --- docs/installation/otp_en.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/installation/otp_en.md b/docs/installation/otp_en.md index 4b88b11e0..d11cccccb 100644 --- a/docs/installation/otp_en.md +++ b/docs/installation/otp_en.md @@ -48,7 +48,7 @@ Other than things bundled in the OTP release Pleroma depends on: ``` apt install curl unzip libncurses5 postgresql postgresql-contrib nginx certbot libmagic-dev ``` - On Debian 13, replace `libncurses5` by `libncurses6` and add `libvips42t64`. So + On Debian 13 and Ubuntu 24.04, replace `libncurses5` by `libncurses6` and add `libvips42t64`. So ``` apt install curl unzip libncurses6 libvips42t64 postgresql postgresql-contrib nginx certbot libmagic-dev ``` @@ -93,8 +93,12 @@ RUM indexes are an alternative indexing scheme that is not included in PostgreSQ === "Debian/Ubuntu" ``` - # Available only on Buster/19.04 - apt install postgresql-11-rum + # On Trixie and Ubuntu 25.04 + apt install postgresql-17-rum + ``` + # On Ubuntu 24.04 + ``` + apt install postgresql-16-rum ``` #### (Optional) Performance configuration From ce188baa924d6275393147d80ffd804cb71895f8 Mon Sep 17 00:00:00 2001 From: Dryusdan Date: Thu, 22 Jan 2026 08:55:14 +0100 Subject: [PATCH 22/90] Add more Postgresql-rum version --- docs/installation/otp_en.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/installation/otp_en.md b/docs/installation/otp_en.md index d11cccccb..a1238b160 100644 --- a/docs/installation/otp_en.md +++ b/docs/installation/otp_en.md @@ -96,10 +96,18 @@ RUM indexes are an alternative indexing scheme that is not included in PostgreSQ # On Trixie and Ubuntu 25.04 apt install postgresql-17-rum ``` + # On Debian 12 + ``` + apt install postgresql-15-rum + ``` # On Ubuntu 24.04 ``` apt install postgresql-16-rum ``` + # On Ubuntu 22.04 + ``` + apt install postgresql-14-rum + ``` #### (Optional) Performance configuration It is encouraged to check [Optimizing your PostgreSQL performance](../configuration/postgresql.md) document, for tips on PostgreSQL tuning. From 9901e6367e760f5022397224aecf4aab31db10cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?nicole=20miko=C5=82ajczyk?= Date: Sun, 25 Jan 2026 19:47:21 +0100 Subject: [PATCH 23/90] Fix event type field name MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: nicole mikołajczyk --- lib/pleroma/web/admin_api/views/webhook_view.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/admin_api/views/webhook_view.ex b/lib/pleroma/web/admin_api/views/webhook_view.ex index 552aba4f1..0b839090b 100644 --- a/lib/pleroma/web/admin_api/views/webhook_view.ex +++ b/lib/pleroma/web/admin_api/views/webhook_view.ex @@ -26,7 +26,7 @@ defmodule Pleroma.Web.AdminAPI.WebhookView do def render("event.json", %{type: type, object: object}) do %{ - type: type, + event: type, created_at: Utils.to_masto_date(NaiveDateTime.utc_now()), object: object } From 8eca6ca87c0f3ee7482fe7211d9dd1bedf71c7f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?nicole=20miko=C5=82ajczyk?= Date: Fri, 27 Feb 2026 23:11:37 +0100 Subject: [PATCH 24/90] webhook fixes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: nicole mikołajczyk --- .../controllers/webhook_controller.ex | 6 ++-- .../mastodon_api/admin/views/account_view.ex | 2 +- lib/pleroma/webhook.ex | 30 ++++++++----------- 3 files changed, 18 insertions(+), 20 deletions(-) diff --git a/lib/pleroma/web/admin_api/controllers/webhook_controller.ex b/lib/pleroma/web/admin_api/controllers/webhook_controller.ex index dd9712a31..cc606916e 100644 --- a/lib/pleroma/web/admin_api/controllers/webhook_controller.ex +++ b/lib/pleroma/web/admin_api/controllers/webhook_controller.ex @@ -40,17 +40,18 @@ defmodule Pleroma.Web.AdminAPI.WebhookController do end def create(%{body_params: params} = conn, _) do - with webhook <- Webhook.create(params) do + with {:ok, webhook} <- Webhook.create(params) do render(conn, "show.json", webhook: webhook) end end def update(%{body_params: params} = conn, %{id: id}) do with %Webhook{internal: false} = webhook <- Webhook.get(id), - webhook <- Webhook.update(webhook, params) do + {:ok, webhook} <- Webhook.update(webhook, params) do render(conn, "show.json", webhook: webhook) else %Webhook{internal: true} -> {:error, :forbidden} + nil -> {:error, :not_found} end end @@ -60,6 +61,7 @@ defmodule Pleroma.Web.AdminAPI.WebhookController do render(conn, "show.json", webhook: webhook) else %Webhook{internal: true} -> {:error, :forbidden} + nil -> {:error, :not_found} end end diff --git a/lib/pleroma/web/mastodon_api/admin/views/account_view.ex b/lib/pleroma/web/mastodon_api/admin/views/account_view.ex index 40fac94ca..bf36d2c97 100644 --- a/lib/pleroma/web/mastodon_api/admin/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/admin/views/account_view.ex @@ -27,7 +27,7 @@ defmodule Pleroma.Web.MastodonAPI.Admin.AccountView do role: role(user), confirmed: user.is_confirmed, sensitized: nil, - suspened: nil, + suspended: nil, silenced: nil, disabled: !user.is_active, approved: user.is_approved, diff --git a/lib/pleroma/webhook.ex b/lib/pleroma/webhook.ex index f1e1b7678..83ce344fa 100644 --- a/lib/pleroma/webhook.ex +++ b/lib/pleroma/webhook.ex @@ -28,6 +28,7 @@ defmodule Pleroma.Webhook do def get_by_type(type) do __MODULE__ |> where([w], ^type in w.events) + |> where([w], w.enabled == true) |> Repo.all() end @@ -48,21 +49,15 @@ defmodule Pleroma.Webhook do end def create(params) do - {:ok, webhook} = - %__MODULE__{} - |> changeset(params) - |> Repo.insert() - - webhook + %__MODULE__{} + |> changeset(params) + |> Repo.insert() end def update(%__MODULE__{} = webhook, params) do - {:ok, webhook} = - webhook - |> update_changeset(params) - |> Repo.update() - webhook + |> update_changeset(params) + |> Repo.update() end def delete(webhook), do: webhook |> Repo.delete() @@ -80,12 +75,13 @@ defmodule Pleroma.Webhook do |> Repo.update() end - defp strip_events(params) do - if Map.has_key?(params, :events) do - params - |> Map.put(:events, Enum.filter(params[:events], &Enum.member?(@event_types, &1))) - else - params + defp strip_events(changeset) do + case get_change(changeset, :events) do + nil -> + changeset + + events -> + put_change(changeset, :events, Enum.filter(events, &(&1 in @event_types))) end end From e4f2f0994c395933b6f3590e60ec2743d531b1b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?nicole=20miko=C5=82ajczyk?= Date: Fri, 27 Feb 2026 23:13:15 +0100 Subject: [PATCH 25/90] Use encode16(case: :lower) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: nicole mikołajczyk --- lib/pleroma/webhook/notify.ex | 2 +- test/pleroma/webhook/notify_test.exs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/webhook/notify.ex b/lib/pleroma/webhook/notify.ex index 7dbfc52a0..743bb4012 100644 --- a/lib/pleroma/webhook/notify.ex +++ b/lib/pleroma/webhook/notify.ex @@ -78,6 +78,6 @@ defmodule Pleroma.Webhook.Notify do end defp signature(body, secret) do - :crypto.mac(:hmac, :sha256, secret, body) |> Base.encode16() + :crypto.mac(:hmac, :sha256, secret, body) |> Base.encode16(case: :lower) end end diff --git a/test/pleroma/webhook/notify_test.exs b/test/pleroma/webhook/notify_test.exs index e03890aab..c99489940 100644 --- a/test/pleroma/webhook/notify_test.exs +++ b/test/pleroma/webhook/notify_test.exs @@ -66,7 +66,7 @@ defmodule Pleroma.Webhook.NotifyTest do {"X-Hub-Signature", "sha256=" <> signature} = Enum.find(headers, fn {key, _} -> key == "X-Hub-Signature" end) - assert signature == :crypto.mac(:hmac, :sha256, secret, body) |> Base.encode16() + assert signature == :crypto.mac(:hmac, :sha256, secret, body) |> Base.encode16(case: :lower) %Tesla.Env{status: 200, body: ""} end) From 3a87250e15f46b276171f93ba99b8475aa483ebf Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 3 Mar 2026 12:17:34 +0400 Subject: [PATCH 26/90] Add ParadeDB external search backend Introduce an optional search backend that indexes public/unlisted Notes into a separate ParadeDB (pg_search) Postgres and queries it for status search results. --- changelog.d/paradedb-search.add | 1 + config/config.exs | 8 + config/description.exs | 29 +++- docs/configuration/search.md | 34 ++++ lib/pleroma/application.ex | 23 ++- lib/pleroma/search/paradedb.ex | 213 ++++++++++++++++++++++++++ lib/pleroma/search/paradedb/client.ex | 24 +++ lib/pleroma/search/paradedb/repo.ex | 24 +++ test/pleroma/search/paradedb_test.exs | 168 ++++++++++++++++++++ test/support/mocks.ex | 2 + 10 files changed, 520 insertions(+), 6 deletions(-) create mode 100644 changelog.d/paradedb-search.add create mode 100644 lib/pleroma/search/paradedb.ex create mode 100644 lib/pleroma/search/paradedb/client.ex create mode 100644 lib/pleroma/search/paradedb/repo.ex create mode 100644 test/pleroma/search/paradedb_test.exs diff --git a/changelog.d/paradedb-search.add b/changelog.d/paradedb-search.add new file mode 100644 index 000000000..00a6e2361 --- /dev/null +++ b/changelog.d/paradedb-search.add @@ -0,0 +1 @@ +Add ParadeDB (pg_search) as an optional external search backend. diff --git a/config/config.exs b/config/config.exs index 683805fe3..7e374b22b 100644 --- a/config/config.exs +++ b/config/config.exs @@ -938,6 +938,14 @@ config :pleroma, Pleroma.Search.Meilisearch, private_key: nil, initial_indexing_chunk_size: 100_000 +config :pleroma, Pleroma.Search.ParadeDB, + url: nil, + table: "pleroma_search_documents" + +config :pleroma, Pleroma.Search.ParadeDB.Repo, + pool_size: 10, + prepare: :unnamed + config :pleroma, Pleroma.Application, background_migrators: true, internal_fetch: true, diff --git a/config/description.exs b/config/description.exs index c388d17c3..de454dfb2 100644 --- a/config/description.exs +++ b/config/description.exs @@ -3503,7 +3503,12 @@ config :pleroma, :config_description, [ key: :module, type: :keyword, description: "Selected search module.", - suggestion: [Pleroma.Search.DatabaseSearch, Pleroma.Search.Meilisearch] + suggestion: [ + Pleroma.Search.DatabaseSearch, + Pleroma.Search.Meilisearch, + Pleroma.Search.QdrantSearch, + Pleroma.Search.ParadeDB + ] } ] }, @@ -3536,6 +3541,28 @@ config :pleroma, :config_description, [ } ] }, + %{ + group: :pleroma, + key: Pleroma.Search.ParadeDB, + type: :group, + description: "ParadeDB settings.", + children: [ + %{ + key: :url, + type: :string, + description: + "ParadeDB database URL (separate Postgres instance). " <> + "Can also be set via `PARADEDB_DATABASE_URL`.", + suggestion: ["postgres://postgres:postgres@127.0.0.1:5432/paradedb"] + }, + %{ + key: :table, + type: :string, + description: "Table name used for search documents.", + suggestion: ["pleroma_search_documents"] + } + ] + }, %{ group: :pleroma, key: Pleroma.Language.LanguageDetector, diff --git a/docs/configuration/search.md b/docs/configuration/search.md index d34f84d4f..780904d10 100644 --- a/docs/configuration/search.md +++ b/docs/configuration/search.md @@ -145,3 +145,37 @@ This will clear **all** the posts from the search index. Note, that deleted post there is no need to actually clear the whole index, unless you want **all** of it gone. That said, the index does not hold any information that cannot be re-created from the database, it should also generally be a lot smaller than the size of your database. Still, the size depends on the amount of text in posts. + +## ParadeDB + +[ParadeDB](https://www.paradedb.com/) is a Postgres extension that provides BM25 full-text search. In Pleroma, it can be used as an +external search backend by pointing Pleroma at a separate Postgres instance with the `pg_search` extension installed. + +Pleroma will maintain a small `pleroma_search_documents` table in that database (via the existing search indexing queue) and run search +queries against it. + +To use it, set the search module to `Pleroma.Search.ParadeDB` and configure the ParadeDB database URL: + +> config :pleroma, Pleroma.Search, module: Pleroma.Search.ParadeDB +> +> config :pleroma, Pleroma.Search.ParadeDB, +> url: System.get_env("PARADEDB_DATABASE_URL") || "postgres://postgres:postgres@127.0.0.1:5432/paradedb", +> table: "pleroma_search_documents" + +Then, create the table and BM25 index once: + +=== "From Source" + ```sh + mix pleroma.search.indexer create_index + ``` + +After that, run an initial indexing pass to backfill existing posts: + +=== "From Source" + ```sh + mix pleroma.search.indexer index + ``` + +Note: Like the other external search backends, only public/unlisted Notes are indexed. + +Indexing runs via the Oban `search_indexing` queue, so ensure that queue is enabled (not paused) in your Oban configuration. diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 8e1c5de0d..8de71b816 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -80,12 +80,15 @@ defmodule Pleroma.Application do [ Pleroma.PromEx, Pleroma.LDAP, - Pleroma.Repo, - Config.TransferTask, - Pleroma.Emoji, - Pleroma.Web.Plugs.RateLimiter.Supervisor, - {Task.Supervisor, name: Pleroma.TaskSupervisor} + Pleroma.Repo ] ++ + search_children() ++ + [ + Config.TransferTask, + Pleroma.Emoji, + Pleroma.Web.Plugs.RateLimiter.Supervisor, + {Task.Supervisor, name: Pleroma.TaskSupervisor} + ] ++ cachex_children() ++ http_children(adapter) ++ [ @@ -159,6 +162,16 @@ defmodule Pleroma.Application do ] end + defp search_children do + case Config.get([Pleroma.Search, :module]) do + Pleroma.Search.ParadeDB -> + [Pleroma.Search.ParadeDB.Repo] + + _ -> + [] + end + end + defp emoji_packs_expiration, do: expiration(default: :timer.seconds(5 * 60), interval: :timer.seconds(60)) diff --git a/lib/pleroma/search/paradedb.ex b/lib/pleroma/search/paradedb.ex new file mode 100644 index 000000000..963d573e8 --- /dev/null +++ b/lib/pleroma/search/paradedb.ex @@ -0,0 +1,213 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Search.ParadeDB do + @behaviour Pleroma.Search.SearchBackend + + require Logger + + import Ecto.Query + + import Pleroma.Search.DatabaseSearch, + only: [ + maybe_fetch: 3, + maybe_restrict_author: 2, + maybe_restrict_blocked: 2, + maybe_restrict_local: 2 + ] + + import Pleroma.Search.Meilisearch, only: [object_to_search_data: 1] + + alias Pleroma.Activity + alias Pleroma.Config.Getting, as: Config + + @max_limit 40 + @default_table "pleroma_search_documents" + + defp client_impl do + Config.get([Pleroma.Search.ParadeDB, :client_impl]) || + Pleroma.Search.ParadeDB.Client.Postgres + end + + defp table do + Config.get([Pleroma.Search.ParadeDB, :table], @default_table) + end + + @impl true + def create_index do + with {:ok, _} <- client_impl().query("CREATE EXTENSION IF NOT EXISTS pg_search", []), + {:ok, _} <- client_impl().query(create_table_sql(table()), []), + {:ok, _} <- client_impl().query(create_object_id_index_sql(table()), []), + {:ok, _} <- client_impl().query(create_bm25_index_sql(table()), []) do + :ok + else + {:error, error} -> + Logger.error("ParadeDB create_index failed: #{inspect(error)}") + {:error, error} + end + end + + @impl true + def drop_index do + with {:ok, _} <- client_impl().query("DROP TABLE IF EXISTS #{table()}", []) do + :ok + else + {:error, error} -> + Logger.error("ParadeDB drop_index failed: #{inspect(error)}") + {:error, error} + end + end + + @impl true + def healthcheck_endpoints, do: nil + + @impl true + def add_to_index(activity) do + maybe_search_data = object_to_search_data(activity.object) + + if activity.data["type"] == "Create" and maybe_search_data do + actor_ap_id = activity.data["actor"] + published_at = DateTime.from_unix!(maybe_search_data.published) + + sql = + """ + INSERT INTO #{table()} (id, object_id, object_ap_id, actor_ap_id, content, published_at) + VALUES ($1, $2, $3, $4, $5, $6) + ON CONFLICT (id) DO UPDATE SET + object_id = EXCLUDED.object_id, + object_ap_id = EXCLUDED.object_ap_id, + actor_ap_id = EXCLUDED.actor_ap_id, + content = EXCLUDED.content, + published_at = EXCLUDED.published_at + """ + + params = [ + activity.id, + maybe_search_data.id, + maybe_search_data.ap, + actor_ap_id, + maybe_search_data.content, + published_at + ] + + case client_impl().query(sql, params) do + {:ok, _} -> + :ok + + {:error, error} -> + Logger.error("ParadeDB add_to_index failed: #{inspect(error)}") + {:error, error} + end + else + :ok + end + end + + @impl true + def remove_from_index(object) do + sql = "DELETE FROM #{table()} WHERE object_id = $1" + + case client_impl().query(sql, [object.id]) do + {:ok, _} -> + :ok + + {:error, error} -> + Logger.error("ParadeDB remove_from_index failed: #{inspect(error)}") + {:error, error} + end + end + + @impl true + def search(user, query, options \\ []) do + limit = options |> Keyword.get(:limit, @max_limit) |> min(@max_limit) + offset = Keyword.get(options, :offset, 0) + author = Keyword.get(options, :author) + + case search_ids(query, limit, offset, author) do + {:ok, []} -> + maybe_fetch([], user, query) + + {:ok, ids} -> + ids + |> fetch_activities(user, query, author) + + {:error, error} -> + Logger.error("ParadeDB search failed: #{inspect(error)}") + maybe_fetch([], user, query) + end + end + + defp fetch_activities(ids, user, query, author) do + dumped_ids = Enum.map(ids, &FlakeId.from_string/1) + + from(a in Activity, where: a.id in ^dumped_ids) + |> Activity.with_preloaded_object() + |> Activity.restrict_deactivated_users() + |> maybe_restrict_local(user) + |> maybe_restrict_author(author) + |> maybe_restrict_blocked(user) + |> order_by( + [a], + fragment( + "array_position(?, ?)", + ^dumped_ids, + a.id + ) + ) + |> Pleroma.Repo.all() + |> maybe_fetch(user, query) + end + + defp search_ids(query, limit, offset, author) do + base_sql = "SELECT id FROM #{table()} WHERE content &&& $1" + params = [query] + + {base_sql, params} = + case author do + %Pleroma.User{ap_id: ap_id} when is_binary(ap_id) -> + {base_sql <> " AND actor_ap_id = $2", params ++ [ap_id]} + + _ -> + {base_sql, params} + end + + limit_arg = length(params) + 1 + offset_arg = length(params) + 2 + + sql = + base_sql <> + " ORDER BY published_at DESC LIMIT $#{limit_arg} OFFSET $#{offset_arg}" + + params = params ++ [limit, offset] + + with {:ok, %{rows: rows}} <- client_impl().query(sql, params) do + {:ok, Enum.map(rows, fn [id] -> id end)} + end + end + + defp create_table_sql(table) do + """ + CREATE TABLE IF NOT EXISTS #{table} ( + id text PRIMARY KEY, + object_id bigint NOT NULL, + object_ap_id text NOT NULL, + actor_ap_id text, + content text NOT NULL, + published_at timestamptz + ) + """ + end + + defp create_object_id_index_sql(table) do + "CREATE INDEX IF NOT EXISTS #{table}_object_id_idx ON #{table} (object_id)" + end + + defp create_bm25_index_sql(table) do + """ + CREATE INDEX IF NOT EXISTS #{table}_bm25_idx ON #{table} + USING bm25 (id, content, (actor_ap_id::pdb.literal), published_at) + WITH (key_field = 'id') + """ + end +end diff --git a/lib/pleroma/search/paradedb/client.ex b/lib/pleroma/search/paradedb/client.ex new file mode 100644 index 000000000..d6dce0dd4 --- /dev/null +++ b/lib/pleroma/search/paradedb/client.ex @@ -0,0 +1,24 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Search.ParadeDB.Client do + @moduledoc false + + @type sql :: String.t() + @type params :: list() + @type result :: %{optional(:rows) => list(list())} + + @callback query(sql(), params()) :: {:ok, result()} | {:error, any()} +end + +defmodule Pleroma.Search.ParadeDB.Client.Postgres do + @moduledoc false + + @behaviour Pleroma.Search.ParadeDB.Client + + @impl true + def query(sql, params \\ []) do + Ecto.Adapters.SQL.query(Pleroma.Search.ParadeDB.Repo, sql, params) + end +end diff --git a/lib/pleroma/search/paradedb/repo.ex b/lib/pleroma/search/paradedb/repo.ex new file mode 100644 index 000000000..684271847 --- /dev/null +++ b/lib/pleroma/search/paradedb/repo.ex @@ -0,0 +1,24 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Search.ParadeDB.Repo do + use Ecto.Repo, + otp_app: :pleroma, + adapter: Ecto.Adapters.Postgres, + migration_timestamps: [type: :naive_datetime_usec] + + @doc """ + Loads the ParadeDB database URL from `PARADEDB_DATABASE_URL`. + + If the env var is not set, falls back to `config :pleroma, Pleroma.Search.ParadeDB, url: ...`. + """ + def init(_, opts) do + config = Application.get_env(:pleroma, Pleroma.Search.ParadeDB, []) + + url = System.get_env("PARADEDB_DATABASE_URL") || config[:url] + url = if is_nil(url), do: nil, else: to_string(url) + + {:ok, Keyword.put(opts, :url, url)} + end +end diff --git a/test/pleroma/search/paradedb_test.exs b/test/pleroma/search/paradedb_test.exs new file mode 100644 index 000000000..c55e2aa47 --- /dev/null +++ b/test/pleroma/search/paradedb_test.exs @@ -0,0 +1,168 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Search.ParadeDBTest do + use Pleroma.DataCase, async: true + use Oban.Testing, repo: Pleroma.Repo + + import Mox + import Pleroma.Factory + + alias Pleroma.Search.ParadeDB + alias Pleroma.Search.ParadeDB.ClientMock + alias Pleroma.UnstubbedConfigMock, as: Config + alias Pleroma.Web.CommonAPI + alias Pleroma.Workers.SearchIndexingWorker + + describe "ParadeDB" do + test "indexes a public post on creation" do + user = insert(:user) + + {:ok, activity} = + CommonAPI.post(user, %{ + status: "guys i just don't wanna leave the swamp", + visibility: "public" + }) + + args = %{"op" => "add_to_index", "activity" => activity.id} + + assert_enqueued(worker: SearchIndexingWorker, args: args) + + ClientMock + |> expect(:query, fn sql, params -> + assert sql =~ "INSERT INTO pleroma_search_documents" + + [activity_id, object_id, object_ap_id, actor_ap_id, content, published_at] = params + + assert activity_id == activity.id + assert is_integer(object_id) + assert is_binary(object_ap_id) + assert actor_ap_id == activity.data["actor"] + assert content == "guys i just don't wanna leave the swamp" + assert %DateTime{} = published_at + + send(self(), :inserted) + {:ok, %{}} + end) + + Config + |> expect(:get, 3, fn + [Pleroma.Search, :module], nil -> + ParadeDB + + [Pleroma.Search.ParadeDB, :client_impl], nil -> + ClientMock + + [Pleroma.Search.ParadeDB, :table], "pleroma_search_documents" -> + "pleroma_search_documents" + end) + + assert :ok = perform_job(SearchIndexingWorker, args) + assert_received(:inserted) + end + + test "doesn't index posts that are not public" do + user = insert(:user) + + Enum.each(["private", "direct"], fn visibility -> + {:ok, activity} = + CommonAPI.post(user, %{ + status: "guys i just don't wanna leave the swamp", + visibility: visibility + }) + + args = %{"op" => "add_to_index", "activity" => activity.id} + assert_enqueued(worker: SearchIndexingWorker, args: args) + + Config + |> expect(:get, fn + [Pleroma.Search, :module], nil -> + ParadeDB + end) + + assert :ok = perform_job(SearchIndexingWorker, args) + end) + end + + test "deletes posts from index when deleted locally" do + user = insert(:user) + + ClientMock + |> expect(:query, 2, fn sql, params -> + cond do + String.contains?(sql, "INSERT INTO pleroma_search_documents") -> + send(self(), :inserted) + {:ok, %{}} + + String.contains?(sql, "DELETE FROM pleroma_search_documents") -> + assert [object_id] = params + assert is_integer(object_id) + + send(self(), :deleted) + {:ok, %{}} + + true -> + flunk("Unexpected ParadeDB SQL: #{inspect(sql)}") + end + end) + + Config + |> expect(:get, 6, fn + [Pleroma.Search, :module], nil -> + ParadeDB + + [Pleroma.Search.ParadeDB, :client_impl], nil -> + ClientMock + + [Pleroma.Search.ParadeDB, :table], "pleroma_search_documents" -> + "pleroma_search_documents" + end) + + {:ok, activity} = + CommonAPI.post(user, %{ + status: "guys i just don't wanna leave the swamp", + visibility: "public" + }) + + args = %{"op" => "add_to_index", "activity" => activity.id} + assert_enqueued(worker: SearchIndexingWorker, args: args) + assert :ok = perform_job(SearchIndexingWorker, args) + assert_received(:inserted) + + {:ok, _} = CommonAPI.delete(activity.id, user) + + delete_args = %{"op" => "remove_from_index", "object" => activity.object.id} + assert_enqueued(worker: SearchIndexingWorker, args: delete_args) + assert :ok = perform_job(SearchIndexingWorker, delete_args) + assert_received(:deleted) + end + + test "search returns activities in backend order" do + user = insert(:user) + + {:ok, activity1} = CommonAPI.post(user, %{status: "first swamp", visibility: "public"}) + {:ok, activity2} = CommonAPI.post(user, %{status: "second swamp", visibility: "public"}) + + ClientMock + |> expect(:query, fn sql, params -> + assert sql =~ "SELECT id FROM pleroma_search_documents" + assert ["swamp", _limit, _offset] = params + + {:ok, %{rows: [[activity2.id], [activity1.id]]}} + end) + + Config + |> expect(:get, 2, fn + [Pleroma.Search.ParadeDB, :client_impl], nil -> + ClientMock + + [Pleroma.Search.ParadeDB, :table], "pleroma_search_documents" -> + "pleroma_search_documents" + end) + + assert [%{id: ^activity2.id}, %{id: ^activity1.id}] = + ParadeDB.search(nil, "swamp", limit: 40, offset: 0) + end + end +end diff --git a/test/support/mocks.ex b/test/support/mocks.ex index 7e1244220..84465509a 100644 --- a/test/support/mocks.ex +++ b/test/support/mocks.ex @@ -46,3 +46,5 @@ Mox.defmock(Pleroma.SignatureMock, for: Pleroma.Signature.API) Mox.defmock(Pleroma.Web.ActivityPub.TransmogrifierMock, for: Pleroma.Web.ActivityPub.Transmogrifier.API ) + +Mox.defmock(Pleroma.Search.ParadeDB.ClientMock, for: Pleroma.Search.ParadeDB.Client) From 911d84411dc4eabe8bbca8d90cb898ca9f52f4bd Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 3 Mar 2026 12:36:01 +0400 Subject: [PATCH 27/90] Add ParadeDB docker integration test Add a skipped-by-default integration test that boots a ParadeDB container, creates the BM25 index, indexes posts, and verifies search results against the real backend. --- .../search/paradedb_integration_test.exs | 209 ++++++++++++++++++ test/test_helper.exs | 2 +- 2 files changed, 210 insertions(+), 1 deletion(-) create mode 100644 test/pleroma/search/paradedb_integration_test.exs diff --git a/test/pleroma/search/paradedb_integration_test.exs b/test/pleroma/search/paradedb_integration_test.exs new file mode 100644 index 000000000..04e6a9d59 --- /dev/null +++ b/test/pleroma/search/paradedb_integration_test.exs @@ -0,0 +1,209 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Search.ParadeDBIntegrationTest do + use Pleroma.DataCase, async: false + + @moduletag :integration + @moduletag timeout: 180_000 + + import Mox + import Pleroma.Factory + + alias Pleroma.Activity + alias Pleroma.Search.ParadeDB + alias Pleroma.Web.CommonAPI + + @image "paradedb/paradedb:latest" + @db_user "postgres" + @db_password "postgres" + @db_name "paradedb" + @docker_startup_timeout_ms 60_000 + + setup do + Mox.stub_with(Pleroma.UnstubbedConfigMock, Pleroma.Config) + :ok + end + + setup_all do + with :ok <- ensure_docker_available() do + container = "pleroma-paradedb-it-#{System.unique_integer([:positive])}" + + on_exit(fn -> + _ = docker(["rm", "-f", container]) + end) + + {id, 0} = + docker([ + "run", + "--rm", + "--name", + container, + "-e", + "POSTGRES_USER=#{@db_user}", + "-e", + "POSTGRES_PASSWORD=#{@db_password}", + "-e", + "POSTGRES_DB=#{@db_name}", + "-p", + "5432", + "-d", + @image + ]) + + container_id = String.trim(id) + + host_port = + container + |> wait_for_port!(@docker_startup_timeout_ms) + + wait_for_pg_ready!(container, @docker_startup_timeout_ms) + + url = "postgres://#{@db_user}:#{@db_password}@127.0.0.1:#{host_port}/#{@db_name}" + + prev_env = System.get_env("PARADEDB_DATABASE_URL") + System.put_env("PARADEDB_DATABASE_URL", url) + + on_exit(fn -> + if is_nil(prev_env) do + System.delete_env("PARADEDB_DATABASE_URL") + else + System.put_env("PARADEDB_DATABASE_URL", prev_env) + end + end) + + {:ok, _pid} = start_supervised(Pleroma.Search.ParadeDB.Repo) + + table = "pleroma_search_documents_it_#{System.unique_integer([:positive])}" + Pleroma.Config.put([Pleroma.Search.ParadeDB, :table], table) + + {:ok, + %{ + container: container, + container_id: container_id, + url: url, + table: table + }} + end + end + + test "indexes and searches against real ParadeDB" do + :ok = ParadeDB.drop_index() + assert :ok = ParadeDB.create_index() + + user = insert(:user) + + {:ok, a1} = + CommonAPI.post(user, %{status: "paradedb integration first", visibility: "public"}) + + :timer.sleep(1100) + + {:ok, a2} = + CommonAPI.post(user, %{status: "paradedb integration second", visibility: "public"}) + + {:ok, private} = + CommonAPI.post(user, %{status: "paradedb integration private", visibility: "private"}) + + a1 = Activity.get_by_id_with_object(a1.id) + a2 = Activity.get_by_id_with_object(a2.id) + private = Activity.get_by_id_with_object(private.id) + + assert :ok = ParadeDB.add_to_index(a1) + assert :ok = ParadeDB.add_to_index(a2) + assert :ok = ParadeDB.add_to_index(private) + + results = ParadeDB.search(nil, "paradedb integration", limit: 40, offset: 0) + ids = Enum.map(results, & &1.id) + + assert a1.id in ids + assert a2.id in ids + refute private.id in ids + assert List.first(ids) == a2.id + + # Removing an object should remove it from the ParadeDB index. + assert :ok = ParadeDB.remove_from_index(a2.object) + + results_after_delete = ParadeDB.search(nil, "paradedb integration", limit: 40, offset: 0) + ids_after_delete = Enum.map(results_after_delete, & &1.id) + + assert a1.id in ids_after_delete + refute a2.id in ids_after_delete + end + + defp ensure_docker_available do + if is_nil(System.find_executable("docker")) do + {:skip, "Docker not installed"} + else + case docker(["info"]) do + {_out, 0} -> :ok + {out, _} -> {:skip, "Docker not available: #{String.trim(out)}"} + end + end + end + + defp wait_for_port!(container, timeout_ms) do + deadline = System.monotonic_time(:millisecond) + timeout_ms + + wait = fn wait -> + case docker(["port", container, "5432/tcp"]) do + {out, 0} when byte_size(out) > 0 -> + out + |> String.split("\n", trim: true) + |> List.first() + |> parse_host_port!() + + _ -> + if System.monotonic_time(:millisecond) > deadline do + raise "Timed out waiting for docker port mapping" + else + :timer.sleep(250) + wait.(wait) + end + end + end + + wait.(wait) + end + + defp parse_host_port!(mapping) do + mapping + |> String.trim() + |> String.split(":") + |> List.last() + |> String.to_integer() + end + + defp wait_for_pg_ready!(container, timeout_ms) do + deadline = System.monotonic_time(:millisecond) + timeout_ms + + wait = fn wait -> + case docker([ + "exec", + container, + "pg_isready", + "-U", + @db_user, + "-d", + @db_name + ]) do + {_out, 0} -> + :ok + + _ -> + if System.monotonic_time(:millisecond) > deadline do + raise "Timed out waiting for ParadeDB Postgres to become ready" + else + :timer.sleep(250) + wait.(wait) + end + end + end + + wait.(wait) + end + + defp docker(args) do + System.cmd("docker", args, stderr_to_stdout: true) + end +end diff --git a/test/test_helper.exs b/test/test_helper.exs index dc6c05a74..bbc00b7df 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -4,7 +4,7 @@ ExUnit.configure(capture_log: true, max_cases: System.schedulers_online()) -ExUnit.start(exclude: [:federated]) +ExUnit.start(exclude: [:federated, :integration]) if match?({:unix, :darwin}, :os.type()) do excluded = ExUnit.configuration() |> Keyword.get(:exclude, []) From efb75488d9f0f8cd8d9e629ec6c925420d7b174c Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 3 Mar 2026 12:49:23 +0400 Subject: [PATCH 28/90] Tighten ParadeDB integration test Remove warnings that break --warnings-as-errors, add TCP readiness check, and start the ParadeDB Repo with a small pool to reduce connection churn while booting the dockerized backend. --- .../search/paradedb_integration_test.exs | 26 +++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/test/pleroma/search/paradedb_integration_test.exs b/test/pleroma/search/paradedb_integration_test.exs index 04e6a9d59..1da131c6b 100644 --- a/test/pleroma/search/paradedb_integration_test.exs +++ b/test/pleroma/search/paradedb_integration_test.exs @@ -8,7 +8,6 @@ defmodule Pleroma.Search.ParadeDBIntegrationTest do @moduletag :integration @moduletag timeout: 180_000 - import Mox import Pleroma.Factory alias Pleroma.Activity @@ -59,6 +58,7 @@ defmodule Pleroma.Search.ParadeDBIntegrationTest do |> wait_for_port!(@docker_startup_timeout_ms) wait_for_pg_ready!(container, @docker_startup_timeout_ms) + wait_for_tcp_ready!(host_port, @docker_startup_timeout_ms) url = "postgres://#{@db_user}:#{@db_password}@127.0.0.1:#{host_port}/#{@db_name}" @@ -73,7 +73,7 @@ defmodule Pleroma.Search.ParadeDBIntegrationTest do end end) - {:ok, _pid} = start_supervised(Pleroma.Search.ParadeDB.Repo) + {:ok, _pid} = start_supervised({Pleroma.Search.ParadeDB.Repo, pool_size: 1}) table = "pleroma_search_documents_it_#{System.unique_integer([:positive])}" Pleroma.Config.put([Pleroma.Search.ParadeDB, :table], table) @@ -203,6 +203,28 @@ defmodule Pleroma.Search.ParadeDBIntegrationTest do wait.(wait) end + defp wait_for_tcp_ready!(port, timeout_ms) do + deadline = System.monotonic_time(:millisecond) + timeout_ms + + wait = fn wait -> + case :gen_tcp.connect(~c"127.0.0.1", port, [:binary, active: false], 500) do + {:ok, socket} -> + :gen_tcp.close(socket) + :ok + + {:error, _} -> + if System.monotonic_time(:millisecond) > deadline do + raise "Timed out waiting for TCP port #{port} to accept connections" + else + :timer.sleep(250) + wait.(wait) + end + end + end + + wait.(wait) + end + defp docker(args) do System.cmd("docker", args, stderr_to_stdout: true) end From dee7ac0415e0de1c1f2d44d1dcd1202d5c7d099c Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Wed, 4 Mar 2026 13:04:41 +0400 Subject: [PATCH 29/90] Fix ParadeDB key field handling Use uuid ids in the ParadeDB documents table to satisfy BM25 key_field requirements and adjust indexing/search to dump FlakeIds consistently. Also fix the ParadeDB backend unit test to avoid invalid pinning on struct fields. --- lib/pleroma/search/paradedb.ex | 74 +++++++++++++++++---------- test/pleroma/search/paradedb_test.exs | 14 +++-- 2 files changed, 58 insertions(+), 30 deletions(-) diff --git a/lib/pleroma/search/paradedb.ex b/lib/pleroma/search/paradedb.ex index 963d573e8..e5d3018d7 100644 --- a/lib/pleroma/search/paradedb.ex +++ b/lib/pleroma/search/paradedb.ex @@ -67,37 +67,43 @@ defmodule Pleroma.Search.ParadeDB do maybe_search_data = object_to_search_data(activity.object) if activity.data["type"] == "Create" and maybe_search_data do + dumped_activity_id = dump_activity_id(activity.id) actor_ap_id = activity.data["actor"] published_at = DateTime.from_unix!(maybe_search_data.published) - sql = - """ - INSERT INTO #{table()} (id, object_id, object_ap_id, actor_ap_id, content, published_at) - VALUES ($1, $2, $3, $4, $5, $6) - ON CONFLICT (id) DO UPDATE SET - object_id = EXCLUDED.object_id, - object_ap_id = EXCLUDED.object_ap_id, - actor_ap_id = EXCLUDED.actor_ap_id, - content = EXCLUDED.content, - published_at = EXCLUDED.published_at - """ + if is_nil(dumped_activity_id) do + Logger.error("ParadeDB add_to_index failed: invalid activity id #{inspect(activity.id)}") + :ok + else + sql = + """ + INSERT INTO #{table()} (id, object_id, object_ap_id, actor_ap_id, content, published_at) + VALUES ($1, $2, $3, $4, $5, $6) + ON CONFLICT (id) DO UPDATE SET + object_id = EXCLUDED.object_id, + object_ap_id = EXCLUDED.object_ap_id, + actor_ap_id = EXCLUDED.actor_ap_id, + content = EXCLUDED.content, + published_at = EXCLUDED.published_at + """ - params = [ - activity.id, - maybe_search_data.id, - maybe_search_data.ap, - actor_ap_id, - maybe_search_data.content, - published_at - ] + params = [ + dumped_activity_id, + maybe_search_data.id, + maybe_search_data.ap, + actor_ap_id, + maybe_search_data.content, + published_at + ] - case client_impl().query(sql, params) do - {:ok, _} -> - :ok + case client_impl().query(sql, params) do + {:ok, _} -> + :ok - {:error, error} -> - Logger.error("ParadeDB add_to_index failed: #{inspect(error)}") - {:error, error} + {:error, error} -> + Logger.error("ParadeDB add_to_index failed: #{inspect(error)}") + {:error, error} + end end else :ok @@ -139,7 +145,10 @@ defmodule Pleroma.Search.ParadeDB do end defp fetch_activities(ids, user, query, author) do - dumped_ids = Enum.map(ids, &FlakeId.from_string/1) + dumped_ids = + ids + |> Enum.map(&dump_activity_id/1) + |> Enum.reject(&is_nil/1) from(a in Activity, where: a.id in ^dumped_ids) |> Activity.with_preloaded_object() @@ -189,7 +198,7 @@ defmodule Pleroma.Search.ParadeDB do defp create_table_sql(table) do """ CREATE TABLE IF NOT EXISTS #{table} ( - id text PRIMARY KEY, + id uuid PRIMARY KEY, object_id bigint NOT NULL, object_ap_id text NOT NULL, actor_ap_id text, @@ -210,4 +219,15 @@ defmodule Pleroma.Search.ParadeDB do WITH (key_field = 'id') """ end + + defp dump_activity_id(<<_::binary-size(16)>> = uuid), do: uuid + + defp dump_activity_id(id) when is_binary(id) do + case FlakeId.Ecto.CompatType.dump(id) do + {:ok, <<_::binary-size(16)>> = uuid} -> uuid + _ -> nil + end + end + + defp dump_activity_id(_), do: nil end diff --git a/test/pleroma/search/paradedb_test.exs b/test/pleroma/search/paradedb_test.exs index c55e2aa47..3338c78ef 100644 --- a/test/pleroma/search/paradedb_test.exs +++ b/test/pleroma/search/paradedb_test.exs @@ -35,7 +35,9 @@ defmodule Pleroma.Search.ParadeDBTest do [activity_id, object_id, object_ap_id, actor_ap_id, content, published_at] = params - assert activity_id == activity.id + {:ok, dumped_activity_id} = FlakeId.Ecto.CompatType.dump(activity.id) + + assert activity_id == dumped_activity_id assert is_integer(object_id) assert is_binary(object_ap_id) assert actor_ap_id == activity.data["actor"] @@ -144,12 +146,18 @@ defmodule Pleroma.Search.ParadeDBTest do {:ok, activity1} = CommonAPI.post(user, %{status: "first swamp", visibility: "public"}) {:ok, activity2} = CommonAPI.post(user, %{status: "second swamp", visibility: "public"}) + activity1_id = activity1.id + activity2_id = activity2.id + + {:ok, dumped_activity1_id} = FlakeId.Ecto.CompatType.dump(activity1_id) + {:ok, dumped_activity2_id} = FlakeId.Ecto.CompatType.dump(activity2_id) + ClientMock |> expect(:query, fn sql, params -> assert sql =~ "SELECT id FROM pleroma_search_documents" assert ["swamp", _limit, _offset] = params - {:ok, %{rows: [[activity2.id], [activity1.id]]}} + {:ok, %{rows: [[dumped_activity2_id], [dumped_activity1_id]]}} end) Config @@ -161,7 +169,7 @@ defmodule Pleroma.Search.ParadeDBTest do "pleroma_search_documents" end) - assert [%{id: ^activity2.id}, %{id: ^activity1.id}] = + assert [%{id: ^activity2_id}, %{id: ^activity1_id}] = ParadeDB.search(nil, "swamp", limit: 40, offset: 0) end end From 74da3e01f7f0b27fcbbec313db363118c51388a8 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Thu, 5 Mar 2026 16:36:22 +0400 Subject: [PATCH 30/90] Harden ParadeDB search result safety Re-apply public Create visibility checks when resolving backend hits, improve UUID/FlakeId id parsing, and add tests for poisoned hits and UUID-string ids. Also tighten the docker integration test with localhost-only binding and config restoration. --- lib/pleroma/search/paradedb.ex | 41 ++++++++++-- .../search/paradedb_integration_test.exs | 13 +++- test/pleroma/search/paradedb_test.exs | 63 +++++++++++++++++++ 3 files changed, 111 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/search/paradedb.ex b/lib/pleroma/search/paradedb.ex index e5d3018d7..01cbc323f 100644 --- a/lib/pleroma/search/paradedb.ex +++ b/lib/pleroma/search/paradedb.ex @@ -6,6 +6,7 @@ defmodule Pleroma.Search.ParadeDB do @behaviour Pleroma.Search.SearchBackend require Logger + require Pleroma.Constants import Ecto.Query @@ -153,6 +154,7 @@ defmodule Pleroma.Search.ParadeDB do from(a in Activity, where: a.id in ^dumped_ids) |> Activity.with_preloaded_object() |> Activity.restrict_deactivated_users() + |> maybe_restrict_public(user) |> maybe_restrict_local(user) |> maybe_restrict_author(author) |> maybe_restrict_blocked(user) @@ -168,6 +170,25 @@ defmodule Pleroma.Search.ParadeDB do |> maybe_fetch(user, query) end + defp maybe_restrict_public(query, %Pleroma.User{}) do + intended_recipients = [ + Pleroma.Constants.as_public(), + Pleroma.Web.ActivityPub.Utils.as_local_public() + ] + + from(a in query, + where: fragment("?->>'type' = 'Create'", a.data), + where: fragment("? && ?", ^intended_recipients, a.recipients) + ) + end + + defp maybe_restrict_public(query, _user) do + from(a in query, + where: fragment("?->>'type' = 'Create'", a.data), + where: ^Pleroma.Constants.as_public() in a.recipients + ) + end + defp search_ids(query, limit, offset, author) do base_sql = "SELECT id FROM #{table()} WHERE content &&& $1" params = [query] @@ -186,7 +207,7 @@ defmodule Pleroma.Search.ParadeDB do sql = base_sql <> - " ORDER BY published_at DESC LIMIT $#{limit_arg} OFFSET $#{offset_arg}" + " ORDER BY published_at DESC, id DESC LIMIT $#{limit_arg} OFFSET $#{offset_arg}" params = params ++ [limit, offset] @@ -223,11 +244,23 @@ defmodule Pleroma.Search.ParadeDB do defp dump_activity_id(<<_::binary-size(16)>> = uuid), do: uuid defp dump_activity_id(id) when is_binary(id) do - case FlakeId.Ecto.CompatType.dump(id) do - {:ok, <<_::binary-size(16)>> = uuid} -> uuid - _ -> nil + case Ecto.UUID.dump(id) do + {:ok, <<_::binary-size(16)>> = uuid} -> + uuid + + :error -> + dump_flake_activity_id(id) end end defp dump_activity_id(_), do: nil + + defp dump_flake_activity_id(id) do + case FlakeId.Ecto.CompatType.dump(id) do + {:ok, <<_::binary-size(16)>> = uuid} -> uuid + _ -> nil + end + rescue + _ -> nil + end end diff --git a/test/pleroma/search/paradedb_integration_test.exs b/test/pleroma/search/paradedb_integration_test.exs index 1da131c6b..42d12c4d6 100644 --- a/test/pleroma/search/paradedb_integration_test.exs +++ b/test/pleroma/search/paradedb_integration_test.exs @@ -14,7 +14,7 @@ defmodule Pleroma.Search.ParadeDBIntegrationTest do alias Pleroma.Search.ParadeDB alias Pleroma.Web.CommonAPI - @image "paradedb/paradedb:latest" + @image System.get_env("PARADEDB_TEST_IMAGE", "paradedb/paradedb:latest") @db_user "postgres" @db_password "postgres" @db_name "paradedb" @@ -46,7 +46,7 @@ defmodule Pleroma.Search.ParadeDBIntegrationTest do "-e", "POSTGRES_DB=#{@db_name}", "-p", - "5432", + "127.0.0.1::5432", "-d", @image ]) @@ -75,9 +75,18 @@ defmodule Pleroma.Search.ParadeDBIntegrationTest do {:ok, _pid} = start_supervised({Pleroma.Search.ParadeDB.Repo, pool_size: 1}) + previous_table = Pleroma.Config.get([Pleroma.Search.ParadeDB, :table], nil) table = "pleroma_search_documents_it_#{System.unique_integer([:positive])}" Pleroma.Config.put([Pleroma.Search.ParadeDB, :table], table) + on_exit(fn -> + if is_nil(previous_table) do + Pleroma.Config.delete([Pleroma.Search.ParadeDB, :table]) + else + Pleroma.Config.put([Pleroma.Search.ParadeDB, :table], previous_table) + end + end) + {:ok, %{ container: container, diff --git a/test/pleroma/search/paradedb_test.exs b/test/pleroma/search/paradedb_test.exs index 3338c78ef..c9bb216df 100644 --- a/test/pleroma/search/paradedb_test.exs +++ b/test/pleroma/search/paradedb_test.exs @@ -172,5 +172,68 @@ defmodule Pleroma.Search.ParadeDBTest do assert [%{id: ^activity2_id}, %{id: ^activity1_id}] = ParadeDB.search(nil, "swamp", limit: 40, offset: 0) end + + test "search excludes non-public activities from backend hits" do + user = insert(:user) + + {:ok, public_activity} = + CommonAPI.post(user, %{status: "public swamp", visibility: "public"}) + + {:ok, private_activity} = + CommonAPI.post(user, %{status: "private swamp", visibility: "private"}) + + public_activity_id = public_activity.id + + {:ok, dumped_public_id} = FlakeId.Ecto.CompatType.dump(public_activity.id) + {:ok, dumped_private_id} = FlakeId.Ecto.CompatType.dump(private_activity.id) + + ClientMock + |> expect(:query, fn sql, params -> + assert sql =~ "SELECT id FROM pleroma_search_documents" + assert ["swamp", _limit, _offset] = params + + {:ok, %{rows: [[dumped_private_id], [dumped_public_id]]}} + end) + + Config + |> expect(:get, 2, fn + [Pleroma.Search.ParadeDB, :client_impl], nil -> + ClientMock + + [Pleroma.Search.ParadeDB, :table], "pleroma_search_documents" -> + "pleroma_search_documents" + end) + + assert [%{id: ^public_activity_id}] = ParadeDB.search(nil, "swamp", limit: 40, offset: 0) + end + + test "search supports backend ids returned as uuid strings" do + user = insert(:user) + + {:ok, activity} = CommonAPI.post(user, %{status: "uuid swamp", visibility: "public"}) + + activity_id = activity.id + {:ok, dumped_id} = FlakeId.Ecto.CompatType.dump(activity_id) + {:ok, uuid_string} = Ecto.UUID.load(dumped_id) + + ClientMock + |> expect(:query, fn sql, params -> + assert sql =~ "SELECT id FROM pleroma_search_documents" + assert ["swamp", _limit, _offset] = params + + {:ok, %{rows: [[uuid_string]]}} + end) + + Config + |> expect(:get, 2, fn + [Pleroma.Search.ParadeDB, :client_impl], nil -> + ClientMock + + [Pleroma.Search.ParadeDB, :table], "pleroma_search_documents" -> + "pleroma_search_documents" + end) + + assert [%{id: ^activity_id}] = ParadeDB.search(nil, "swamp", limit: 40, offset: 0) + end end end From 7efd296afbf49e5fb4ea8cdf659e1f0cdedd7de9 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Thu, 5 Mar 2026 17:12:03 +0400 Subject: [PATCH 31/90] Add fuzzy matching to ParadeDB search Make ParadeDB search typo-tolerant via configurable fuzzy distance and document the new setting. Add coverage for fuzzy query SQL and id parsing, and keep integration test stability under connection load. --- config/config.exs | 3 +- config/description.exs | 6 +++ docs/configuration/search.md | 5 +- lib/pleroma/search/paradedb.ex | 23 ++++++++- .../search/paradedb_integration_test.exs | 2 +- test/pleroma/search/paradedb_test.exs | 47 +++++++++++++++++-- 6 files changed, 79 insertions(+), 7 deletions(-) diff --git a/config/config.exs b/config/config.exs index 7e374b22b..01dd09a18 100644 --- a/config/config.exs +++ b/config/config.exs @@ -940,7 +940,8 @@ config :pleroma, Pleroma.Search.Meilisearch, config :pleroma, Pleroma.Search.ParadeDB, url: nil, - table: "pleroma_search_documents" + table: "pleroma_search_documents", + fuzzy_distance: 0 config :pleroma, Pleroma.Search.ParadeDB.Repo, pool_size: 10, diff --git a/config/description.exs b/config/description.exs index de454dfb2..9a00afdcd 100644 --- a/config/description.exs +++ b/config/description.exs @@ -3560,6 +3560,12 @@ config :pleroma, :config_description, [ type: :string, description: "Table name used for search documents.", suggestion: ["pleroma_search_documents"] + }, + %{ + key: :fuzzy_distance, + type: :integer, + description: "Edit-distance used for fuzzy token matching (0 disables fuzziness, max 2).", + suggestion: [0, 1, 2] } ] }, diff --git a/docs/configuration/search.md b/docs/configuration/search.md index 780904d10..c0d3f8462 100644 --- a/docs/configuration/search.md +++ b/docs/configuration/search.md @@ -160,7 +160,8 @@ To use it, set the search module to `Pleroma.Search.ParadeDB` and configure the > > config :pleroma, Pleroma.Search.ParadeDB, > url: System.get_env("PARADEDB_DATABASE_URL") || "postgres://postgres:postgres@127.0.0.1:5432/paradedb", -> table: "pleroma_search_documents" +> table: "pleroma_search_documents", +> fuzzy_distance: 0 Then, create the table and BM25 index once: @@ -178,4 +179,6 @@ After that, run an initial indexing pass to backfill existing posts: Note: Like the other external search backends, only public/unlisted Notes are indexed. +Set `fuzzy_distance` to `1` or `2` to allow typo-tolerant matching (`2` is the maximum ParadeDB supports). + Indexing runs via the Oban `search_indexing` queue, so ensure that queue is enabled (not paused) in your Oban configuration. diff --git a/lib/pleroma/search/paradedb.ex b/lib/pleroma/search/paradedb.ex index 01cbc323f..57e5599b5 100644 --- a/lib/pleroma/search/paradedb.ex +++ b/lib/pleroma/search/paradedb.ex @@ -35,6 +35,22 @@ defmodule Pleroma.Search.ParadeDB do Config.get([Pleroma.Search.ParadeDB, :table], @default_table) end + defp fuzzy_distance do + case Config.get([Pleroma.Search.ParadeDB, :fuzzy_distance], 0) do + value when is_integer(value) and value in 0..2 -> + value + + value when is_binary(value) -> + case Integer.parse(value) do + {parsed, ""} when parsed in 0..2 -> parsed + _ -> 0 + end + + _ -> + 0 + end + end + @impl true def create_index do with {:ok, _} <- client_impl().query("CREATE EXTENSION IF NOT EXISTS pg_search", []), @@ -190,7 +206,9 @@ defmodule Pleroma.Search.ParadeDB do end defp search_ids(query, limit, offset, author) do - base_sql = "SELECT id FROM #{table()} WHERE content &&& $1" + base_sql = + "SELECT id FROM #{table()} WHERE content &&& #{search_query_param_sql(fuzzy_distance())}" + params = [query] {base_sql, params} = @@ -216,6 +234,9 @@ defmodule Pleroma.Search.ParadeDB do end end + defp search_query_param_sql(0), do: "$1" + defp search_query_param_sql(distance), do: "$1::pdb.fuzzy(#{distance})" + defp create_table_sql(table) do """ CREATE TABLE IF NOT EXISTS #{table} ( diff --git a/test/pleroma/search/paradedb_integration_test.exs b/test/pleroma/search/paradedb_integration_test.exs index 42d12c4d6..743dcbaff 100644 --- a/test/pleroma/search/paradedb_integration_test.exs +++ b/test/pleroma/search/paradedb_integration_test.exs @@ -73,7 +73,7 @@ defmodule Pleroma.Search.ParadeDBIntegrationTest do end end) - {:ok, _pid} = start_supervised({Pleroma.Search.ParadeDB.Repo, pool_size: 1}) + {:ok, _pid} = start_supervised({Pleroma.Search.ParadeDB.Repo, pool_size: 5}) previous_table = Pleroma.Config.get([Pleroma.Search.ParadeDB, :table], nil) table = "pleroma_search_documents_it_#{System.unique_integer([:positive])}" diff --git a/test/pleroma/search/paradedb_test.exs b/test/pleroma/search/paradedb_test.exs index c9bb216df..c4fb57cb3 100644 --- a/test/pleroma/search/paradedb_test.exs +++ b/test/pleroma/search/paradedb_test.exs @@ -161,12 +161,15 @@ defmodule Pleroma.Search.ParadeDBTest do end) Config - |> expect(:get, 2, fn + |> expect(:get, 3, fn [Pleroma.Search.ParadeDB, :client_impl], nil -> ClientMock [Pleroma.Search.ParadeDB, :table], "pleroma_search_documents" -> "pleroma_search_documents" + + [Pleroma.Search.ParadeDB, :fuzzy_distance], 0 -> + 0 end) assert [%{id: ^activity2_id}, %{id: ^activity1_id}] = @@ -196,12 +199,15 @@ defmodule Pleroma.Search.ParadeDBTest do end) Config - |> expect(:get, 2, fn + |> expect(:get, 3, fn [Pleroma.Search.ParadeDB, :client_impl], nil -> ClientMock [Pleroma.Search.ParadeDB, :table], "pleroma_search_documents" -> "pleroma_search_documents" + + [Pleroma.Search.ParadeDB, :fuzzy_distance], 0 -> + 0 end) assert [%{id: ^public_activity_id}] = ParadeDB.search(nil, "swamp", limit: 40, offset: 0) @@ -225,15 +231,50 @@ defmodule Pleroma.Search.ParadeDBTest do end) Config - |> expect(:get, 2, fn + |> expect(:get, 3, fn [Pleroma.Search.ParadeDB, :client_impl], nil -> ClientMock [Pleroma.Search.ParadeDB, :table], "pleroma_search_documents" -> "pleroma_search_documents" + + [Pleroma.Search.ParadeDB, :fuzzy_distance], 0 -> + 0 end) assert [%{id: ^activity_id}] = ParadeDB.search(nil, "swamp", limit: 40, offset: 0) end + + test "search uses configured fuzzy distance" do + user = insert(:user) + + {:ok, activity} = CommonAPI.post(user, %{status: "runing shoes", visibility: "public"}) + + activity_id = activity.id + {:ok, dumped_id} = FlakeId.Ecto.CompatType.dump(activity_id) + + ClientMock + |> expect(:query, fn sql, params -> + assert sql =~ "SELECT id FROM pleroma_search_documents" + assert sql =~ "$1::pdb.fuzzy(1)" + assert ["running shoes", _limit, _offset] = params + + {:ok, %{rows: [[dumped_id]]}} + end) + + Config + |> expect(:get, 3, fn + [Pleroma.Search.ParadeDB, :client_impl], nil -> + ClientMock + + [Pleroma.Search.ParadeDB, :table], "pleroma_search_documents" -> + "pleroma_search_documents" + + [Pleroma.Search.ParadeDB, :fuzzy_distance], 0 -> + 1 + end) + + assert [%{id: ^activity_id}] = ParadeDB.search(nil, "running shoes", limit: 40, offset: 0) + end end end From 4e65bbf13469d8147dc780c7e655fefdb12bdebf Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Thu, 5 Mar 2026 17:34:48 +0400 Subject: [PATCH 32/90] Fix ParadeDB fuzzy query parameter casting Cast the bound search parameter through text before so Postgrex can handle the prepared statement type inference. Add matching test assertion updates. --- lib/pleroma/search/paradedb.ex | 2 +- test/pleroma/search/paradedb_test.exs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/search/paradedb.ex b/lib/pleroma/search/paradedb.ex index 57e5599b5..a6649a894 100644 --- a/lib/pleroma/search/paradedb.ex +++ b/lib/pleroma/search/paradedb.ex @@ -235,7 +235,7 @@ defmodule Pleroma.Search.ParadeDB do end defp search_query_param_sql(0), do: "$1" - defp search_query_param_sql(distance), do: "$1::pdb.fuzzy(#{distance})" + defp search_query_param_sql(distance), do: "($1::text)::pdb.fuzzy(#{distance})" defp create_table_sql(table) do """ diff --git a/test/pleroma/search/paradedb_test.exs b/test/pleroma/search/paradedb_test.exs index c4fb57cb3..3a3b8298f 100644 --- a/test/pleroma/search/paradedb_test.exs +++ b/test/pleroma/search/paradedb_test.exs @@ -256,7 +256,7 @@ defmodule Pleroma.Search.ParadeDBTest do ClientMock |> expect(:query, fn sql, params -> assert sql =~ "SELECT id FROM pleroma_search_documents" - assert sql =~ "$1::pdb.fuzzy(1)" + assert sql =~ "($1::text)::pdb.fuzzy(1)" assert ["running shoes", _limit, _offset] = params {:ok, %{rows: [[dumped_id]]}} From cae3c9a980945f107b2cd6a321b0d1d1e0d243ea Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Thu, 5 Mar 2026 17:39:34 +0400 Subject: [PATCH 33/90] Add ParadeDB fuzzy integration coverage Extend the docker-backed ParadeDB integration test to verify typo-tolerant status search with fuzzy_distance enabled, covering prepared-parameter fuzzy casts against a real backend. --- test/pleroma/search/paradedb_integration_test.exs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/test/pleroma/search/paradedb_integration_test.exs b/test/pleroma/search/paradedb_integration_test.exs index 743dcbaff..2687260cf 100644 --- a/test/pleroma/search/paradedb_integration_test.exs +++ b/test/pleroma/search/paradedb_integration_test.exs @@ -130,6 +130,20 @@ defmodule Pleroma.Search.ParadeDBIntegrationTest do refute private.id in ids assert List.first(ids) == a2.id + previous_fuzzy_distance = Pleroma.Config.get([Pleroma.Search.ParadeDB, :fuzzy_distance], 0) + Pleroma.Config.put([Pleroma.Search.ParadeDB, :fuzzy_distance], 1) + + on_exit(fn -> + Pleroma.Config.put([Pleroma.Search.ParadeDB, :fuzzy_distance], previous_fuzzy_distance) + end) + + fuzzy_results = ParadeDB.search(nil, "paradedb integraion", limit: 40, offset: 0) + fuzzy_ids = Enum.map(fuzzy_results, & &1.id) + + assert a1.id in fuzzy_ids + assert a2.id in fuzzy_ids + refute private.id in fuzzy_ids + # Removing an object should remove it from the ParadeDB index. assert :ok = ParadeDB.remove_from_index(a2.object) From 0e0931a3c23d8c966aef8ded80e82da6f3c4205e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?nicole=20miko=C5=82ajczyk?= Date: Fri, 19 Sep 2025 16:55:40 +0200 Subject: [PATCH 34/90] MastoAPI: /api/v1/admin/accounts, /api/v1/admin/reports MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: nicole mikołajczyk --- changelog.d/mastodon-admin.add | 1 + lib/pleroma/moderation_log.ex | 10 + lib/pleroma/user.ex | 6 + lib/pleroma/user/query.ex | 19 +- lib/pleroma/web/api_spec.ex | 7 + .../mastodon_admin/account_operation.ex | 258 ++++++++++++++++++ .../mastodon_admin/report_operation.ex | 150 ++++++++++ .../admin/controllers/account_controller.ex | 220 +++++++++++++++ .../admin/controllers/report_controller.ex | 107 ++++++++ .../mastodon_api/admin/views/account_view.ex | 64 +++++ .../mastodon_api/admin/views/report_view.ex | 56 ++++ lib/pleroma/web/router.ex | 40 +++ .../controllers/account_controller_test.exs | 175 ++++++++++++ .../controllers/report_controller_test.exs | 118 ++++++++ 14 files changed, 1230 insertions(+), 1 deletion(-) create mode 100644 changelog.d/mastodon-admin.add create mode 100644 lib/pleroma/web/api_spec/operations/mastodon_admin/account_operation.ex create mode 100644 lib/pleroma/web/api_spec/operations/mastodon_admin/report_operation.ex create mode 100644 lib/pleroma/web/mastodon_api/admin/controllers/account_controller.ex create mode 100644 lib/pleroma/web/mastodon_api/admin/controllers/report_controller.ex create mode 100644 lib/pleroma/web/mastodon_api/admin/views/account_view.ex create mode 100644 lib/pleroma/web/mastodon_api/admin/views/report_view.ex create mode 100644 test/pleroma/web/mastodon_api/admin/controllers/account_controller_test.exs create mode 100644 test/pleroma/web/mastodon_api/admin/controllers/report_controller_test.exs diff --git a/changelog.d/mastodon-admin.add b/changelog.d/mastodon-admin.add new file mode 100644 index 000000000..64f2ca987 --- /dev/null +++ b/changelog.d/mastodon-admin.add @@ -0,0 +1 @@ +Implement basics of Mastodon admin API \ No newline at end of file diff --git a/lib/pleroma/moderation_log.ex b/lib/pleroma/moderation_log.ex index 90219312c..260c139ec 100644 --- a/lib/pleroma/moderation_log.ex +++ b/lib/pleroma/moderation_log.ex @@ -346,6 +346,16 @@ defmodule Pleroma.ModerationLog do "@#{actor_nickname} approved users: #{users_to_nicknames_string(users)}" end + def get_log_entry_message(%ModerationLog{ + data: %{ + "actor" => %{"nickname" => actor_nickname}, + "action" => "reject", + "subject" => users + } + }) do + "@#{actor_nickname} rejected users: #{users_to_nicknames_string(users)}" + end + def get_log_entry_message(%ModerationLog{ data: %{ "actor" => %{"nickname" => actor_nickname}, diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 75da41da9..4b421f389 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -1964,6 +1964,12 @@ defmodule Pleroma.User do def approve(%User{} = user), do: {:ok, user} + def reject(%User{is_approved: false} = user) do + delete(user) + end + + def reject(%User{} = _user), do: {:error, "User is approved"} + def confirm(users) when is_list(users) do Repo.transaction(fn -> Enum.map(users, fn user -> diff --git a/lib/pleroma/user/query.ex b/lib/pleroma/user/query.ex index cd9586452..b877b52b2 100644 --- a/lib/pleroma/user/query.ex +++ b/lib/pleroma/user/query.ex @@ -63,7 +63,8 @@ defmodule Pleroma.User.Query do limit: pos_integer(), actor_types: [String.t()], birthday_day: pos_integer(), - birthday_month: pos_integer() + birthday_month: pos_integer(), + staff: boolean() } | map() @@ -179,6 +180,10 @@ defmodule Pleroma.User.Query do defp compose_query({:external, _}, query), do: location_query(query, false) + defp compose_query({:active, false}, query) do + where(query, [u], u.is_active == false or u.is_approved == false or u.is_confirmed == false) + end + defp compose_query({:active, _}, query) do where(query, [u], u.is_active == true) |> where([u], u.is_approved == true) @@ -202,6 +207,10 @@ defmodule Pleroma.User.Query do where(query, [u], u.is_confirmed != ^bool) end + defp compose_query({:need_approval, false}, query) do + where(query, [u], u.is_approved == true) + end + defp compose_query({:need_approval, _}, query) do where(query, [u], u.is_approved == false) end @@ -285,6 +294,14 @@ defmodule Pleroma.User.Query do |> where([u], fragment("date_part('month', ?)", u.birthday) == ^month) end + defp compose_query({:staff, false}, query) do + where(query, [u], u.is_admin == false and u.is_moderator == false) + end + + defp compose_query({:staff, _}, query) do + where(query, [u], u.is_admin == true or u.is_moderator == true) + end + defp compose_query(_unsupported_param, query), do: query defp location_query(query, local) do diff --git a/lib/pleroma/web/api_spec.ex b/lib/pleroma/web/api_spec.ex index 5006167ea..ff6363603 100644 --- a/lib/pleroma/web/api_spec.ex +++ b/lib/pleroma/web/api_spec.ex @@ -108,6 +108,13 @@ defmodule Pleroma.Web.ApiSpec do "Announcement management" ] }, + %{ + "name" => "Administration (Mastodon API)", + "tags" => [ + "User administration (Mastodon API)", + "Report management (Mastodon API)" + ] + }, %{"name" => "Applications", "tags" => ["Applications", "Push subscriptions"]}, %{ "name" => "Current account", diff --git a/lib/pleroma/web/api_spec/operations/mastodon_admin/account_operation.ex b/lib/pleroma/web/api_spec/operations/mastodon_admin/account_operation.ex new file mode 100644 index 000000000..23dae0306 --- /dev/null +++ b/lib/pleroma/web/api_spec/operations/mastodon_admin/account_operation.ex @@ -0,0 +1,258 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ApiSpec.MastodonAdmin.AccountOperation do + alias OpenApiSpex.Operation + alias OpenApiSpex.Schema + alias Pleroma.Web.ApiSpec.Schemas.Account + alias Pleroma.Web.ApiSpec.Schemas.ApiError + alias Pleroma.Web.ApiSpec.Schemas.FlakeID + + import Pleroma.Web.ApiSpec.Helpers + + def open_api_operation(action) do + operation = String.to_existing_atom("#{action}_operation") + apply(__MODULE__, operation, []) + end + + def index_operation do + %Operation{ + tags: ["User administration (Mastodon API)"], + summary: "View accounts by criteria", + operationId: "MastodonAdmin.AccountController.index", + description: "View accounts matching certain criteria for filtering, up to 40 at a time.", + security: [%{"oAuth" => ["admin:read:accounts"]}], + parameters: + [ + Operation.parameter(:local, :query, :boolean, "Filter for local accounts?"), + Operation.parameter(:remote, :query, :boolean, "Filter for remote accounts?"), + Operation.parameter( + :by_domain, + :query, + :string, + "Filter by the given domain (not implemented yet)" + ), + Operation.parameter( + :active, + :query, + :boolean, + "Filter for currently active accounts??" + ), + Operation.parameter( + :pending, + :query, + :boolean, + "Filter for currently pending accounts?" + ), + Operation.parameter( + :disabled, + :query, + :boolean, + "Filter for currently disabled accounts?" + ), + Operation.parameter( + :sensitized, + :query, + :boolean, + "Filter for currently sensitized accounts? (not implemented yet)" + ), + Operation.parameter( + :silenced, + :query, + :boolean, + "Filter for currently silenced accounts? (not implemented yet)" + ), + Operation.parameter( + :suspended, + :query, + :boolean, + "Filter for currently suspended accounts? (not implemented yet)" + ), + Operation.parameter(:username, :query, :string, "Username to search for"), + Operation.parameter(:display_name, :query, :string, "Display name to search for"), + Operation.parameter(:email, :query, :string, "Lookup a user with this email"), + Operation.parameter( + :ip, + :query, + :string, + "Lookup users by this IP address (not implemented yet)" + ), + Operation.parameter(:staff, :query, :boolean, "Filter for staff accounts?") + ] ++ + pagination_params(), + responses: %{ + 200 => + Operation.response("Account", "application/json", %Schema{ + title: "ArrayOfAccounts", + type: :array, + items: account() + }), + 401 => Operation.response("Error", "application/json", ApiError) + } + } + end + + def show_operation do + %Operation{ + tags: ["User administration (Mastodon API)"], + summary: "View a specific account", + operationId: "MastodonAdmin.AccountController.show", + description: "View admin-level information about the given account.", + security: [%{"oAuth" => ["admin:read:accounts"]}], + parameters: [ + Operation.parameter(:id, :path, :string, "ID of the account") + ], + responses: %{ + 200 => Operation.response("Account", "application/json", account()), + 401 => Operation.response("Error", "application/json", ApiError), + 404 => Operation.response("Error", "application/json", ApiError) + } + } + end + + def account_action_operation do + %Operation{ + tags: ["User administration (Mastodon API)"], + summary: "Perform an action against an account", + operationId: "MastodonAdmin.AccountController.account_action", + description: + "Perform an action against an account and log this action in the moderation history.", + security: [%{"oAuth" => ["admin:write:accounts"]}], + parameters: [ + Operation.parameter(:id, :path, :string, "ID of the account") + ], + requestBody: + request_body( + "Parameters", + %Schema{ + type: :object, + properties: %{ + type: %Schema{ + type: :string, + enum: ["none", "disable", "sensitive", "silence", "suspend"] + }, + report_id: %Schema{ + type: :string, + nullable: true, + description: "ID of an associated report that caused this action to be taken" + } + } + }, + required: true + ), + responses: %{ + 204 => no_content_response(), + 401 => Operation.response("Error", "application/json", ApiError) + } + } + end + + def delete_operation do + %Operation{ + tags: ["User administration (Mastodon API)"], + summary: "Delete a specific account", + operationId: "MastodonAdmin.AccountController.delete", + description: "Delete the given account.", + security: [%{"oAuth" => ["admin:write:accounts"]}], + parameters: [ + Operation.parameter(:id, :path, :string, "ID of the account") + ], + responses: %{ + 200 => Operation.response("Account", "application/json", account()), + 401 => Operation.response("Error", "application/json", ApiError), + 404 => Operation.response("Error", "application/json", ApiError) + } + } + end + + def enable_operation do + %Operation{ + tags: ["User administration (Mastodon API)"], + summary: "Re-enable account", + operationId: "MastodonAdmin.AccountController.enable", + description: "Re-enable a local account whose login is currently disabled.", + security: [%{"oAuth" => ["admin:write:accounts"]}], + parameters: [ + Operation.parameter(:id, :path, :string, "ID of the account") + ], + responses: %{ + 200 => Operation.response("Account", "application/json", account()), + 401 => Operation.response("Error", "application/json", ApiError), + 404 => Operation.response("Error", "application/json", ApiError) + } + } + end + + def approve_operation do + %Operation{ + tags: ["User administration (Mastodon API)"], + summary: "Approve pending account", + operationId: "MastodonAdmin.AccountController.approve", + description: "Approve the given local account if it is currently pending approval.", + parameters: [ + Operation.parameter(:id, :path, :string, "ID of the account") + ], + responses: %{ + 200 => Operation.response("Account", "application/json", account()), + 401 => Operation.response("Error", "application/json", ApiError), + 404 => Operation.response("Error", "application/json", ApiError) + } + } + end + + def reject_operation do + %Operation{ + tags: ["User administration (Mastodon API)"], + summary: "Reject pending account", + operationId: "MastodonAdmin.AccountController.reject", + description: "Reject the given local account if it is currently pending approval.", + parameters: [ + Operation.parameter(:id, :path, :string, "ID of the account") + ], + responses: %{ + 200 => Operation.response("Account", "application/json", account()), + 400 => Operation.response("Error", "application/json", ApiError), + 401 => Operation.response("Error", "application/json", ApiError), + 404 => Operation.response("Error", "application/json", ApiError) + } + } + end + + def account do + %Schema{ + title: "Admin::Account", + description: "Admin-level information about a given account.", + type: :object, + properties: %{ + id: FlakeID, + username: %Schema{type: :string}, + domain: %Schema{type: :string, nullable: true}, + created_at: %Schema{type: :string, format: "date-time"}, + email: %Schema{type: :string, format: "email", nullable: true}, + ip: %Schema{type: :string, nullable: true}, + ips: %Schema{ + type: :array, + items: %Schema{ + type: :object, + properties: %{ + ip: %Schema{type: :string}, + used_at: %Schema{type: :string, format: "date-time"} + } + } + }, + locale: %Schema{type: :string, format: "date-time", nullable: true}, + invite_request: %Schema{type: :string, format: "date-time", nullable: true}, + role: %Schema{type: :string, nullable: true}, + confirmed: %Schema{type: :boolean}, + approved: %Schema{type: :boolean}, + disabled: %Schema{type: :boolean}, + silenced: %Schema{type: :boolean, nullable: true}, + suspended: %Schema{type: :boolean, nullable: true}, + account: Account, + created_by_application_id: %Schema{type: :string, nullable: true}, + invited_by_account_id: %Schema{type: :string, nullable: true} + } + } + end +end diff --git a/lib/pleroma/web/api_spec/operations/mastodon_admin/report_operation.ex b/lib/pleroma/web/api_spec/operations/mastodon_admin/report_operation.ex new file mode 100644 index 000000000..274ce3bd9 --- /dev/null +++ b/lib/pleroma/web/api_spec/operations/mastodon_admin/report_operation.ex @@ -0,0 +1,150 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ApiSpec.MastodonAdmin.ReportOperation do + alias OpenApiSpex.Operation + alias OpenApiSpex.Schema + alias Pleroma.Web.ApiSpec.MastodonAdmin.AccountOperation + alias Pleroma.Web.ApiSpec.Schemas.ApiError + alias Pleroma.Web.ApiSpec.Schemas.FlakeID + alias Pleroma.Web.ApiSpec.Schemas.Status + + import Pleroma.Web.ApiSpec.Helpers + + def open_api_operation(action) do + operation = String.to_existing_atom("#{action}_operation") + apply(__MODULE__, operation, []) + end + + def index_operation do + %Operation{ + tags: ["Report management (Mastodon API)"], + summary: "View all reports", + operationId: "MastodonAdmin.ReportController.index", + description: + "View all reports. Pagination may be done with HTTP Link header in the response.", + security: [%{"oAuth" => ["admin:read:reports"]}], + parameters: + [ + Operation.parameter(:resolved, :query, :boolean, "Filter for resolved reports"), + Operation.parameter(:account_id, :query, :string, "Filter by author account id"), + Operation.parameter( + :target_account_id, + :query, + :string, + "Filter by report target account id (not implemented)" + ) + ] ++ + pagination_params(), + responses: %{ + 200 => + Operation.response("Reports", "application/json", %Schema{ + title: "ArrayOfReports", + type: :array, + items: report() + }), + 401 => Operation.response("Error", "application/json", ApiError) + } + } + end + + def show_operation do + %Operation{ + tags: ["Report management (Mastodon API)"], + summary: "View a single report", + operationId: "MastodonAdmin.ReportController.show", + description: "View information about the report with the given ID.", + security: [%{"oAuth" => ["admin:read:reports"]}], + parameters: [ + Operation.parameter(:id, :path, :string, "ID of the report") + ], + responses: %{ + 200 => Operation.response("Report", "application/json", report()), + 401 => Operation.response("Error", "application/json", ApiError), + 404 => Operation.response("Error", "application/json", ApiError) + } + } + end + + def resolve_operation do + %Operation{ + tags: ["Report management (Mastodon API)"], + summary: "Mark as resolved", + operationId: "MastodonAdmin.ReportController.resolve", + description: "Mark a report as resolved with no further action taken.", + security: [%{"oAuth" => ["admin:write:reports"]}], + parameters: [ + Operation.parameter(:id, :path, :string, "ID of the report") + ], + responses: %{ + 200 => Operation.response("Report", "application/json", report()), + 400 => Operation.response("Error", "application/json", ApiError), + 401 => Operation.response("Error", "application/json", ApiError) + } + } + end + + def reopen_operation do + %Operation{ + tags: ["Report management (Mastodon API)"], + summary: "Re-open report", + operationId: "MastodonAdmin.ReportController.reopen", + description: "Reopen a currently closed report.", + security: [%{"oAuth" => ["admin:write:reports"]}], + parameters: [ + Operation.parameter(:id, :path, :string, "ID of the report") + ], + responses: %{ + 200 => Operation.response("Report", "application/json", report()), + 400 => Operation.response("Error", "application/json", ApiError), + 401 => Operation.response("Error", "application/json", ApiError) + } + } + end + + defp report do + %Schema{ + title: "Admin::Report", + type: :object, + properties: %{ + id: FlakeID, + action_taken: %Schema{type: :boolean}, + action_taken_at: %Schema{type: :string, format: "date-time", nullable: true}, + category: %Schema{type: :string, enum: ["spam", "violation", "other"]}, + comment: %Schema{type: :string, nullable: true}, + forwarded: %Schema{type: :boolean, nullable: true}, + created_at: %Schema{type: :string, format: "date-time"}, + updated_at: %Schema{type: :string, format: "date-time"}, + account: AccountOperation.account(), + target_account: AccountOperation.account(), + assigned_account: %Schema{ + oneOf: [ + AccountOperation.account(), + %Schema{type: :null} + ] + }, + action_taken_by_account: %Schema{ + oneOf: [ + AccountOperation.account(), + %Schema{type: :null} + ] + }, + statuses: %Schema{ + type: :array, + items: Status + }, + rules: %Schema{ + type: :array, + items: %Schema{ + type: :object, + properties: %{ + id: %Schema{type: :integer}, + text: %Schema{type: :string} + } + } + } + } + } + end +end diff --git a/lib/pleroma/web/mastodon_api/admin/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/admin/controllers/account_controller.ex new file mode 100644 index 000000000..a1eac7fe3 --- /dev/null +++ b/lib/pleroma/web/mastodon_api/admin/controllers/account_controller.ex @@ -0,0 +1,220 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.MastodonAPI.Admin.AccountController do + use Pleroma.Web, :controller + + import Pleroma.Web.ControllerHelper, + only: [ + add_link_headers: 2, + assign_account_by_id: 2, + json_response: 3 + ] + + alias Pleroma.Activity + alias Pleroma.ModerationLog + alias Pleroma.Pagination + alias Pleroma.User + alias Pleroma.Web.ActivityPub.Builder + alias Pleroma.Web.ActivityPub.Pipeline + alias Pleroma.Web.CommonAPI + alias Pleroma.Web.Plugs.OAuthScopesPlug + + @filter_params ~W( + local external active needing_approval deactivated nickname name email staff + ) + + plug(Pleroma.Web.ApiSpec.CastAndValidate) + + plug( + OAuthScopesPlug, + %{scopes: ["admin:read:accounts"]} + when action in [:index, :show] + ) + + plug( + OAuthScopesPlug, + %{scopes: ["admin:write:accounts"]} + when action in [ + :delete, + :enable, + :account_action, + :approve, + :reject + ] + ) + + plug( + :assign_account_by_id + when action in [ + :show, + :delete, + :enable, + :account_action, + :approve, + :reject + ] + ) + + defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.MastodonAdmin.AccountOperation + + def index(conn, params) do + users = + params + |> build_criteria() + |> User.Query.build() + |> Pagination.fetch_paginated(params) + + conn + |> add_link_headers(users) + |> render("index.json", users: users) + end + + def show(%{assigns: %{user: _admin, account: user}} = conn, _params) do + render(conn, "show.json", user: user) + end + + def account_action( + %{assigns: %{user: admin, account: user}, body_params: %{type: type} = body_params} = + conn, + _params + ) do + {:ok, _user} = handle_account_action(user, admin, type) + + resolve_report(admin, body_params) + + json_response(conn, :no_content, "") + end + + def delete(%{assigns: %{user: admin, account: user}} = conn, _params) do + {:ok, delete_data, _} = Builder.delete(admin, user.ap_id) + Pipeline.common_pipeline(delete_data, local: true) + + ModerationLog.insert_log(%{ + actor: admin, + subject: [user], + action: "delete" + }) + + render(conn, "show.json", user: user) + end + + def enable(%{assigns: %{user: admin, account: user}} = conn, _params) do + {:ok, user} = User.set_activation(user, true) + + ModerationLog.insert_log(%{ + actor: admin, + subject: [user], + action: "activate" + }) + + render(conn, "show.json", user: user) + end + + def approve(%{assigns: %{user: admin, account: user}} = conn, _params) do + {:ok, user} = User.approve(user) + + ModerationLog.insert_log(%{ + actor: admin, + subject: [user], + action: "approve" + }) + + render(conn, "show.json", user: user) + end + + def reject(%{assigns: %{user: admin, account: user}} = conn, _params) do + with {:ok, _} <- User.reject(user) do + ModerationLog.insert_log(%{ + actor: admin, + subject: [user], + action: "reject" + }) + + render(conn, "show.json", user: user) + else + {:error, error} -> + json_response(conn, :bad_request, %{error: error}) + end + end + + defp handle_account_action(%User{local: true} = user, admin, "disable") do + ModerationLog.insert_log(%{ + actor: admin, + subject: [user], + action: "deactivate" + }) + + User.set_activation(user, false) + end + + defp handle_account_action(user, _admin, _type) do + {:ok, user} + end + + defp build_criteria(params) do + %{} + |> maybe_filter_local(params) + |> maybe_filter_external(params) + |> maybe_filter_active(params) + |> maybe_filter_needing_approval(params) + |> maybe_filter_deactivated(params) + |> maybe_filter_nickname(params) + |> maybe_filter_name(params) + |> maybe_filter_email(params) + |> maybe_filter_staff(params) + end + + defp maybe_filter_local(criteria, %{local: true} = _params), + do: Map.put(criteria, :local, true) + + defp maybe_filter_local(criteria, %{local: false} = _params), + do: Map.put(criteria, :external, true) + + defp maybe_filter_external(criteria, %{remote: true} = _params), + do: Map.put(criteria, :external, true) + + defp maybe_filter_external(criteria, %{remote: false} = _params), + do: Map.put(criteria, :local, true) + + defp maybe_filter_active(criteria, %{active: active} = _params), + do: Map.put(criteria, :active, active) + + defp maybe_filter_needing_approval(criteria, %{pending: need_approval} = _params), + do: Map.put(criteria, :need_approval, need_approval) + + defp maybe_filter_deactivated(criteria, %{disabled: deactivated} = _params), + do: Map.put(criteria, :deactivated, deactivated) + + defp maybe_filter_nickname(criteria, %{username: nickname} = _params), + do: Map.put(criteria, :nickname, nickname) + + defp maybe_filter_name(criteria, %{display_name: name} = _params), + do: Map.put(criteria, :name, name) + + defp maybe_filter_email(criteria, %{email: email} = _params), + do: Map.put(criteria, :email, email) + + defp maybe_filter_staff(criteria, %{staff: staff} = _params), + do: Map.put(criteria, :staff, staff) + + for filter_param <- @filter_params do + defp unquote(:"maybe_filter_#{filter_param}")(criteria, _params), do: criteria + end + + defp resolve_report(admin, %{report_id: id}) do + with {:ok, activity} <- CommonAPI.update_report_state(id, "resolved"), + report <- Activity.get_by_id_with_user_actor(activity.id) do + ModerationLog.insert_log(%{ + action: "report_update", + actor: admin, + subject: activity, + subject_actor: report.user_actor + }) + end + end + + defp resolve_report(_admin, _params) do + end +end diff --git a/lib/pleroma/web/mastodon_api/admin/controllers/report_controller.ex b/lib/pleroma/web/mastodon_api/admin/controllers/report_controller.ex new file mode 100644 index 000000000..217a97cc9 --- /dev/null +++ b/lib/pleroma/web/mastodon_api/admin/controllers/report_controller.ex @@ -0,0 +1,107 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.MastodonAPI.Admin.ReportController do + use Pleroma.Web, :controller + + import Pleroma.Web.ControllerHelper, + only: [ + add_link_headers: 2, + json_response: 3 + ] + + alias Pleroma.Activity + alias Pleroma.ModerationLog + alias Pleroma.Pagination + alias Pleroma.User + alias Pleroma.Web.ActivityPub.ActivityPub + alias Pleroma.Web.AdminAPI.Report + alias Pleroma.Web.CommonAPI + alias Pleroma.Web.Plugs.OAuthScopesPlug + + plug(Pleroma.Web.ApiSpec.CastAndValidate) + + plug(OAuthScopesPlug, %{scopes: ["admin:read:reports"]} when action in [:index, :show]) + + plug(OAuthScopesPlug, %{scopes: ["admin:write:reports"]} when action in [:resolve, :reopen]) + + defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.MastodonAdmin.ReportOperation + + def index(conn, params) do + opts = + %{} + |> Map.put(:type, "Flag") + |> Map.put(:skip_preload, true) + |> Map.put(:preload_report_notes, true) + |> Map.put(:total, true) + |> restrict_state(params) + |> restrict_actor(params) + + reports = + ActivityPub.fetch_activities_query([], opts) + |> Pagination.fetch_paginated(params) + + conn + |> add_link_headers(reports) + |> render("index.json", reports: reports) + end + + def show(conn, %{id: id}) do + with %Activity{} = report <- Activity.get_report(id) do + render(conn, "show.json", Report.extract_report_info(report)) + else + _ -> {:error, :not_found} + end + end + + def resolve(%{assigns: %{user: admin}} = conn, %{id: id}) do + with {:ok, activity} <- CommonAPI.update_report_state(id, "resolved"), + report <- Activity.get_by_id_with_user_actor(activity.id) do + ModerationLog.insert_log(%{ + action: "report_update", + actor: admin, + subject: activity, + subject_actor: report.user_actor + }) + + render(conn, "show.json", Report.extract_report_info(activity)) + else + {:error, error} -> + json_response(conn, :bad_request, %{error: error}) + end + end + + def reopen(%{assigns: %{user: admin}} = conn, %{id: id}) do + with {:ok, activity} <- CommonAPI.update_report_state(id, "open"), + report <- Activity.get_by_id_with_user_actor(activity.id) do + ModerationLog.insert_log(%{ + action: "report_update", + actor: admin, + subject: activity, + subject_actor: report.user_actor + }) + + render(conn, "show.json", Report.extract_report_info(report)) + else + {:error, error} -> + json_response(conn, :bad_request, %{error: error}) + end + end + + defp restrict_state(opts, %{resolved: true}), do: Map.put(opts, :state, "resolved") + + defp restrict_state(opts, %{resolved: false}), do: Map.put(opts, :state, "open") + + defp restrict_state(opts, _params), do: opts + + defp restrict_actor(opts, %{account_id: actor}) do + with %User{ap_id: ap_id} <- User.get_by_id(actor) do + Map.put(opts, :actor_id, ap_id) + else + _ -> Map.put(opts, :actor_id, actor) + end + end + + defp restrict_actor(opts, _params), do: opts +end diff --git a/lib/pleroma/web/mastodon_api/admin/views/account_view.ex b/lib/pleroma/web/mastodon_api/admin/views/account_view.ex new file mode 100644 index 000000000..20ccfd51c --- /dev/null +++ b/lib/pleroma/web/mastodon_api/admin/views/account_view.ex @@ -0,0 +1,64 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.MastodonAPI.Admin.AccountView do + use Pleroma.Web, :view + + alias Pleroma.User + alias Pleroma.Web.CommonAPI.Utils + alias Pleroma.Web.MastodonAPI + + def render("index.json", %{users: users}) do + render_many(users, __MODULE__, "show.json", as: :user) + end + + def render("show.json", %{user: user}) do + account = + MastodonAPI.AccountView.render("show.json", %{user: user, skip_visibility_check: true}) + + %{ + id: user.id, + username: username_from_nickname(user.nickname), + domain: domain_from_nickname(user.nickname), + created_at: Utils.to_masto_date(user.inserted_at), + email: user.email, + ip: nil, + ips: [], + locale: nil, + invite_request: user.registration_reason, + role: role(user), + confirmed: user.is_confirmed, + approved: user.is_approved, + disabled: !user.is_active, + silenced: nil, + suspended: nil, + account: account + } + end + + defp username_from_nickname(string) when is_binary(string) do + hd(String.split(string, "@")) + end + + defp username_from_nickname(_), do: nil + + defp domain_from_nickname(string) when is_binary(string) do + String.split(string, "@") + |> Enum.at(1, nil) + end + + defp domain_from_nickname(_), do: nil + + defp role(%User{is_admin: true}) do + "admin" + end + + defp role(%User{is_moderator: true}) do + "moderator" + end + + defp role(_user) do + nil + end +end diff --git a/lib/pleroma/web/mastodon_api/admin/views/report_view.ex b/lib/pleroma/web/mastodon_api/admin/views/report_view.ex new file mode 100644 index 000000000..664f58b03 --- /dev/null +++ b/lib/pleroma/web/mastodon_api/admin/views/report_view.ex @@ -0,0 +1,56 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.MastodonAPI.Admin.ReportView do + use Pleroma.Web, :view + + alias Pleroma.HTML + alias Pleroma.Web.AdminAPI.Report + alias Pleroma.Web.CommonAPI.Utils + alias Pleroma.Web.MastodonAPI.Admin.AccountView + alias Pleroma.Web.MastodonAPI.StatusView + + def render("index.json", %{reports: reports}) do + reports + |> Enum.map(&Report.extract_report_info/1) + |> Enum.map(&render(__MODULE__, "show.json", &1)) + end + + def render("show.json", %{ + report: report, + user: account, + account: target_account, + statuses: statuses + }) do + created_at = Utils.to_masto_date(report.data["published"]) + + content = + unless is_nil(report.data["content"]) do + HTML.filter_tags(report.data["content"]) + else + nil + end + + %{ + id: report.id, + action_taken: report.data["state"] != "open", + action_taken_at: nil, + category: "other", + comment: content, + forwarded: nil, + created_at: created_at, + updated_at: created_at, + account: AccountView.render("show.json", %{user: account}), + target_account: AccountView.render("show.json", %{user: target_account}), + assigned_account: nil, + action_taken_by_account: nil, + statuses: + StatusView.render("index.json", %{ + activities: statuses, + as: :activity + }), + rules: [] + } + end +end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 20ac1c67b..a8ba84036 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -418,6 +418,46 @@ defmodule Pleroma.Web.Router do delete("/chats/:id/messages/:message_id", ChatController, :delete_message) end + # Mastodon AdminAPI: admins and mods (staff) can perform these actions (if privileged by role) + scope "/api/v1/admin", Pleroma.Web.MastodonAPI.Admin do + pipe_through([:require_privileged_role_users_read]) + + get("/accounts", AccountController, :index) + get("/accounts/:id", AccountController, :show) + end + + # Mastodon AdminAPI: admins and mods (staff) can perform these actions (if privileged by role) + scope "/api/v1/admin", Pleroma.Web.MastodonAPI.Admin do + pipe_through(:require_privileged_role_users_delete) + + delete("/accounts/:id", AccountController, :delete) + end + + # Mastodon AdminAPI: admins and mods (staff) can perform these actions (if privileged by role) + scope "/api/v1/admin", Pleroma.Web.MastodonAPI.Admin do + pipe_through([:require_privileged_role_users_manage_activation_state]) + + post("/accounts/:id/action", AccountController, :account_action) + post("/accounts/:id/enable", AccountController, :enable) + end + + # Mastodon AdminAPI: admins and mods (staff) can perform these actions (if privileged by role) + scope "/api/v1/admin", Pleroma.Web.MastodonAPI.Admin do + pipe_through([:require_privileged_role_users_manage_invites]) + post("/accounts/:id/approve", AccountController, :approve) + post("/accounts/:id/reject", AccountController, :reject) + end + + # Mastodon AdminAPI: admins and mods (staff) can perform these actions (if privileged by role) + scope "/api/v1/admin", Pleroma.Web.MastodonAPI.Admin do + pipe_through([:require_privileged_role_reports_manage_reports]) + + get("/reports", ReportController, :index) + get("/reports/:id", ReportController, :show) + post("/reports/:id/resolve", ReportController, :resolve) + post("/reports/:id/reopen", ReportController, :reopen) + end + # AdminAPI: admins and mods (staff) can perform these actions (if privileged by role) scope "/api/v1/pleroma/admin", Pleroma.Web.AdminAPI do pipe_through(:require_privileged_role_emoji_manage_emoji) diff --git a/test/pleroma/web/mastodon_api/admin/controllers/account_controller_test.exs b/test/pleroma/web/mastodon_api/admin/controllers/account_controller_test.exs new file mode 100644 index 000000000..7280691d4 --- /dev/null +++ b/test/pleroma/web/mastodon_api/admin/controllers/account_controller_test.exs @@ -0,0 +1,175 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.MastodonAPI.Admin.AccountControllerTest do + use Pleroma.Web.ConnCase + use Oban.Testing, repo: Pleroma.Repo + + import Pleroma.Factory + + alias Pleroma.Repo + alias Pleroma.User + alias Pleroma.Web.CommonAPI + + setup_all do + Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) + + :ok + end + + setup do + admin = insert(:user, is_admin: true) + token = insert(:oauth_admin_token, user: admin) + + conn = + build_conn() + |> assign(:user, admin) + |> assign(:token, token) + + {:ok, %{admin: admin, token: token, conn: conn}} + end + + describe "GET /api/v1/admin/accounts" do + test "search by display name", %{conn: conn} do + %{id: id} = insert(:user, name: "Display name") + insert(:user, name: "Other name") + + assert [%{"id" => ^id}] = + conn + |> get("/api/v1/admin/accounts?display_name=Display") + |> json_response_and_validate_schema(200) + end + end + + describe "GET /api/v1/admin/accounts/:id" do + test "show admin-level information", %{conn: conn} do + %{id: id} = + insert(:user, + email: "email@example.com", + is_confirmed: false, + is_moderator: true + ) + + assert %{ + "id" => ^id, + "email" => "email@example.com", + "confirmed" => false, + "role" => "moderator" + } = + conn + |> get("/api/v1/admin/accounts/#{id}") + |> json_response_and_validate_schema(200) + end + end + + describe "DELETE /api/v1/admin/accounts/:id" do + test "delete account", %{conn: conn} do + %{id: id} = user = insert(:user) + + conn + |> delete("/api/v1/admin/accounts/#{id}") + |> json_response_and_validate_schema(200) + + user = Repo.reload!(user) + + assert %{is_active: false} = user + end + end + + describe "POST /api/v1/admin/accounts/:id/action" do + test "disable account", %{conn: conn} do + %{id: id} = user = insert(:user) + + conn + |> put_req_header("content-type", "application/json") + |> post("/api/v1/admin/accounts/#{id}/action", %{ + "type" => "disable" + }) + |> json_response_and_validate_schema(204) + + user = Repo.reload!(user) + + assert %{is_active: false} = user + end + + test "perform action with assigned report", %{conn: conn} do + [reporter, target_user] = insert_pair(:user) + + {:ok, %{id: report_id} = report} = + CommonAPI.report(reporter, %{ + account_id: target_user.id + }) + + %{id: id} = insert(:user) + + conn + |> put_req_header("content-type", "application/json") + |> post("/api/v1/admin/accounts/#{id}/action", %{ + "type" => "none", + "report_id" => report_id + }) + |> json_response_and_validate_schema(204) + + report = Repo.reload!(report) + + assert %{data: %{"state" => "resolved"}} = report + end + end + + describe "POST /api/v1/admin/accounts/:id/enable" do + test "enable account", %{conn: conn} do + %{id: id} = user = insert(:user) + User.set_activation(user, false) + + conn + |> post("/api/v1/admin/accounts/#{id}/enable") + |> json_response_and_validate_schema(200) + + user = Repo.reload!(user) + + assert %{is_active: true} = user + end + end + + describe "POST /api/v1/admin/accounts/:id/approve" do + test "approve account", %{conn: conn} do + %{id: id} = user = insert(:user, is_approved: false) + + conn + |> post("/api/v1/admin/accounts/#{id}/approve") + |> json_response_and_validate_schema(200) + + user = Repo.reload!(user) + + assert %{is_approved: true} = user + end + end + + describe "POST /api/v1/admin/accounts/:id/rejct" do + test "reject account", %{conn: conn} do + %{id: id} = user = insert(:user, is_approved: false) + + conn + |> post("/api/v1/admin/accounts/#{id}/reject") + |> json_response_and_validate_schema(200) + + user = Repo.reload!(user) + + assert %{is_active: false} = user + end + + test "do not allow rejecting already accepted accounts", %{conn: conn} do + %{id: id} = user = insert(:user, is_approved: true) + + assert %{"error" => "User is approved"} == + conn + |> post("/api/v1/admin/accounts/#{id}/reject") + |> json_response_and_validate_schema(400) + + user = Repo.reload!(user) + + assert %{is_approved: true} = user + end + end +end diff --git a/test/pleroma/web/mastodon_api/admin/controllers/report_controller_test.exs b/test/pleroma/web/mastodon_api/admin/controllers/report_controller_test.exs new file mode 100644 index 000000000..a0cc5f459 --- /dev/null +++ b/test/pleroma/web/mastodon_api/admin/controllers/report_controller_test.exs @@ -0,0 +1,118 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.MastodonAPI.Admin.ReportControllerTest do + use Pleroma.Web.ConnCase + use Oban.Testing, repo: Pleroma.Repo + + import Pleroma.Factory + + alias Pleroma.Web.CommonAPI + + setup_all do + Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) + + :ok + end + + setup do + admin = insert(:user, is_admin: true) + token = insert(:oauth_admin_token, user: admin) + + conn = + build_conn() + |> assign(:user, admin) + |> assign(:token, token) + + {:ok, %{admin: admin, token: token, conn: conn}} + end + + describe "GET /api/v1/admin/reports" do + test "get reports by state", %{conn: conn} do + [reporter, target_user] = insert_pair(:user) + activity = insert(:note_activity, user: target_user) + + {:ok, %{id: report_id1}} = + CommonAPI.report(reporter, %{ + account_id: target_user.id, + comment: "this user uses an app called Hitler Tusky", + status_ids: [activity.id] + }) + + {:ok, %{id: report_id2}} = + CommonAPI.report(reporter, %{ + account_id: target_user.id, + status_ids: [activity.id] + }) + + CommonAPI.update_report_state(report_id2, "resolved") + + assert [%{"id" => ^report_id1}] = + conn + |> get("/api/v1/admin/reports?resolved=false") + |> json_response_and_validate_schema(200) + + assert [%{"id" => ^report_id2}] = + conn + |> get("/api/v1/admin/reports?resolved=true") + |> json_response_and_validate_schema(200) + end + end + + describe "GET /api/v1/admin/reports/:id" do + test "get report by id", %{conn: conn} do + [reporter, target_user] = insert_pair(:user) + activity = insert(:note_activity, user: target_user) + + {:ok, %{id: report_id}} = + CommonAPI.report(reporter, %{ + account_id: target_user.id, + status_ids: [activity.id] + }) + + assert %{"id" => ^report_id} = + conn + |> get("/api/v1/admin/reports/#{report_id}") + |> json_response_and_validate_schema(200) + end + end + + describe "POST /api/v1/admin/reports/:id/resolve" do + test "resolve a report", %{conn: conn} do + [reporter, target_user] = insert_pair(:user) + activity = insert(:note_activity, user: target_user) + + {:ok, %{id: report_id}} = + CommonAPI.report(reporter, %{ + account_id: target_user.id, + status_ids: [activity.id] + }) + + assert %{"id" => ^report_id, "action_taken" => true} = + conn + |> post("/api/v1/admin/reports/#{report_id}/resolve") + |> json_response_and_validate_schema(200) + end + end + + describe "POST /api/v1/admin/reports/:id/reopen" do + test "reopen a report", %{conn: conn} do + [reporter, target_user] = insert_pair(:user) + activity = insert(:note_activity, user: target_user) + + {:ok, %{id: report_id}} = + CommonAPI.report(reporter, %{ + account_id: target_user.id, + status_ids: [activity.id] + }) + + CommonAPI.update_report_state(report_id, "resolved") + + assert %{"id" => ^report_id, "action_taken" => false} = + conn + |> post("/api/v1/admin/reports/#{report_id}/reopen") + |> json_response_and_validate_schema(200) + end + end +end From 2c14121389d0dadd0609695b941881eb5d93cbae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?nicole=20miko=C5=82ajczyk?= Date: Fri, 19 Sep 2025 16:59:36 +0200 Subject: [PATCH 35/90] MastoAPI: /api/v2/admin/accounts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: nicole mikołajczyk --- lib/pleroma/user/query.ex | 5 + .../mastodon_admin/account_operation.ex | 118 +++++++++++++++--- .../mastodon_admin/report_operation.ex | 12 +- .../admin/controllers/account_controller.ex | 48 ++++++- lib/pleroma/web/router.ex | 7 ++ 5 files changed, 164 insertions(+), 26 deletions(-) diff --git a/lib/pleroma/user/query.ex b/lib/pleroma/user/query.ex index b877b52b2..33f19c454 100644 --- a/lib/pleroma/user/query.ex +++ b/lib/pleroma/user/query.ex @@ -302,6 +302,11 @@ defmodule Pleroma.User.Query do where(query, [u], u.is_admin == true or u.is_moderator == true) end + defp compose_query({:domain, domain}, query) do + query + |> where([u], like(u.nickname, ^"%@#{domain}")) + end + defp compose_query(_unsupported_param, query), do: query defp location_query(query, local) do diff --git a/lib/pleroma/web/api_spec/operations/mastodon_admin/account_operation.ex b/lib/pleroma/web/api_spec/operations/mastodon_admin/account_operation.ex index 23dae0306..19ccf97fb 100644 --- a/lib/pleroma/web/api_spec/operations/mastodon_admin/account_operation.ex +++ b/lib/pleroma/web/api_spec/operations/mastodon_admin/account_operation.ex @@ -19,7 +19,7 @@ defmodule Pleroma.Web.ApiSpec.MastodonAdmin.AccountOperation do def index_operation do %Operation{ tags: ["User administration (Mastodon API)"], - summary: "View accounts by criteria", + summary: "View accounts by criteria (v1)", operationId: "MastodonAdmin.AccountController.index", description: "View accounts matching certain criteria for filtering, up to 40 at a time.", security: [%{"oAuth" => ["admin:read:accounts"]}], @@ -27,12 +27,6 @@ defmodule Pleroma.Web.ApiSpec.MastodonAdmin.AccountOperation do [ Operation.parameter(:local, :query, :boolean, "Filter for local accounts?"), Operation.parameter(:remote, :query, :boolean, "Filter for remote accounts?"), - Operation.parameter( - :by_domain, - :query, - :string, - "Filter by the given domain (not implemented yet)" - ), Operation.parameter( :active, :query, @@ -51,12 +45,6 @@ defmodule Pleroma.Web.ApiSpec.MastodonAdmin.AccountOperation do :boolean, "Filter for currently disabled accounts?" ), - Operation.parameter( - :sensitized, - :query, - :boolean, - "Filter for currently sensitized accounts? (not implemented yet)" - ), Operation.parameter( :silenced, :query, @@ -69,14 +57,31 @@ defmodule Pleroma.Web.ApiSpec.MastodonAdmin.AccountOperation do :boolean, "Filter for currently suspended accounts? (not implemented yet)" ), - Operation.parameter(:username, :query, :string, "Username to search for"), - Operation.parameter(:display_name, :query, :string, "Display name to search for"), + Operation.parameter( + :sensitized, + :query, + :boolean, + "Filter for accounts force-marked as sensitive? (not implemented yet)" + ), + Operation.parameter(:username, :query, :string, "Search for the given username"), + Operation.parameter( + :display_name, + :query, + :string, + "Search for the given display name" + ), + Operation.parameter( + :by_domain, + :query, + :string, + "Filter by the given domain" + ), Operation.parameter(:email, :query, :string, "Lookup a user with this email"), Operation.parameter( :ip, :query, :string, - "Lookup users by this IP address (not implemented yet)" + "Lookup users with this IP address (not implemented yet)" ), Operation.parameter(:staff, :query, :boolean, "Filter for staff accounts?") ] ++ @@ -93,6 +98,87 @@ defmodule Pleroma.Web.ApiSpec.MastodonAdmin.AccountOperation do } end + def index2_operation do + %Operation{ + tags: ["User administration (Mastodon API)"], + summary: "View accounts by criteria (v2)", + operationId: "MastodonAdmin.AccountController.index2", + description: "View accounts matching certain criteria for filtering, up to 40 at a time.", + security: [%{"oAuth" => ["admin:read:accounts"]}], + parameters: + [ + Operation.parameter( + :origin, + :query, + %Schema{type: :string, enum: ["local", "remote"]}, + "Filter for local or remote accounts" + ), + Operation.parameter( + :status, + :query, + %Schema{ + type: :string, + enum: ["active", "inactive", "pending", "disabled", "silenced", "suspended"] + }, + "Filter for active, pending, disabled, silenced or suspended accounts" + ), + Operation.parameter( + :permissions, + :query, + :string, + "Filter for accounts with staff permissions (users that can manage reports). (not implemented yet)" + ), + Operation.parameter( + :role_ids, + :query, + %Schema{ + oneOf: [ + %Schema{type: :array, items: %Schema{type: :string}}, + %Schema{type: :string} + ] + }, + "Filter for users with these roles. (not implemented yet)" + ), + Operation.parameter( + :invited_by, + :query, + :string, + "Lookup users invited by the account with this ID. (not implemented yet)" + ), + Operation.parameter(:username, :query, :string, "Search for the given username"), + Operation.parameter( + :display_name, + :query, + :string, + "Search for the given display name" + ), + Operation.parameter( + :by_domain, + :query, + :string, + "Filter by the given domain" + ), + Operation.parameter(:email, :query, :string, "Lookup a user with this email"), + Operation.parameter( + :ip, + :query, + :string, + "Lookup users with this IP address (not implemented yet)" + ) + ] ++ + pagination_params(), + responses: %{ + 200 => + Operation.response("Account", "application/json", %Schema{ + title: "ArrayOfAccounts", + type: :array, + items: account() + }), + 401 => Operation.response("Error", "application/json", ApiError) + } + } + end + def show_operation do %Operation{ tags: ["User administration (Mastodon API)"], diff --git a/lib/pleroma/web/api_spec/operations/mastodon_admin/report_operation.ex b/lib/pleroma/web/api_spec/operations/mastodon_admin/report_operation.ex index 274ce3bd9..487e7853e 100644 --- a/lib/pleroma/web/api_spec/operations/mastodon_admin/report_operation.ex +++ b/lib/pleroma/web/api_spec/operations/mastodon_admin/report_operation.ex @@ -119,16 +119,12 @@ defmodule Pleroma.Web.ApiSpec.MastodonAdmin.ReportOperation do account: AccountOperation.account(), target_account: AccountOperation.account(), assigned_account: %Schema{ - oneOf: [ - AccountOperation.account(), - %Schema{type: :null} - ] + type: AccountOperation.account(), + nullable: true }, action_taken_by_account: %Schema{ - oneOf: [ - AccountOperation.account(), - %Schema{type: :null} - ] + type: AccountOperation.account(), + nullable: true }, statuses: %Schema{ type: :array, diff --git a/lib/pleroma/web/mastodon_api/admin/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/admin/controllers/account_controller.ex index a1eac7fe3..833bf1db5 100644 --- a/lib/pleroma/web/mastodon_api/admin/controllers/account_controller.ex +++ b/lib/pleroma/web/mastodon_api/admin/controllers/account_controller.ex @@ -22,7 +22,7 @@ defmodule Pleroma.Web.MastodonAPI.Admin.AccountController do alias Pleroma.Web.Plugs.OAuthScopesPlug @filter_params ~W( - local external active needing_approval deactivated nickname name email staff + local external active needing_approval deactivated nickname name domain email staff origin status ) plug(Pleroma.Web.ApiSpec.CastAndValidate) @@ -30,7 +30,7 @@ defmodule Pleroma.Web.MastodonAPI.Admin.AccountController do plug( OAuthScopesPlug, %{scopes: ["admin:read:accounts"]} - when action in [:index, :show] + when action in [:index, :index2, :show] ) plug( @@ -71,6 +71,18 @@ defmodule Pleroma.Web.MastodonAPI.Admin.AccountController do |> render("index.json", users: users) end + def index2(conn, params) do + users = + params + |> build_criteria_v2() + |> User.Query.build() + |> Pagination.fetch_paginated(params) + + conn + |> add_link_headers(users) + |> render("index.json", users: users) + end + def show(%{assigns: %{user: _admin, account: user}} = conn, _params) do render(conn, "show.json", user: user) end @@ -162,10 +174,21 @@ defmodule Pleroma.Web.MastodonAPI.Admin.AccountController do |> maybe_filter_deactivated(params) |> maybe_filter_nickname(params) |> maybe_filter_name(params) + |> maybe_filter_domain(params) |> maybe_filter_email(params) |> maybe_filter_staff(params) end + defp build_criteria_v2(params) do + %{} + |> maybe_filter_origin(params) + |> maybe_filter_status(params) + |> maybe_filter_nickname(params) + |> maybe_filter_name(params) + |> maybe_filter_domain(params) + |> maybe_filter_email(params) + end + defp maybe_filter_local(criteria, %{local: true} = _params), do: Map.put(criteria, :local, true) @@ -178,6 +201,12 @@ defmodule Pleroma.Web.MastodonAPI.Admin.AccountController do defp maybe_filter_external(criteria, %{remote: false} = _params), do: Map.put(criteria, :local, true) + defp maybe_filter_origin(criteria, %{origin: "local"} = _params), + do: Map.put(criteria, :local, true) + + defp maybe_filter_origin(criteria, %{origin: "remote"} = _params), + do: Map.put(criteria, :external, true) + defp maybe_filter_active(criteria, %{active: active} = _params), do: Map.put(criteria, :active, active) @@ -187,12 +216,27 @@ defmodule Pleroma.Web.MastodonAPI.Admin.AccountController do defp maybe_filter_deactivated(criteria, %{disabled: deactivated} = _params), do: Map.put(criteria, :deactivated, deactivated) + defp maybe_filter_status(criteria, %{status: "active"} = _params), + do: Map.put(criteria, :active, true) + + defp maybe_filter_status(criteria, %{status: "inactive"} = _params), + do: Map.put(criteria, :active, false) + + defp maybe_filter_status(criteria, %{status: "pending"} = _params), + do: Map.put(criteria, :need_approval, true) + + defp maybe_filter_status(criteria, %{status: "disabled"} = _params), + do: Map.put(criteria, :deactivated, true) + defp maybe_filter_nickname(criteria, %{username: nickname} = _params), do: Map.put(criteria, :nickname, nickname) defp maybe_filter_name(criteria, %{display_name: name} = _params), do: Map.put(criteria, :name, name) + defp maybe_filter_domain(criteria, %{by_domain: domain} = _params), + do: Map.put(criteria, :domain, domain) + defp maybe_filter_email(criteria, %{email: email} = _params), do: Map.put(criteria, :email, email) diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index a8ba84036..87568b481 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -458,6 +458,13 @@ defmodule Pleroma.Web.Router do post("/reports/:id/reopen", ReportController, :reopen) end + # Mastodon AdminAPI: admins and mods (staff) can perform these actions (if privileged by role) + scope "/api/v2/admin", Pleroma.Web.MastodonAPI.Admin do + pipe_through([:require_privileged_role_users_read]) + + get("/accounts", AccountController, :index2) + end + # AdminAPI: admins and mods (staff) can perform these actions (if privileged by role) scope "/api/v1/pleroma/admin", Pleroma.Web.AdminAPI do pipe_through(:require_privileged_role_emoji_manage_emoji) From 87e6cd3dc009f977fec7fddb664c73cc525d9fe1 Mon Sep 17 00:00:00 2001 From: mkljczk Date: Tue, 18 Mar 2025 00:34:37 +0100 Subject: [PATCH 36/90] Adapt tests for v2/admin/accounts from Mastodon MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: mkljczk Signed-off-by: nicole mikołajczyk --- .../controllers/account_controller_test.exs | 90 +++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/test/pleroma/web/mastodon_api/admin/controllers/account_controller_test.exs b/test/pleroma/web/mastodon_api/admin/controllers/account_controller_test.exs index 7280691d4..e06b3a70a 100644 --- a/test/pleroma/web/mastodon_api/admin/controllers/account_controller_test.exs +++ b/test/pleroma/web/mastodon_api/admin/controllers/account_controller_test.exs @@ -42,6 +42,96 @@ defmodule Pleroma.Web.MastodonAPI.Admin.AccountControllerTest do end end + # Adapted from + # https://github.com/mastodon/mastodon/blob/main/spec/requests/api/v2/admin/accounts_spec.rb + describe "GET /api/v2/admin/accounts" do + setup do + remote_account = insert(:user, nickname: "remote@example.org", local: false) + other_remote_account = insert(:user, nickname: "other@foo.bar", local: false) + # suspended_account = insert(:user) + # suspended_remote = insert(:user) + disabled_account = insert(:user, is_active: false) + pending_account = insert(:user, is_approved: false) + admin_account = insert(:user, is_admin: true) + + {:ok, + %{ + remote_account: remote_account, + other_remote_account: other_remote_account, + disabled_account: disabled_account, + pending_account: pending_account, + admin_account: admin_account + }} + end + + # test "returns the correct accounts when called with status active + # and origin local and permissions staff", %{ + # conn: conn, + # admin_account: %{id: admin_account_id} + # } do + # assert [%{"id" => ^admin_account_id}] = + # conn + # |> get("/api/v2/admin/accounts?status=active&origin=local&permissions=staff") + # |> json_response_and_validate_schema(200) + # end + + test "returns the correct accounts when called with by_domain value and origin remote", %{ + conn: conn, + remote_account: %{id: remote_account_id} + } do + assert [%{"id" => ^remote_account_id}] = + conn + |> get("/api/v2/admin/accounts?by_domain=example.org&origin=remote") + |> json_response_and_validate_schema(200) + end + + # test "returns the correct accounts when called with status suspended", %{ + # conn: conn, + # suspended_account: %{id: suspended_account_id} + # } do + # assert [%{"id" => ^suspended_account_id}] = + # conn + # |> get("/api/v2/admin/accounts?status=suspended") + # |> json_response_and_validate_schema(200) + # end + + test "returns the correct accounts when called with status disabled", %{ + conn: conn, + disabled_account: %{id: disabled_account_id} + } do + assert [%{"id" => ^disabled_account_id}] = + conn + |> get("/api/v2/admin/accounts?status=disabled") + |> json_response_and_validate_schema(200) + end + + test "returns the correct accounts when called with status pending", %{ + conn: conn, + pending_account: %{id: pending_account_id} + } do + assert [%{"id" => ^pending_account_id}] = + conn + |> get("/api/v2/admin/accounts?status=pending") + |> json_response_and_validate_schema(200) + end + + test "sets the correct pagination headers with limit param", %{ + conn: conn, + admin_account: %{id: admin_account_id} + } do + response = + conn + |> get("/api/v2/admin/accounts?limit=1") + + next_url = + ~r{<.+?(?/api[^>]+)>; rel=\"next\"} + |> Regex.named_captures(get_resp_header(response, "link") |> Enum.at(0)) + |> Map.get("link") + + next_url =~ "&limit=1&max_id=#{admin_account_id}" + end + end + describe "GET /api/v1/admin/accounts/:id" do test "show admin-level information", %{conn: conn} do %{id: id} = From 8626a337f807d6ab97840419148f187d081d18bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?nicole=20miko=C5=82ajczyk?= Date: Fri, 27 Feb 2026 22:34:18 +0100 Subject: [PATCH 37/90] Mastodon admin API v2 improvements MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: nicole mikołajczyk --- .../api_spec/operations/mastodon_admin/account_operation.ex | 6 +++--- .../api_spec/operations/mastodon_admin/report_operation.ex | 4 ++-- .../web/mastodon_api/admin/controllers/report_controller.ex | 2 +- .../admin/controllers/account_controller_test.exs | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/pleroma/web/api_spec/operations/mastodon_admin/account_operation.ex b/lib/pleroma/web/api_spec/operations/mastodon_admin/account_operation.ex index 19ccf97fb..e9143364c 100644 --- a/lib/pleroma/web/api_spec/operations/mastodon_admin/account_operation.ex +++ b/lib/pleroma/web/api_spec/operations/mastodon_admin/account_operation.ex @@ -31,7 +31,7 @@ defmodule Pleroma.Web.ApiSpec.MastodonAdmin.AccountOperation do :active, :query, :boolean, - "Filter for currently active accounts??" + "Filter for currently active accounts?" ), Operation.parameter( :pending, @@ -327,8 +327,8 @@ defmodule Pleroma.Web.ApiSpec.MastodonAdmin.AccountOperation do } } }, - locale: %Schema{type: :string, format: "date-time", nullable: true}, - invite_request: %Schema{type: :string, format: "date-time", nullable: true}, + locale: %Schema{type: :string, nullable: true}, + invite_request: %Schema{type: :string, nullable: true}, role: %Schema{type: :string, nullable: true}, confirmed: %Schema{type: :boolean}, approved: %Schema{type: :boolean}, diff --git a/lib/pleroma/web/api_spec/operations/mastodon_admin/report_operation.ex b/lib/pleroma/web/api_spec/operations/mastodon_admin/report_operation.ex index 487e7853e..3360cb843 100644 --- a/lib/pleroma/web/api_spec/operations/mastodon_admin/report_operation.ex +++ b/lib/pleroma/web/api_spec/operations/mastodon_admin/report_operation.ex @@ -119,11 +119,11 @@ defmodule Pleroma.Web.ApiSpec.MastodonAdmin.ReportOperation do account: AccountOperation.account(), target_account: AccountOperation.account(), assigned_account: %Schema{ - type: AccountOperation.account(), + oneOf: [AccountOperation.account()], nullable: true }, action_taken_by_account: %Schema{ - type: AccountOperation.account(), + oneOf: [AccountOperation.account()], nullable: true }, statuses: %Schema{ diff --git a/lib/pleroma/web/mastodon_api/admin/controllers/report_controller.ex b/lib/pleroma/web/mastodon_api/admin/controllers/report_controller.ex index 217a97cc9..2d89d13bc 100644 --- a/lib/pleroma/web/mastodon_api/admin/controllers/report_controller.ex +++ b/lib/pleroma/web/mastodon_api/admin/controllers/report_controller.ex @@ -65,7 +65,7 @@ defmodule Pleroma.Web.MastodonAPI.Admin.ReportController do subject_actor: report.user_actor }) - render(conn, "show.json", Report.extract_report_info(activity)) + render(conn, "show.json", Report.extract_report_info(report)) else {:error, error} -> json_response(conn, :bad_request, %{error: error}) diff --git a/test/pleroma/web/mastodon_api/admin/controllers/account_controller_test.exs b/test/pleroma/web/mastodon_api/admin/controllers/account_controller_test.exs index e06b3a70a..113b242dd 100644 --- a/test/pleroma/web/mastodon_api/admin/controllers/account_controller_test.exs +++ b/test/pleroma/web/mastodon_api/admin/controllers/account_controller_test.exs @@ -236,7 +236,7 @@ defmodule Pleroma.Web.MastodonAPI.Admin.AccountControllerTest do end end - describe "POST /api/v1/admin/accounts/:id/rejct" do + describe "POST /api/v1/admin/accounts/:id/reject" do test "reject account", %{conn: conn} do %{id: id} = user = insert(:user, is_approved: false) From 8ac967518b1729aa2530bb2e3003a31bc40cabfb Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 25 Mar 2026 15:11:45 -0700 Subject: [PATCH 38/90] Switch to the newly relocated Pleroma.Search.object_to_search_data/1 --- lib/pleroma/search/paradedb.ex | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/search/paradedb.ex b/lib/pleroma/search/paradedb.ex index a6649a894..20be5b6f9 100644 --- a/lib/pleroma/search/paradedb.ex +++ b/lib/pleroma/search/paradedb.ex @@ -18,10 +18,9 @@ defmodule Pleroma.Search.ParadeDB do maybe_restrict_local: 2 ] - import Pleroma.Search.Meilisearch, only: [object_to_search_data: 1] - alias Pleroma.Activity alias Pleroma.Config.Getting, as: Config + alias Pleroma.Search @max_limit 40 @default_table "pleroma_search_documents" @@ -81,7 +80,7 @@ defmodule Pleroma.Search.ParadeDB do @impl true def add_to_index(activity) do - maybe_search_data = object_to_search_data(activity.object) + maybe_search_data = Search.object_to_search_data(activity.object) if activity.data["type"] == "Create" and maybe_search_data do dumped_activity_id = dump_activity_id(activity.id) From 1e3456a1604ebe5818e319f47bbeb79828c3d6e5 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 25 Mar 2026 15:14:35 -0700 Subject: [PATCH 39/90] Ensure the Ecto Repo for ParadeDB is started for the data indexing Mix task --- lib/mix/pleroma.ex | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/lib/mix/pleroma.ex b/lib/mix/pleroma.ex index e7869919c..372bf3399 100644 --- a/lib/mix/pleroma.ex +++ b/lib/mix/pleroma.ex @@ -62,7 +62,8 @@ defmodule Mix.Pleroma do {Majic.Pool, [name: Pleroma.MajicPool, pool_size: Pleroma.Config.get([:majic_pool, :size], 2)]} ] ++ - http_children(adapter) + http_children(adapter) ++ + search_children() cachex_children = Enum.map(@cachex_children, &Pleroma.Application.build_cachex(&1, [])) @@ -141,4 +142,14 @@ defmodule Mix.Pleroma do end defp http_children(_), do: [] + + defp search_children do + case Config.get([Pleroma.Search, :module]) do + Pleroma.Search.ParadeDB -> + [Pleroma.Search.ParadeDB.Repo] + + _ -> + [] + end + end end From 3776f13f20172fa7c88127ba28583fba27cb1e1c Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 1 Apr 2026 19:10:29 +0000 Subject: [PATCH 40/90] Fix warning --- lib/mix/pleroma.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/mix/pleroma.ex b/lib/mix/pleroma.ex index 372bf3399..aebad6984 100644 --- a/lib/mix/pleroma.ex +++ b/lib/mix/pleroma.ex @@ -144,7 +144,7 @@ defmodule Mix.Pleroma do defp http_children(_), do: [] defp search_children do - case Config.get([Pleroma.Search, :module]) do + case Pleroma.Config.get([Pleroma.Search, :module]) do Pleroma.Search.ParadeDB -> [Pleroma.Search.ParadeDB.Repo] From ae763ff6cf97fdb78edc5bdad13aab917cea37ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?nicole=20miko=C5=82ajczyk?= Date: Mon, 4 May 2026 01:41:12 +0200 Subject: [PATCH 41/90] api/v2/admin/accounts: support 'permissions' param MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: nicole mikołajczyk --- .../api_spec/operations/mastodon_admin/account_operation.ex | 2 +- .../web/mastodon_api/admin/controllers/account_controller.ex | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/api_spec/operations/mastodon_admin/account_operation.ex b/lib/pleroma/web/api_spec/operations/mastodon_admin/account_operation.ex index e9143364c..ad79d43d7 100644 --- a/lib/pleroma/web/api_spec/operations/mastodon_admin/account_operation.ex +++ b/lib/pleroma/web/api_spec/operations/mastodon_admin/account_operation.ex @@ -126,7 +126,7 @@ defmodule Pleroma.Web.ApiSpec.MastodonAdmin.AccountOperation do :permissions, :query, :string, - "Filter for accounts with staff permissions (users that can manage reports). (not implemented yet)" + "Filter for accounts with staff permissions (users that can manage reports)." ), Operation.parameter( :role_ids, diff --git a/lib/pleroma/web/mastodon_api/admin/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/admin/controllers/account_controller.ex index 833bf1db5..e96566774 100644 --- a/lib/pleroma/web/mastodon_api/admin/controllers/account_controller.ex +++ b/lib/pleroma/web/mastodon_api/admin/controllers/account_controller.ex @@ -183,6 +183,7 @@ defmodule Pleroma.Web.MastodonAPI.Admin.AccountController do %{} |> maybe_filter_origin(params) |> maybe_filter_status(params) + |> maybe_filter_staff(params) |> maybe_filter_nickname(params) |> maybe_filter_name(params) |> maybe_filter_domain(params) @@ -240,6 +241,9 @@ defmodule Pleroma.Web.MastodonAPI.Admin.AccountController do defp maybe_filter_email(criteria, %{email: email} = _params), do: Map.put(criteria, :email, email) + defp maybe_filter_staff(criteria, %{permissions: "staff"} = _params), + do: Map.put(criteria, :staff, true) + defp maybe_filter_staff(criteria, %{staff: staff} = _params), do: Map.put(criteria, :staff, staff) From 11281d740a136c349b927587c1d895ea590f76af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?nicole=20miko=C5=82ajczyk?= Date: Mon, 4 May 2026 13:48:38 +0200 Subject: [PATCH 42/90] make GET /api/v1/admin/reports match Mastodon admin API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: nicole mikołajczyk --- .../mastodon_admin/report_operation.ex | 1 + .../admin/controllers/report_controller.ex | 39 ++++++++++++++++++- .../controllers/report_controller_test.exs | 12 +++++- 3 files changed, 49 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/api_spec/operations/mastodon_admin/report_operation.ex b/lib/pleroma/web/api_spec/operations/mastodon_admin/report_operation.ex index 3360cb843..f59db9966 100644 --- a/lib/pleroma/web/api_spec/operations/mastodon_admin/report_operation.ex +++ b/lib/pleroma/web/api_spec/operations/mastodon_admin/report_operation.ex @@ -28,6 +28,7 @@ defmodule Pleroma.Web.ApiSpec.MastodonAdmin.ReportOperation do parameters: [ Operation.parameter(:resolved, :query, :boolean, "Filter for resolved reports"), + Operation.parameter(:unresolved, :query, :boolean, "Filter for open reports"), Operation.parameter(:account_id, :query, :string, "Filter by author account id"), Operation.parameter( :target_account_id, diff --git a/lib/pleroma/web/mastodon_api/admin/controllers/report_controller.ex b/lib/pleroma/web/mastodon_api/admin/controllers/report_controller.ex index 2d89d13bc..5dbf19aba 100644 --- a/lib/pleroma/web/mastodon_api/admin/controllers/report_controller.ex +++ b/lib/pleroma/web/mastodon_api/admin/controllers/report_controller.ex @@ -89,11 +89,46 @@ defmodule Pleroma.Web.MastodonAPI.Admin.ReportController do end end + def assign_to_self(%{assigns: %{user: admin}} = conn, %{id: id}) do + with {:ok, activity} <- CommonAPI.assign_report_to_account(id, admin.id), + report <- Activity.get_by_id_with_user_actor(activity.id) do + ModerationLog.insert_log(%{ + action: "report_assigned", + actor: admin, + subject: activity, + subject_actor: report.user_actor, + assigned_account: admin.nickname + }) + + render(conn, "show.json", Report.extract_report_info(report)) + else + {:error, error} -> + json_response(conn, :bad_request, %{error: error}) + end + end + + def unassign(%{assigns: %{user: admin}} = conn, %{id: id}) do + with {:ok, activity} <- CommonAPI.assign_report_to_account(id, nil), + report <- Activity.get_by_id_with_user_actor(activity.id) do + ModerationLog.insert_log(%{ + action: "report_unassigned", + actor: admin, + subject: activity, + subject_actor: report.user_actor + }) + + render(conn, "show.json", Report.extract_report_info(report)) + else + {:error, error} -> + json_response(conn, :bad_request, %{error: error}) + end + end + defp restrict_state(opts, %{resolved: true}), do: Map.put(opts, :state, "resolved") - defp restrict_state(opts, %{resolved: false}), do: Map.put(opts, :state, "open") + defp restrict_state(opts, %{resolved: _}), do: Map.put(opts, :state, "resolved") - defp restrict_state(opts, _params), do: opts + defp restrict_state(opts, _params), do: Map.put(opts, :state, "open") defp restrict_actor(opts, %{account_id: actor}) do with %User{ap_id: ap_id} <- User.get_by_id(actor) do diff --git a/test/pleroma/web/mastodon_api/admin/controllers/report_controller_test.exs b/test/pleroma/web/mastodon_api/admin/controllers/report_controller_test.exs index a0cc5f459..702dcf1db 100644 --- a/test/pleroma/web/mastodon_api/admin/controllers/report_controller_test.exs +++ b/test/pleroma/web/mastodon_api/admin/controllers/report_controller_test.exs @@ -50,7 +50,17 @@ defmodule Pleroma.Web.MastodonAPI.Admin.ReportControllerTest do assert [%{"id" => ^report_id1}] = conn - |> get("/api/v1/admin/reports?resolved=false") + |> get("/api/v1/admin/reports?unresolved=true") + |> json_response_and_validate_schema(200) + + assert [%{"id" => ^report_id1}] = + conn + |> get("/api/v1/admin/reports") + |> json_response_and_validate_schema(200) + + assert [%{"id" => ^report_id2}, %{"id" => ^report_id1}] = + conn + |> get("/api/v1/admin/reports?resolved=true&unresolved=true") |> json_response_and_validate_schema(200) assert [%{"id" => ^report_id2}] = From cd28c2089c2fddce4e480bd097830ce22f923c47 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Mon, 25 May 2026 18:45:47 +0400 Subject: [PATCH 43/90] Fix ParadeDB search indexing --- changelog.d/paradedb-search.add | 2 +- config/config.exs | 2 +- docs/configuration/search.md | 8 +- lib/pleroma/search.ex | 74 +++++++++------ .../search/{paradedb.ex => parade_db.ex} | 15 +-- .../search/{paradedb => parade_db}/client.ex | 0 .../search/{paradedb => parade_db}/repo.ex | 0 .../activity_pub/mrf/anti_followbot_policy.ex | 6 +- .../web/activity_pub/mrf/simple_policy.ex | 2 +- .../web/mastodon_api/views/status_view.ex | 4 +- lib/pleroma/workers/mailer_worker.ex | 4 +- test/pleroma/search/meilisearch_test.exs | 2 +- ...est.exs => parade_db_integration_test.exs} | 0 .../{paradedb_test.exs => parade_db_test.exs} | 20 ++-- test/pleroma/search_test.exs | 94 +++++++++++++++++-- 15 files changed, 175 insertions(+), 58 deletions(-) rename lib/pleroma/search/{paradedb.ex => parade_db.ex} (96%) rename lib/pleroma/search/{paradedb => parade_db}/client.ex (100%) rename lib/pleroma/search/{paradedb => parade_db}/repo.ex (100%) rename test/pleroma/search/{paradedb_integration_test.exs => parade_db_integration_test.exs} (100%) rename test/pleroma/search/{paradedb_test.exs => parade_db_test.exs} (94%) diff --git a/changelog.d/paradedb-search.add b/changelog.d/paradedb-search.add index 00a6e2361..f72ebca14 100644 --- a/changelog.d/paradedb-search.add +++ b/changelog.d/paradedb-search.add @@ -1 +1 @@ -Add ParadeDB (pg_search) as an optional external search backend. +Add ParadeDB (pg_search) as an optional external search backend, indexing status content, content warnings, and attachment descriptions. diff --git a/config/config.exs b/config/config.exs index 2e405413c..3c308f093 100644 --- a/config/config.exs +++ b/config/config.exs @@ -946,7 +946,7 @@ config :pleroma, Pleroma.Search.ParadeDB, fuzzy_distance: 0 config :pleroma, Pleroma.Search.ParadeDB.Repo, - pool_size: 10, + pool_size: 2, prepare: :unnamed config :pleroma, Pleroma.Application, diff --git a/docs/configuration/search.md b/docs/configuration/search.md index c0d3f8462..1d5056ddf 100644 --- a/docs/configuration/search.md +++ b/docs/configuration/search.md @@ -85,7 +85,7 @@ You will see a "Default Admin API Key", this is the key you actually put into yo ### Initial indexing -After setting up the configuration, you'll want to index all of your already existing posts. Only public posts are indexed. You'll only +After setting up the configuration, you'll want to index all of your already existing posts. Only public/unlisted posts are indexed. You'll only have to do it one time, but it might take a while, depending on the amount of posts your instance has seen. This is also a fairly RAM consuming process for `meilisearch`, and it will take a lot of RAM when running if you have a lot of posts (seems to be around 5G for ~1.2 million posts while idle and up to 7G while indexing initially, but your experience may be different). @@ -153,8 +153,10 @@ external search backend by pointing Pleroma at a separate Postgres instance with Pleroma will maintain a small `pleroma_search_documents` table in that database (via the existing search indexing queue) and run search queries against it. +The indexed text includes status content, content warnings, and attachment descriptions. To use it, set the search module to `Pleroma.Search.ParadeDB` and configure the ParadeDB database URL: +These settings must be in static/runtime config (or `PARADEDB_DATABASE_URL`) so the dedicated ParadeDB Repo is started with the application. > config :pleroma, Pleroma.Search, module: Pleroma.Search.ParadeDB > @@ -162,6 +164,10 @@ To use it, set the search module to `Pleroma.Search.ParadeDB` and configure the > url: System.get_env("PARADEDB_DATABASE_URL") || "postgres://postgres:postgres@127.0.0.1:5432/paradedb", > table: "pleroma_search_documents", > fuzzy_distance: 0 +> +> config :pleroma, Pleroma.Search.ParadeDB.Repo, +> pool_size: 2, +> prepare: :unnamed Then, create the table and BM25 index once: diff --git a/lib/pleroma/search.ex b/lib/pleroma/search.ex index 9cd2768c4..cd4907026 100644 --- a/lib/pleroma/search.ex +++ b/lib/pleroma/search.ex @@ -5,13 +5,12 @@ defmodule Pleroma.Search do alias Pleroma.Workers.SearchIndexingWorker @spec add_to_index(Activity.t()) :: {:ok, Oban.Job.t() | :noop} | {:error, Oban.Job.changeset()} - def add_to_index(%Activity{id: activity_id, object: %Object{} = object} = activity) do - with {_, true} <- {:indexable, indexable?(activity)}, - {_, "public"} <- {:visibility, Visibility.get_visibility(object)} do + def add_to_index(%Activity{id: activity_id, object: %Object{}} = activity) do + if indexable?(activity) do SearchIndexingWorker.new(%{"op" => "add_to_index", "activity" => activity_id}) |> Oban.insert() else - _ -> {:ok, :noop} + {:ok, :noop} end end @@ -41,40 +40,59 @@ defmodule Pleroma.Search do def object_to_search_data(%Object{} = object) do data = object.data - content_str = - case data["content"] do - [nil | rest] -> to_string(rest) - str -> str - end - content = - with {:ok, scrubbed} <- - FastSanitize.Sanitizer.scrub(content_str, Pleroma.HTML.Scrubber.SearchIndexing), - trimmed <- String.trim(scrubbed) do - trimmed - end - - # Make sure we have a non-empty string - if content != "" do - {:ok, published, _} = DateTime.from_iso8601(data["published"]) + data + |> search_texts() + |> Enum.map(&sanitize_search_text/1) + |> Enum.reject(&(&1 in ["", "."])) + |> Enum.join(" ") + with true <- content != "", + published when is_binary(published) <- data["published"], + {:ok, published, _} <- DateTime.from_iso8601(published) do %{ id: object.id, content: content, ap: data["id"], published: published |> DateTime.to_unix() } + else + _ -> nil end end - defp indexable?(%Activity{ - data: %{"type" => "Create"}, - object: %Object{ - data: %{"content" => content, "published" => published, "type" => "Note"} - } - }) - when not is_nil(content) and content not in ["", "."] and not is_nil(published), - do: true + def indexable?(%Activity{ + data: %{"type" => "Create"}, + object: %Object{data: %{"published" => published, "type" => "Note"}} = object + }) + when not is_nil(published) do + Visibility.get_visibility(object) in ["public", "unlisted"] and + not is_nil(object_to_search_data(object)) + end - defp indexable?(_), do: false + def indexable?(_), do: false + + defp search_texts(data) do + [data["content"], data["summary"] | attachment_names(data["attachment"])] + end + + defp attachment_names(attachments) when is_list(attachments) do + Enum.map(attachments, fn + %{"name" => name} -> name + _ -> nil + end) + end + + defp attachment_names(_), do: [] + + defp sanitize_search_text([nil | rest]), do: sanitize_search_text(to_string(rest)) + + defp sanitize_search_text(text) when is_binary(text) do + case FastSanitize.Sanitizer.scrub(text, Pleroma.HTML.Scrubber.SearchIndexing) do + {:ok, scrubbed} -> String.trim(scrubbed) + _ -> "" + end + end + + defp sanitize_search_text(_), do: "" end diff --git a/lib/pleroma/search/paradedb.ex b/lib/pleroma/search/parade_db.ex similarity index 96% rename from lib/pleroma/search/paradedb.ex rename to lib/pleroma/search/parade_db.ex index 20be5b6f9..ee00e74af 100644 --- a/lib/pleroma/search/paradedb.ex +++ b/lib/pleroma/search/parade_db.ex @@ -66,9 +66,10 @@ defmodule Pleroma.Search.ParadeDB do @impl true def drop_index do - with {:ok, _} <- client_impl().query("DROP TABLE IF EXISTS #{table()}", []) do - :ok - else + case client_impl().query("DROP TABLE IF EXISTS #{table()}", []) do + {:ok, _} -> + :ok + {:error, error} -> Logger.error("ParadeDB drop_index failed: #{inspect(error)}") {:error, error} @@ -80,9 +81,9 @@ defmodule Pleroma.Search.ParadeDB do @impl true def add_to_index(activity) do - maybe_search_data = Search.object_to_search_data(activity.object) - - if activity.data["type"] == "Create" and maybe_search_data do + with true <- Search.indexable?(activity), + maybe_search_data when not is_nil(maybe_search_data) <- + Search.object_to_search_data(activity.object) do dumped_activity_id = dump_activity_id(activity.id) actor_ap_id = activity.data["actor"] published_at = DateTime.from_unix!(maybe_search_data.published) @@ -122,7 +123,7 @@ defmodule Pleroma.Search.ParadeDB do end end else - :ok + _ -> :ok end end diff --git a/lib/pleroma/search/paradedb/client.ex b/lib/pleroma/search/parade_db/client.ex similarity index 100% rename from lib/pleroma/search/paradedb/client.ex rename to lib/pleroma/search/parade_db/client.ex diff --git a/lib/pleroma/search/paradedb/repo.ex b/lib/pleroma/search/parade_db/repo.ex similarity index 100% rename from lib/pleroma/search/paradedb/repo.ex rename to lib/pleroma/search/parade_db/repo.ex diff --git a/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex b/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex index 8ea61aec2..d0320ce49 100644 --- a/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex @@ -24,7 +24,11 @@ defmodule Pleroma.Web.ActivityPub.MRF.AntiFollowbotPolicy do defp score_displayname("fedibot"), do: 1.0 defp score_displayname(_), do: 0.0 - defp determine_if_followbot(%User{nickname: nickname, name: displayname, actor_type: actor_type}) do + defp determine_if_followbot(%User{ + nickname: nickname, + name: displayname, + actor_type: actor_type + }) do # nickname will be a binary string except when following a relay nick_score = if is_binary(nickname) do diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex index a97e8db7b..93f0616ec 100644 --- a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex @@ -108,7 +108,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do end defp intersection(list1, list2) do - list1 -- list1 -- list2 + list1 -- (list1 -- list2) end defp check_followers_only(%{host: actor_host} = _actor_info, activity) do diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 6949c5f2d..5b18261a7 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -680,7 +680,9 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do %{ancestors: ancestors, descendants: descendants} = activities |> Enum.reverse() - |> Enum.group_by(fn %{id: id} -> if id < activity.id, do: :ancestors, else: :descendants end) + |> Enum.group_by(fn %{id: id} -> + if id < activity.id, do: :ancestors, else: :descendants + end) |> Map.put_new(:ancestors, []) |> Map.put_new(:descendants, []) diff --git a/lib/pleroma/workers/mailer_worker.ex b/lib/pleroma/workers/mailer_worker.ex index b0259b191..9ef6a80c9 100644 --- a/lib/pleroma/workers/mailer_worker.ex +++ b/lib/pleroma/workers/mailer_worker.ex @@ -6,7 +6,9 @@ defmodule Pleroma.Workers.MailerWorker do use Oban.Worker, queue: :background @impl true - def perform(%Job{args: %{"op" => "email", "encoded_email" => encoded_email, "config" => config}}) do + def perform(%Job{ + args: %{"op" => "email", "encoded_email" => encoded_email, "config" => config} + }) do encoded_email |> Base.decode64!() |> :erlang.binary_to_term() diff --git a/test/pleroma/search/meilisearch_test.exs b/test/pleroma/search/meilisearch_test.exs index ff32491c5..6e4a42c95 100644 --- a/test/pleroma/search/meilisearch_test.exs +++ b/test/pleroma/search/meilisearch_test.exs @@ -99,7 +99,7 @@ defmodule Pleroma.Search.MeilisearchTest do %{method: :delete, url: "http://127.0.0.1:7700/indexes/objects/documents/" <> id} -> send(self(), "called_delete") - assert String.length(id) > 1 + assert id != "" json(%{}) end) diff --git a/test/pleroma/search/paradedb_integration_test.exs b/test/pleroma/search/parade_db_integration_test.exs similarity index 100% rename from test/pleroma/search/paradedb_integration_test.exs rename to test/pleroma/search/parade_db_integration_test.exs diff --git a/test/pleroma/search/paradedb_test.exs b/test/pleroma/search/parade_db_test.exs similarity index 94% rename from test/pleroma/search/paradedb_test.exs rename to test/pleroma/search/parade_db_test.exs index 3a3b8298f..3133b6b38 100644 --- a/test/pleroma/search/paradedb_test.exs +++ b/test/pleroma/search/parade_db_test.exs @@ -75,15 +75,21 @@ defmodule Pleroma.Search.ParadeDBTest do }) args = %{"op" => "add_to_index", "activity" => activity.id} - assert_enqueued(worker: SearchIndexingWorker, args: args) + refute_enqueued(worker: SearchIndexingWorker, args: args) + end) + end - Config - |> expect(:get, fn - [Pleroma.Search, :module], nil -> - ParadeDB - end) + test "backend skips non-public posts when called directly" do + user = insert(:user) - assert :ok = perform_job(SearchIndexingWorker, args) + Enum.each(["private", "direct"], fn visibility -> + {:ok, activity} = + CommonAPI.post(user, %{ + status: "guys i just don't wanna leave the swamp", + visibility: visibility + }) + + assert :ok = ParadeDB.add_to_index(activity) end) end diff --git a/test/pleroma/search_test.exs b/test/pleroma/search_test.exs index d777bcda2..00d8b10ed 100644 --- a/test/pleroma/search_test.exs +++ b/test/pleroma/search_test.exs @@ -3,26 +3,32 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.SearchTest do + require Pleroma.Constants + use Pleroma.DataCase, async: true use Oban.Testing, repo: Pleroma.Repo import Pleroma.Factory + alias Pleroma.Object + alias Pleroma.Search alias Pleroma.Web.CommonAPI alias Pleroma.Workers.SearchIndexingWorker - test "indexes posts that are public" do + test "indexes posts that are public or unlisted" do user = insert(:user) - {:ok, activity} = - CommonAPI.post(user, %{ - status: "Well this is a story all about how my life got flipped turned upside down", - visibility: "public" - }) + Enum.each(["public", "unlisted"], fn visibility -> + {:ok, activity} = + CommonAPI.post(user, %{ + status: "Well this is a story all about how my life got flipped turned upside down", + visibility: visibility + }) - args = %{"op" => "add_to_index", "activity" => activity.id} + args = %{"op" => "add_to_index", "activity" => activity.id} - assert_enqueued(worker: SearchIndexingWorker, args: args) + assert_enqueued(worker: SearchIndexingWorker, args: args) + end) end test "doesn't index posts that are not public" do @@ -66,4 +72,76 @@ defmodule Pleroma.SearchTest do refute_enqueued(worker: SearchIndexingWorker, args: args) end + + test "object_to_search_data includes content warnings and attachment descriptions" do + object = %Object{ + id: 1, + data: %{ + "id" => "https://example.com/objects/1", + "type" => "Note", + "content" => "body text", + "summary" => "subject text", + "attachment" => [ + %{"type" => "Document", "name" => "image description"}, + %{"type" => "Document", "name" => ""} + ], + "published" => DateTime.utc_now() |> DateTime.to_iso8601() + } + } + + assert %{content: content} = Search.object_to_search_data(object) + assert content =~ "body text" + assert content =~ "subject text" + assert content =~ "image description" + end + + test "object_to_search_data ignores objects with invalid published dates" do + for published <- [nil, 123, "not a date"] do + object = %Object{ + id: 1, + data: %{ + "id" => "https://example.com/objects/1", + "type" => "Note", + "content" => "body text", + "published" => published + } + } + + refute Search.object_to_search_data(object) + end + end + + test "indexable? accepts searchable content warnings and attachment descriptions" do + public = Pleroma.Constants.as_public() + published = DateTime.utc_now() |> DateTime.to_iso8601() + + for data <- [ + %{"content" => ".", "summary" => "subject text", "attachment" => []}, + %{ + "content" => ".", + "summary" => "", + "attachment" => [%{"type" => "Document", "name" => "image description"}] + } + ] do + object = %Object{ + id: 1, + data: + Map.merge(data, %{ + "id" => "https://example.com/objects/1", + "type" => "Note", + "to" => [public], + "published" => published + }) + } + + activity = %Pleroma.Activity{ + id: Ecto.UUID.generate(), + data: %{"type" => "Create"}, + object: object, + recipients: [public] + } + + assert Search.indexable?(activity) + end + end end From afe33bcc44b2dede05b58135c302960e813c6560 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Mon, 25 May 2026 18:37:37 +0400 Subject: [PATCH 44/90] Fix webhook create test expectations --- .../controllers/webhook_controller_test.exs | 6 +++--- test/pleroma/webhook/notify_test.exs | 4 ++-- test/pleroma/webhook_test.exs | 19 ++++++++++--------- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/test/pleroma/web/admin_api/controllers/webhook_controller_test.exs b/test/pleroma/web/admin_api/controllers/webhook_controller_test.exs index 0cd00e753..0e3b5311e 100644 --- a/test/pleroma/web/admin_api/controllers/webhook_controller_test.exs +++ b/test/pleroma/web/admin_api/controllers/webhook_controller_test.exs @@ -52,7 +52,7 @@ defmodule Pleroma.Web.AdminAPI.WebhookControllerTest do describe "PATCH /api/pleroma/admin/webhooks" do test "edits a webhook", %{conn: conn} do - %{id: id} = + {:ok, %{id: id}} = Webhook.create(%{url: "https://example.com/webhook1", events: [:"report.created"]}) conn @@ -66,7 +66,7 @@ defmodule Pleroma.Web.AdminAPI.WebhookControllerTest do end test "can't edit an internal webhook", %{conn: conn} do - %{id: id} = + {:ok, %{id: id}} = Webhook.create(%{url: "https://example.com/webhook1", events: [], internal: true}) conn @@ -82,7 +82,7 @@ defmodule Pleroma.Web.AdminAPI.WebhookControllerTest do describe "DELETE /api/pleroma/admin/webhooks" do test "deletes a webhook", %{conn: conn} do - %{id: id} = + {:ok, %{id: id}} = Webhook.create(%{url: "https://example.com/webhook1", events: [:"report.created"]}) conn diff --git a/test/pleroma/webhook/notify_test.exs b/test/pleroma/webhook/notify_test.exs index c99489940..568ed0d14 100644 --- a/test/pleroma/webhook/notify_test.exs +++ b/test/pleroma/webhook/notify_test.exs @@ -59,8 +59,8 @@ defmodule Pleroma.Webhook.NotifyTest do test "notifies have a valid signature" do activity = insert(:report_activity) - %{secret: secret} = - webhook = Webhook.create(%{url: "https://example.com/webhook", events: [:"report.created"]}) + {:ok, %{secret: secret} = webhook} = + Webhook.create(%{url: "https://example.com/webhook", events: [:"report.created"]}) Tesla.Mock.mock(fn %{url: "https://example.com/webhook", body: body, headers: headers} = _ -> {"X-Hub-Signature", "sha256=" <> signature} = diff --git a/test/pleroma/webhook_test.exs b/test/pleroma/webhook_test.exs index ea1378730..ea1408708 100644 --- a/test/pleroma/webhook_test.exs +++ b/test/pleroma/webhook_test.exs @@ -8,14 +8,15 @@ defmodule Pleroma.WebhookTest do alias Pleroma.Webhook test "creating a webhook" do - %{id: id} = Webhook.create(%{url: "https://example.com/webhook", events: [:"report.created"]}) + {:ok, %{id: id}} = + Webhook.create(%{url: "https://example.com/webhook", events: [:"report.created"]}) assert %{url: "https://example.com/webhook"} = Webhook.get(id) end test "editing a webhook" do - %{id: id} = - webhook = Webhook.create(%{url: "https://example.com/webhook", events: [:"report.created"]}) + {:ok, %{id: id} = webhook} = + Webhook.create(%{url: "https://example.com/webhook", events: [:"report.created"]}) Webhook.update(webhook, %{events: [:"account.created"]}) @@ -23,10 +24,10 @@ defmodule Pleroma.WebhookTest do end test "filter webhooks by type" do - %{id: id1} = + {:ok, %{id: id1}} = Webhook.create(%{url: "https://example.com/webhook1", events: [:"report.created"]}) - %{id: id2} = + {:ok, %{id: id2}} = Webhook.create(%{ url: "https://example.com/webhook2", events: [:"account.created", :"report.created"] @@ -38,16 +39,16 @@ defmodule Pleroma.WebhookTest do end test "change webhook state" do - %{id: id, enabled: true} = - webhook = Webhook.create(%{url: "https://example.com/webhook", events: [:"report.created"]}) + {:ok, %{id: id, enabled: true} = webhook} = + Webhook.create(%{url: "https://example.com/webhook", events: [:"report.created"]}) Webhook.set_enabled(webhook, false) assert %{enabled: false} = Webhook.get(id) end test "rotate webhook secrets" do - %{id: id, secret: secret} = - webhook = Webhook.create(%{url: "https://example.com/webhook", events: [:"report.created"]}) + {:ok, %{id: id, secret: secret} = webhook} = + Webhook.create(%{url: "https://example.com/webhook", events: [:"report.created"]}) Webhook.rotate_secret(webhook) %{secret: new_secret} = Webhook.get(id) From e44d2329c8d7a9815cb74890b6c7299fc0b424b9 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 26 May 2026 14:29:08 +0400 Subject: [PATCH 45/90] Reject invalid webhook events --- lib/pleroma/webhook.ex | 12 ------------ test/pleroma/webhook_test.exs | 10 ++++++++++ 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/lib/pleroma/webhook.ex b/lib/pleroma/webhook.ex index 83ce344fa..9a1806f71 100644 --- a/lib/pleroma/webhook.ex +++ b/lib/pleroma/webhook.ex @@ -37,7 +37,6 @@ defmodule Pleroma.Webhook do |> cast(params, [:url, :events, :enabled, :internal]) |> validate_required([:url, :events]) |> unique_constraint(:url) - |> strip_events() |> put_secret() end @@ -45,7 +44,6 @@ defmodule Pleroma.Webhook do webhook |> cast(params, [:url, :events, :enabled, :internal]) |> unique_constraint(:url) - |> strip_events() end def create(params) do @@ -75,16 +73,6 @@ defmodule Pleroma.Webhook do |> Repo.update() end - defp strip_events(changeset) do - case get_change(changeset, :events) do - nil -> - changeset - - events -> - put_change(changeset, :events, Enum.filter(events, &(&1 in @event_types))) - end - end - defp put_secret(changeset) do changeset |> put_change(:secret, generate_secret()) diff --git a/test/pleroma/webhook_test.exs b/test/pleroma/webhook_test.exs index ea1408708..1f64d350f 100644 --- a/test/pleroma/webhook_test.exs +++ b/test/pleroma/webhook_test.exs @@ -23,6 +23,16 @@ defmodule Pleroma.WebhookTest do assert %{events: [:"account.created"]} = Webhook.get(id) end + test "rejects invalid events" do + assert {:error, _changeset} = + Webhook.create(%{url: "https://example.com/webhook", events: [:"status.created"]}) + + {:ok, webhook} = + Webhook.create(%{url: "https://example.com/valid-webhook", events: [:"report.created"]}) + + assert {:error, _changeset} = Webhook.update(webhook, %{events: [:"status.created"]}) + end + test "filter webhooks by type" do {:ok, %{id: id1}} = Webhook.create(%{url: "https://example.com/webhook1", events: [:"report.created"]}) From 719b5d65b545403adf5fcfbec5571936b5165be1 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 26 May 2026 14:30:50 +0400 Subject: [PATCH 46/90] Require webhook events --- docs/development/API/admin_api.md | 2 +- .../operations/admin/webhook_operation.ex | 10 +++++----- lib/pleroma/webhook.ex | 11 +++++++++++ test/pleroma/webhook_test.exs | 18 ++++++++++++++++++ 4 files changed, 35 insertions(+), 6 deletions(-) diff --git a/docs/development/API/admin_api.md b/docs/development/API/admin_api.md index a30db9bad..ec58e8c19 100644 --- a/docs/development/API/admin_api.md +++ b/docs/development/API/admin_api.md @@ -1877,7 +1877,7 @@ Note that this differs from the Mastodon API variant: Mastodon API only returns - Method: `POST` - Params: - `url`: **string** Webhook URL - - *optional* `events`: **[string]** Types of events to trigger on (`account.created`, `report.created`) + - `events`: **[string]** Types of events to trigger on (`account.created`, `report.created`) - *optional* `enabled`: **boolean** Whether webhook is enabled - Response: A webhook diff --git a/lib/pleroma/web/api_spec/operations/admin/webhook_operation.ex b/lib/pleroma/web/api_spec/operations/admin/webhook_operation.ex index fd5f3a561..e2869ac44 100644 --- a/lib/pleroma/web/api_spec/operations/admin/webhook_operation.ex +++ b/lib/pleroma/web/api_spec/operations/admin/webhook_operation.ex @@ -55,9 +55,10 @@ defmodule Pleroma.Web.ApiSpec.Admin.WebhookOperation do %Schema{ description: "POST body for creating a webhook", type: :object, + required: [:url, :events], properties: %{ - url: %Schema{type: :string, format: :uri, required: true}, - events: event_type(true), + url: %Schema{type: :string, format: :uri}, + events: event_type(), enabled: %Schema{type: :boolean} } } @@ -179,15 +180,14 @@ defmodule Pleroma.Web.ApiSpec.Admin.WebhookOperation do } end - defp event_type(required \\ nil) do + defp event_type do %Schema{ type: :array, items: %Schema{ title: "Event", description: "Event type", type: :string, - enum: ["account.created", "report.created"], - required: required + enum: ["account.created", "report.created"] } } end diff --git a/lib/pleroma/webhook.ex b/lib/pleroma/webhook.ex index 9a1806f71..f4d65b1a6 100644 --- a/lib/pleroma/webhook.ex +++ b/lib/pleroma/webhook.ex @@ -36,6 +36,7 @@ defmodule Pleroma.Webhook do webhook |> cast(params, [:url, :events, :enabled, :internal]) |> validate_required([:url, :events]) + |> validate_events_present() |> unique_constraint(:url) |> put_secret() end @@ -43,6 +44,7 @@ defmodule Pleroma.Webhook do def update_changeset(%__MODULE__{} = webhook, params \\ %{}) do webhook |> cast(params, [:url, :events, :enabled, :internal]) + |> validate_events_present() |> unique_constraint(:url) end @@ -73,6 +75,15 @@ defmodule Pleroma.Webhook do |> Repo.update() end + defp validate_events_present(changeset) do + cond do + Keyword.has_key?(changeset.errors, :events) -> changeset + get_field(changeset, :internal) -> changeset + match?([_ | _], get_field(changeset, :events)) -> changeset + true -> add_error(changeset, :events, "can't be blank") + end + end + defp put_secret(changeset) do changeset |> put_change(:secret, generate_secret()) diff --git a/test/pleroma/webhook_test.exs b/test/pleroma/webhook_test.exs index 1f64d350f..df84abef6 100644 --- a/test/pleroma/webhook_test.exs +++ b/test/pleroma/webhook_test.exs @@ -33,6 +33,24 @@ defmodule Pleroma.WebhookTest do assert {:error, _changeset} = Webhook.update(webhook, %{events: [:"status.created"]}) end + test "rejects empty events for non-internal webhooks" do + assert {:error, changeset} = Webhook.create(%{url: "https://example.com/webhook", events: []}) + assert %{events: ["can't be blank"]} = errors_on(changeset) + + {:ok, webhook} = + Webhook.create(%{url: "https://example.com/valid-webhook", events: [:"report.created"]}) + + assert {:error, changeset} = Webhook.update(webhook, %{events: []}) + assert %{events: ["can't be blank"]} = errors_on(changeset) + + assert {:ok, _webhook} = + Webhook.create(%{ + url: "https://example.com/internal-webhook", + events: [], + internal: true + }) + end + test "filter webhooks by type" do {:ok, %{id: id1}} = Webhook.create(%{url: "https://example.com/webhook1", events: [:"report.created"]}) From 9061851fed33fde331a7dcda709406f48e79328c Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 26 May 2026 14:31:12 +0400 Subject: [PATCH 47/90] Fix webhook read API scope --- .../web/api_spec/operations/admin/webhook_operation.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/api_spec/operations/admin/webhook_operation.ex b/lib/pleroma/web/api_spec/operations/admin/webhook_operation.ex index e2869ac44..5bce837f7 100644 --- a/lib/pleroma/web/api_spec/operations/admin/webhook_operation.ex +++ b/lib/pleroma/web/api_spec/operations/admin/webhook_operation.ex @@ -19,7 +19,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.WebhookOperation do tags: ["Webhooks"], summary: "Retrieve a list of webhooks", operationId: "AdminAPI.WebhookController.index", - security: [%{"oAuth" => ["admin:show"]}], + security: [%{"oAuth" => ["admin:read"]}], responses: %{ 200 => Operation.response("Array of webhooks", "application/json", %Schema{ @@ -35,7 +35,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.WebhookOperation do tags: ["Webhooks"], summary: "Retrieve a webhook", operationId: "AdminAPI.WebhookController.show", - security: [%{"oAuth" => ["admin:show"]}], + security: [%{"oAuth" => ["admin:read"]}], parameters: [id_param()], responses: %{ 200 => Operation.response("Webhook", "application/json", webhook()) From 27090ad7e6042beeedb92ab151bbd2f418454198 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 26 May 2026 14:32:51 +0400 Subject: [PATCH 48/90] Require admin access for webhooks --- lib/pleroma/web/router.ex | 18 +++++++++--------- .../controllers/webhook_controller_test.exs | 15 ++++++++++++++- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 4b1c4140b..1800c5ede 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -316,6 +316,15 @@ defmodule Pleroma.Web.Router do post("/rules", RuleController, :create) patch("/rules/:id", RuleController, :update) delete("/rules/:id", RuleController, :delete) + + get("/webhooks", WebhookController, :index) + get("/webhooks/:id", WebhookController, :show) + post("/webhooks", WebhookController, :create) + patch("/webhooks/:id", WebhookController, :update) + delete("/webhooks/:id", WebhookController, :delete) + post("/webhooks/:id/enable", WebhookController, :enable) + post("/webhooks/:id/disable", WebhookController, :disable) + post("/webhooks/:id/rotate_secret", WebhookController, :rotate_secret) end # AdminAPI: admins and mods (staff) can perform these actions (if privileged by role) @@ -327,15 +336,6 @@ defmodule Pleroma.Web.Router do get("/announcements/:id", AnnouncementController, :show) patch("/announcements/:id", AnnouncementController, :change) delete("/announcements/:id", AnnouncementController, :delete) - - get("/webhooks", WebhookController, :index) - get("/webhooks/:id", WebhookController, :show) - post("/webhooks", WebhookController, :create) - patch("/webhooks/:id", WebhookController, :update) - delete("/webhooks/:id", WebhookController, :delete) - post("/webhooks/:id/enable", WebhookController, :enable) - post("/webhooks/:id/disable", WebhookController, :disable) - post("/webhooks/:id/rotate_secret", WebhookController, :rotate_secret) end # AdminAPI: admins and mods (staff) can perform these actions (if privileged by role) diff --git a/test/pleroma/web/admin_api/controllers/webhook_controller_test.exs b/test/pleroma/web/admin_api/controllers/webhook_controller_test.exs index 0e3b5311e..78edbf05a 100644 --- a/test/pleroma/web/admin_api/controllers/webhook_controller_test.exs +++ b/test/pleroma/web/admin_api/controllers/webhook_controller_test.exs @@ -3,7 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.AdminAPI.WebhookControllerTest do - use Pleroma.Web.ConnCase, async: true + use Pleroma.Web.ConnCase, async: false import Pleroma.Factory @@ -33,6 +33,19 @@ defmodule Pleroma.Web.AdminAPI.WebhookControllerTest do assert length(response) == 2 end + + test "requires an admin even if a moderator can manage announcements" do + clear_config([:instance, :moderator_privileges], [:announcements_manage_announcements]) + + moderator = insert(:user, is_moderator: true) + token = insert(:oauth_admin_token, user: moderator) + + build_conn() + |> assign(:user, moderator) + |> assign(:token, token) + |> get("/api/pleroma/admin/webhooks") + |> json_response(:forbidden) + end end describe "POST /api/pleroma/admin/webhooks" do From ab634a4aa35905948c99ebbca73bb96d009842d4 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 26 May 2026 15:03:22 +0400 Subject: [PATCH 49/90] Harden webhook API input --- .../controllers/webhook_controller.ex | 8 +++- .../operations/admin/webhook_operation.ex | 19 +++++--- .../controllers/webhook_controller_test.exs | 45 +++++++++++++++++++ 3 files changed, 64 insertions(+), 8 deletions(-) diff --git a/lib/pleroma/web/admin_api/controllers/webhook_controller.ex b/lib/pleroma/web/admin_api/controllers/webhook_controller.ex index cc606916e..4d064e0b5 100644 --- a/lib/pleroma/web/admin_api/controllers/webhook_controller.ex +++ b/lib/pleroma/web/admin_api/controllers/webhook_controller.ex @@ -40,14 +40,14 @@ defmodule Pleroma.Web.AdminAPI.WebhookController do end def create(%{body_params: params} = conn, _) do - with {:ok, webhook} <- Webhook.create(params) do + with {:ok, webhook} <- Webhook.create(webhook_params(params)) do render(conn, "show.json", webhook: webhook) end end def update(%{body_params: params} = conn, %{id: id}) do with %Webhook{internal: false} = webhook <- Webhook.get(id), - {:ok, webhook} <- Webhook.update(webhook, params) do + {:ok, webhook} <- Webhook.update(webhook, webhook_params(params)) do render(conn, "show.json", webhook: webhook) else %Webhook{internal: true} -> {:error, :forbidden} @@ -94,4 +94,8 @@ defmodule Pleroma.Web.AdminAPI.WebhookController do nil -> {:error, :not_found} end end + + defp webhook_params(params) do + Map.drop(params, [:internal, "internal"]) + end end diff --git a/lib/pleroma/web/api_spec/operations/admin/webhook_operation.ex b/lib/pleroma/web/api_spec/operations/admin/webhook_operation.ex index 5bce837f7..2b538ca8b 100644 --- a/lib/pleroma/web/api_spec/operations/admin/webhook_operation.ex +++ b/lib/pleroma/web/api_spec/operations/admin/webhook_operation.ex @@ -58,9 +58,10 @@ defmodule Pleroma.Web.ApiSpec.Admin.WebhookOperation do required: [:url, :events], properties: %{ url: %Schema{type: :string, format: :uri}, - events: event_type(), + events: event_type(min_items: 1), enabled: %Schema{type: :boolean} - } + }, + additionalProperties: false } ), responses: %{ @@ -84,9 +85,10 @@ defmodule Pleroma.Web.ApiSpec.Admin.WebhookOperation do type: :object, properties: %{ url: %Schema{type: :string, format: :uri}, - events: event_type(), + events: event_type(min_items: 1), enabled: %Schema{type: :boolean} - } + }, + additionalProperties: false } ), responses: %{ @@ -180,8 +182,8 @@ defmodule Pleroma.Web.ApiSpec.Admin.WebhookOperation do } end - defp event_type do - %Schema{ + defp event_type(opts \\ []) do + schema = %Schema{ type: :array, items: %Schema{ title: "Event", @@ -190,6 +192,11 @@ defmodule Pleroma.Web.ApiSpec.Admin.WebhookOperation do enum: ["account.created", "report.created"] } } + + case Keyword.fetch(opts, :min_items) do + {:ok, min_items} -> %{schema | minItems: min_items} + :error -> schema + end end defp id_param do diff --git a/test/pleroma/web/admin_api/controllers/webhook_controller_test.exs b/test/pleroma/web/admin_api/controllers/webhook_controller_test.exs index 78edbf05a..800d6367e 100644 --- a/test/pleroma/web/admin_api/controllers/webhook_controller_test.exs +++ b/test/pleroma/web/admin_api/controllers/webhook_controller_test.exs @@ -61,6 +61,27 @@ defmodule Pleroma.Web.AdminAPI.WebhookControllerTest do assert %{url: "http://example.com/webhook", events: [:"account.created"]} = Webhook.get(id) end + + test "rejects empty events", %{conn: conn} do + conn + |> put_req_header("content-type", "application/json") + |> post("/api/pleroma/admin/webhooks", %{ + url: "http://example.com/webhook", + events: [] + }) + |> json_response(:bad_request) + end + + test "rejects internal flag", %{conn: conn} do + conn + |> put_req_header("content-type", "application/json") + |> post("/api/pleroma/admin/webhooks", %{ + url: "http://example.com/webhook", + events: ["account.created"], + internal: true + }) + |> json_response(:bad_request) + end end describe "PATCH /api/pleroma/admin/webhooks" do @@ -78,6 +99,30 @@ defmodule Pleroma.Web.AdminAPI.WebhookControllerTest do assert %{events: [:"report.created", :"account.created"]} = Webhook.get(id) end + test "rejects empty events", %{conn: conn} do + {:ok, %{id: id}} = + Webhook.create(%{url: "https://example.com/webhook1", events: [:"report.created"]}) + + conn + |> put_req_header("content-type", "application/json") + |> patch("/api/pleroma/admin/webhooks/#{id}", %{events: []}) + |> json_response(:bad_request) + + assert %{events: [:"report.created"]} = Webhook.get(id) + end + + test "rejects internal flag", %{conn: conn} do + {:ok, %{id: id}} = + Webhook.create(%{url: "https://example.com/webhook1", events: [:"report.created"]}) + + conn + |> put_req_header("content-type", "application/json") + |> patch("/api/pleroma/admin/webhooks/#{id}", %{internal: true}) + |> json_response(:bad_request) + + assert %{internal: false} = Webhook.get(id) + end + test "can't edit an internal webhook", %{conn: conn} do {:ok, %{id: id}} = Webhook.create(%{url: "https://example.com/webhook1", events: [], internal: true}) From e676ec350713d766b653f60f906fceec34c2c6a1 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Fri, 29 May 2026 18:35:15 +0400 Subject: [PATCH 50/90] Reject unsupported HTTP Message Signatures --- .../activitypub-bot-rfc9421-fallback.fix | 1 + .../activity_pub/activity_pub_controller.ex | 33 ++++++-- .../activity_pub_controller_test.exs | 77 +++++++++++++++++++ 3 files changed, 103 insertions(+), 8 deletions(-) create mode 100644 changelog.d/activitypub-bot-rfc9421-fallback.fix diff --git a/changelog.d/activitypub-bot-rfc9421-fallback.fix b/changelog.d/activitypub-bot-rfc9421-fallback.fix new file mode 100644 index 000000000..22642fee1 --- /dev/null +++ b/changelog.d/activitypub-bot-rfc9421-fallback.fix @@ -0,0 +1 @@ +Reject unsupported RFC 9421 inbox signatures instead of queueing a doomed failed-signature retry, allowing clients to fall back to legacy HTTP Signatures. diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index 3d1875599..dd644c132 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -353,15 +353,21 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do end def inbox(%{assigns: %{valid_signature: false}} = conn, params) do - Federator.incoming_failed_signature_ap_doc(%{ - method: conn.method, - req_headers: conn.req_headers, - request_path: conn.request_path, - params: params, - query_string: conn.query_string - }) + if unsupported_http_message_signature?(conn) do + conn + |> put_status(:unauthorized) + |> json("error, unsupported HTTP Message Signature") + else + Federator.incoming_failed_signature_ap_doc(%{ + method: conn.method, + req_headers: conn.req_headers, + request_path: conn.request_path, + params: params, + query_string: conn.query_string + }) - json(conn, "ok") + json(conn, "ok") + end end # POST /relay/inbox -or- POST /internal/fetch/inbox @@ -381,6 +387,17 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do |> json("error, missing HTTP Signature") end + defp unsupported_http_message_signature?(conn) do + Plug.Conn.get_req_header(conn, "signature-input") != [] and + not cavage_signature?(conn) + end + + defp cavage_signature?(conn) do + conn + |> Plug.Conn.get_req_header("signature") + |> Enum.any?(&Regex.match?(~r/(^|,)\s*keyId\s*=/, &1)) + end + defp post_inbox_relayed_create(conn, params) do Logger.debug( "Signature missing or not from author, relayed Create message, fetching object from source" diff --git a/test/pleroma/web/activity_pub/activity_pub_controller_test.exs b/test/pleroma/web/activity_pub/activity_pub_controller_test.exs index 68e39b3a0..e9a777758 100644 --- a/test/pleroma/web/activity_pub/activity_pub_controller_test.exs +++ b/test/pleroma/web/activity_pub/activity_pub_controller_test.exs @@ -757,6 +757,83 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do assert Activity.get_by_ap_id(data["id"]) end + test "rejects unsupported HTTP Message Signatures so clients can fall back", %{conn: conn} do + data = %{ + "type" => "Create", + "actor" => "https://activitypubbot.example/user/ok", + "id" => "https://activitypubbot.example/activities/rfc9421-create", + "to" => ["https://www.w3.org/ns/activitystreams#Public"], + "cc" => [], + "object" => %{ + "type" => "Note", + "id" => "https://activitypubbot.example/objects/rfc9421-note", + "actor" => "https://activitypubbot.example/user/ok", + "attributedTo" => "https://activitypubbot.example/user/ok", + "content" => "hello from activitypub-bot", + "published" => "2026-05-29T13:33:31Z", + "to" => ["https://www.w3.org/ns/activitystreams#Public"], + "cc" => [] + } + } + + Mox.expect(Pleroma.StubbedHTTPSignaturesMock, :validate_conn, fn _conn -> false end) + + conn = + conn + |> put_req_header("content-type", "application/activity+json") + |> put_req_header("date", "Fri, 29 May 2026 13:33:31 GMT") + |> put_req_header("content-digest", "sha-256=:fake-digest:") + |> put_req_header( + "signature-input", + "sig1=(\"@method\" \"@target-uri\" \"date\" \"content-type\" \"content-digest\");keyid=\"https://activitypubbot.example/user/ok/publickey\";alg=\"rsa-v1_5-sha256\";created=1780061611" + ) + |> put_req_header("signature", "sig1=:fake-signature:") + |> post("/inbox", data) + + assert "error, unsupported HTTP Message Signature" == json_response(conn, 401) + assert [] == all_enqueued(worker: SignatureRetryWorker) + end + + test "keeps failed-signature retry for legacy signatures with Signature-Input", %{conn: conn} do + data = %{ + "type" => "Create", + "actor" => "https://activitypubbot.example/user/ok", + "id" => "https://activitypubbot.example/activities/legacy-create", + "to" => ["https://www.w3.org/ns/activitystreams#Public"], + "cc" => [], + "object" => %{ + "type" => "Note", + "id" => "https://activitypubbot.example/objects/legacy-note", + "actor" => "https://activitypubbot.example/user/ok", + "attributedTo" => "https://activitypubbot.example/user/ok", + "content" => "hello from activitypub-bot", + "published" => "2026-05-29T13:33:31Z", + "to" => ["https://www.w3.org/ns/activitystreams#Public"], + "cc" => [] + } + } + + Mox.expect(Pleroma.StubbedHTTPSignaturesMock, :validate_conn, fn _conn -> false end) + + conn = + conn + |> put_req_header("content-type", "application/activity+json") + |> put_req_header("date", "Fri, 29 May 2026 13:33:31 GMT") + |> put_req_header("digest", "SHA-256=fake-digest") + |> put_req_header( + "signature-input", + "sig1=(\"@method\" \"@target-uri\" \"date\" \"content-type\" \"digest\");keyid=\"https://activitypubbot.example/user/ok/publickey\";alg=\"rsa-v1_5-sha256\";created=1780061611" + ) + |> put_req_header( + "signature", + "keyId=\"https://activitypubbot.example/user/ok#main-key\",algorithm=\"rsa-sha256\",headers=\"(request-target) host date digest content-type\",signature=\"fake-signature\"" + ) + |> post("/inbox", data) + + assert "ok" == json_response(conn, 200) + assert [_] = all_enqueued(worker: SignatureRetryWorker) + end + test "does not create a forged post after failed signature retry", %{conn: conn} do alice = insert(:user, local: false, ap_id: "https://one.com/users/alice") bob = insert(:user, local: false, ap_id: "https://two.com/users/bob") From 36bc7b116d7b66a47f19e53bba3631b010030876 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?nicole=20miko=C5=82ajczyk?= Date: Mon, 20 Jul 2026 19:01:02 +0200 Subject: [PATCH 51/90] Allow creating accounts without a password in the AdminAPI, update docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: nicole mikołajczyk --- .../admin-api-passwordless-accounts.add | 1 + docs/development/API/admin_api.md | 18 +++++++- .../admin_api/controllers/user_controller.ex | 43 ++++++++++++++++--- .../web/admin_api/views/account_view.ex | 30 ++++++++++--- .../operations/admin/user_operation.ex | 13 ++++-- .../controllers/user_controller_test.exs | 28 ++++++++++++ 6 files changed, 114 insertions(+), 19 deletions(-) create mode 100644 changelog.d/admin-api-passwordless-accounts.add diff --git a/changelog.d/admin-api-passwordless-accounts.add b/changelog.d/admin-api-passwordless-accounts.add new file mode 100644 index 000000000..f5ea2626f --- /dev/null +++ b/changelog.d/admin-api-passwordless-accounts.add @@ -0,0 +1 @@ +Allow creating accounts without a password in the AdminAPI diff --git a/docs/development/API/admin_api.md b/docs/development/API/admin_api.md index 3719ceeb9..ef04944af 100644 --- a/docs/development/API/admin_api.md +++ b/docs/development/API/admin_api.md @@ -80,10 +80,24 @@ The `/api/v1/pleroma/admin/*` path is backwards compatible with `/api/pleroma/ad { `nickname`, `email`, - `password` + `password` (optional) } ] -- Response: Array of user objects +- Response: Array of user objects. When a user is created without a `password`, `data` + object contains a `password_reset_link` the user can follow to set their own password. + +```json +[ + { + "data": { + "nickname": string, + "email": string, + "password_reset_link": string (optional) + }, + "type": "success" + } +] +``` ## `POST /api/v1/pleroma/admin/users/follow` diff --git a/lib/pleroma/web/admin_api/controllers/user_controller.ex b/lib/pleroma/web/admin_api/controllers/user_controller.ex index 9ac275396..23af5b53c 100644 --- a/lib/pleroma/web/admin_api/controllers/user_controller.ex +++ b/lib/pleroma/web/admin_api/controllers/user_controller.ex @@ -8,13 +8,16 @@ defmodule Pleroma.Web.AdminAPI.UserController do import Pleroma.Web.ControllerHelper, only: [fetch_integer_param: 3] + alias Ecto.Multi alias Pleroma.ModerationLog alias Pleroma.User alias Pleroma.Web.ActivityPub.Builder alias Pleroma.Web.ActivityPub.Pipeline alias Pleroma.Web.AdminAPI alias Pleroma.Web.AdminAPI.Search + alias Pleroma.Web.Endpoint alias Pleroma.Web.Plugs.OAuthScopesPlug + alias Pleroma.Web.Router @users_page_size 50 @@ -148,9 +151,24 @@ defmodule Pleroma.Web.AdminAPI.UserController do } = conn, _ ) do - changesets = + # Nicknames of users for which a password reset link will be generated. + passwordless_nicknames = users - |> Enum.map(fn %{nickname: nickname, email: email, password: password} -> + |> Enum.filter(fn user -> Map.get(user, :password) in ["", nil] end) + |> Enum.map(& &1.nickname) + + multi = + users + |> Enum.map(fn %{nickname: nickname, email: email} = attrs -> + password = + case Map.get(attrs, :password) do + password when is_binary(password) and password != "" -> + password + + _ -> + :crypto.strong_rand_bytes(16) |> Base.encode64() + end + user_data = %{ nickname: nickname, name: nickname, @@ -162,11 +180,11 @@ defmodule Pleroma.Web.AdminAPI.UserController do User.register_changeset(%User{}, user_data, need_confirmation: false) end) - |> Enum.reduce(Ecto.Multi.new(), fn changeset, multi -> - Ecto.Multi.insert(multi, Ecto.UUID.generate(), changeset) + |> Enum.reduce(Multi.new(), fn changeset, multi -> + Multi.insert(multi, "user-" <> changeset.changes.nickname, changeset) end) - case Pleroma.Repo.transaction(changesets) do + case Pleroma.Repo.transaction(multi) do {:ok, users_map} -> users = users_map @@ -177,17 +195,28 @@ defmodule Pleroma.Web.AdminAPI.UserController do user end) + password_reset_links = + users + |> Enum.filter(fn user -> Enum.member?(passwordless_nicknames, user.nickname) end) + |> Map.new(fn user -> + {:ok, token} = Pleroma.PasswordResetToken.create_token(user) + {user.id, Router.Helpers.reset_password_url(Endpoint, :reset, token.token)} + end) + ModerationLog.insert_log(%{ actor: admin, subjects: users, action: "create" }) - render(conn, "created_many.json", users: users) + render(conn, "created_many.json", + users: users, + password_reset_links: password_reset_links + ) {:error, id, changeset, _} -> changesets = - Enum.map(changesets.operations, fn + Enum.map(multi.operations, fn {^id, {:changeset, _current_changeset, _}} -> changeset diff --git a/lib/pleroma/web/admin_api/views/account_view.ex b/lib/pleroma/web/admin_api/views/account_view.ex index 280152241..cc715c2a5 100644 --- a/lib/pleroma/web/admin_api/views/account_view.ex +++ b/lib/pleroma/web/admin_api/views/account_view.ex @@ -88,18 +88,29 @@ defmodule Pleroma.Web.AdminAPI.AccountView do } end - def render("created_many.json", %{users: users}) do - render_many(users, AccountView, "created.json", as: :user) + def render("created_many.json", %{users: users} = assigns) do + password_reset_links = Map.get(assigns, :password_reset_links, %{}) + + render_many(users, AccountView, "created.json", + as: :user, + password_reset_links: password_reset_links + ) end - def render("created.json", %{user: user}) do - %{ - type: "success", - code: 200, - data: %{ + def render("created.json", %{user: user} = assigns) do + password_reset_links = Map.get(assigns, :password_reset_links, %{}) + + data = + %{ nickname: user.nickname, email: user.email } + |> maybe_put_password_reset_link(Map.get(password_reset_links, user.id)) + + %{ + type: "success", + code: 200, + data: data } end @@ -119,6 +130,11 @@ defmodule Pleroma.Web.AdminAPI.AccountView do } end + defp maybe_put_password_reset_link(data, nil), do: data + + defp maybe_put_password_reset_link(data, link), + do: Map.put(data, :password_reset_link, link) + def merge_account_views(%User{} = user) do MastodonAPI.AccountView.render("show.json", %{user: user, skip_visibility_check: true}) |> Map.merge(AdminAPI.AccountView.render("show.json", %{user: user})) diff --git a/lib/pleroma/web/api_spec/operations/admin/user_operation.ex b/lib/pleroma/web/api_spec/operations/admin/user_operation.ex index a5179ac39..84cf789d8 100644 --- a/lib/pleroma/web/api_spec/operations/admin/user_operation.ex +++ b/lib/pleroma/web/api_spec/operations/admin/user_operation.ex @@ -82,8 +82,14 @@ defmodule Pleroma.Web.ApiSpec.Admin.UserOperation do properties: %{ nickname: %Schema{type: :string}, email: %Schema{type: :string}, - password: %Schema{type: :string} - } + password: %Schema{ + type: :string, + description: + "Optional. When omitted, a `password_reset_link` is returned, " <> + "allowing the user to set their own password." + } + }, + required: [:nickname, :email] } } } @@ -102,7 +108,8 @@ defmodule Pleroma.Web.ApiSpec.Admin.UserOperation do type: :object, properties: %{ email: %Schema{type: :string, format: :email}, - nickname: %Schema{type: :string} + nickname: %Schema{type: :string}, + password_reset_link: %Schema{type: :string} } } } diff --git a/test/pleroma/web/admin_api/controllers/user_controller_test.exs b/test/pleroma/web/admin_api/controllers/user_controller_test.exs index 0e5650285..9808f9d0d 100644 --- a/test/pleroma/web/admin_api/controllers/user_controller_test.exs +++ b/test/pleroma/web/admin_api/controllers/user_controller_test.exs @@ -176,6 +176,34 @@ defmodule Pleroma.Web.AdminAPI.UserControllerTest do assert ["lain", "lain2"] -- Enum.map(log_entry.data["subjects"], & &1["nickname"]) == [] end + test "Creating without a password returns a password reset link", %{conn: conn} do + [account1, account2] = + conn + |> put_req_header("accept", "application/json") + |> put_req_header("content-type", "application/json") + |> post("/api/pleroma/admin/users", %{ + "users" => [ + %{ + "nickname" => "lain", + "email" => "lain@example.org" + }, + %{ + "nickname" => "lain2", + "email" => "lain2@example.org", + "password" => "dupa.8" + } + ] + }) + |> json_response_and_validate_schema(200) + + assert account1["type"] == "success" + assert %{"password_reset_link" => link} = account1["data"] + assert link =~ "/api/v1/pleroma/password_reset/" + + assert account2["type"] == "success" + refute Map.has_key?(account2["data"], "password_reset_link") + end + test "Cannot create user with existing email", %{conn: conn} do user = insert(:user) From 34d827730ac3c1305c1170ed51f9907a499b9680 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Fri, 24 Jul 2026 22:15:14 +0400 Subject: [PATCH 52/90] fix(admin-api): preserve user batch validation --- .../admin_api/controllers/user_controller.ex | 2 +- .../controllers/user_controller_test.exs | 36 ++++++++++++++++++- 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/admin_api/controllers/user_controller.ex b/lib/pleroma/web/admin_api/controllers/user_controller.ex index 23af5b53c..4552f32c7 100644 --- a/lib/pleroma/web/admin_api/controllers/user_controller.ex +++ b/lib/pleroma/web/admin_api/controllers/user_controller.ex @@ -181,7 +181,7 @@ defmodule Pleroma.Web.AdminAPI.UserController do User.register_changeset(%User{}, user_data, need_confirmation: false) end) |> Enum.reduce(Multi.new(), fn changeset, multi -> - Multi.insert(multi, "user-" <> changeset.changes.nickname, changeset) + Multi.insert(multi, Ecto.UUID.generate(), changeset) end) case Pleroma.Repo.transaction(multi) do diff --git a/test/pleroma/web/admin_api/controllers/user_controller_test.exs b/test/pleroma/web/admin_api/controllers/user_controller_test.exs index 9808f9d0d..c2802164d 100644 --- a/test/pleroma/web/admin_api/controllers/user_controller_test.exs +++ b/test/pleroma/web/admin_api/controllers/user_controller_test.exs @@ -177,7 +177,7 @@ defmodule Pleroma.Web.AdminAPI.UserControllerTest do end test "Creating without a password returns a password reset link", %{conn: conn} do - [account1, account2] = + accounts = conn |> put_req_header("accept", "application/json") |> put_req_header("content-type", "application/json") @@ -195,6 +195,10 @@ defmodule Pleroma.Web.AdminAPI.UserControllerTest do ] }) |> json_response_and_validate_schema(200) + |> Map.new(fn account -> {account["data"]["nickname"], account} end) + + account1 = accounts["lain"] + account2 = accounts["lain2"] assert account1["type"] == "success" assert %{"password_reset_link" => link} = account1["data"] @@ -204,6 +208,36 @@ defmodule Pleroma.Web.AdminAPI.UserControllerTest do refute Map.has_key?(account2["data"], "password_reset_link") end + test "Duplicate nicknames in a batch return a conflict", %{conn: conn} do + conn = + conn + |> put_req_header("accept", "application/json") + |> put_req_header("content-type", "application/json") + |> post("/api/pleroma/admin/users", %{ + "users" => [ + %{"nickname" => "lain", "email" => "lain1@example.org", "password" => "test"}, + %{"nickname" => "lain", "email" => "lain2@example.org", "password" => "test"} + ] + }) + + assert [_first, _second] = json_response(conn, 409) + refute User.get_by_nickname("lain") + end + + test "Blank nicknames return a conflict", %{conn: conn} do + conn = + conn + |> put_req_header("accept", "application/json") + |> put_req_header("content-type", "application/json") + |> post("/api/pleroma/admin/users", %{ + "users" => [ + %{"nickname" => "", "email" => "lain@example.org", "password" => "test"} + ] + }) + + assert [%{"error" => "nickname can't be blank"}] = json_response(conn, 409) + end + test "Cannot create user with existing email", %{conn: conn} do user = insert(:user) From d57fdcc7b03dc1381687a49f6afc266439d5c103 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Fri, 24 Jul 2026 22:25:44 +0400 Subject: [PATCH 53/90] fix(admin-api): create reset tokens atomically --- .../admin_api/controllers/user_controller.ex | 63 ++++++++++++------- .../controllers/user_controller_test.exs | 30 +++++++++ 2 files changed, 71 insertions(+), 22 deletions(-) diff --git a/lib/pleroma/web/admin_api/controllers/user_controller.ex b/lib/pleroma/web/admin_api/controllers/user_controller.ex index 4552f32c7..7146e4bea 100644 --- a/lib/pleroma/web/admin_api/controllers/user_controller.ex +++ b/lib/pleroma/web/admin_api/controllers/user_controller.ex @@ -10,6 +10,7 @@ defmodule Pleroma.Web.AdminAPI.UserController do alias Ecto.Multi alias Pleroma.ModerationLog + alias Pleroma.PasswordResetToken alias Pleroma.User alias Pleroma.Web.ActivityPub.Builder alias Pleroma.Web.ActivityPub.Pipeline @@ -151,15 +152,11 @@ defmodule Pleroma.Web.AdminAPI.UserController do } = conn, _ ) do - # Nicknames of users for which a password reset link will be generated. - passwordless_nicknames = - users - |> Enum.filter(fn user -> Map.get(user, :password) in ["", nil] end) - |> Enum.map(& &1.nickname) - multi = users |> Enum.map(fn %{nickname: nickname, email: email} = attrs -> + passwordless? = Map.get(attrs, :password) in ["", nil] + password = case Map.get(attrs, :password) do password when is_binary(password) and password != "" -> @@ -178,17 +175,36 @@ defmodule Pleroma.Web.AdminAPI.UserController do bio: "." } - User.register_changeset(%User{}, user_data, need_confirmation: false) + {User.register_changeset(%User{}, user_data, need_confirmation: false), passwordless?} end) - |> Enum.reduce(Multi.new(), fn changeset, multi -> - Multi.insert(multi, Ecto.UUID.generate(), changeset) + |> Enum.reduce(Multi.new(), fn {changeset, passwordless?}, multi -> + user_operation = {:user, Ecto.UUID.generate()} + multi = Multi.insert(multi, user_operation, changeset) + + if passwordless? do + Multi.run( + multi, + {:password_reset_token, Ecto.UUID.generate()}, + fn _repo, changes -> + changes + |> Map.fetch!(user_operation) + |> PasswordResetToken.create_token() + end + ) + else + multi + end end) case Pleroma.Repo.transaction(multi) do - {:ok, users_map} -> - users = - users_map + {:ok, results} -> + {users, password_reset_tokens} = + results |> Map.values() + |> Enum.split_with(&match?(%User{}, &1)) + + users = + users |> Enum.map(fn user -> {:ok, user} = User.post_register_action(user) @@ -196,11 +212,8 @@ defmodule Pleroma.Web.AdminAPI.UserController do end) password_reset_links = - users - |> Enum.filter(fn user -> Enum.member?(passwordless_nicknames, user.nickname) end) - |> Map.new(fn user -> - {:ok, token} = Pleroma.PasswordResetToken.create_token(user) - {user.id, Router.Helpers.reset_password_url(Endpoint, :reset, token.token)} + Map.new(password_reset_tokens, fn token -> + {token.user_id, Router.Helpers.reset_password_url(Endpoint, :reset, token.token)} end) ModerationLog.insert_log(%{ @@ -214,19 +227,25 @@ defmodule Pleroma.Web.AdminAPI.UserController do password_reset_links: password_reset_links ) - {:error, id, changeset, _} -> + {:error, {:user, _} = id, %Ecto.Changeset{} = changeset, _} -> changesets = - Enum.map(multi.operations, fn + Enum.flat_map(multi.operations, fn {^id, {:changeset, _current_changeset, _}} -> - changeset + [changeset] - {_, {:changeset, current_changeset, _}} -> - current_changeset + {{:user, _}, {:changeset, current_changeset, _}} -> + [current_changeset] + + _ -> + [] end) conn |> put_status(:conflict) |> render("create_errors.json", changesets: changesets) + + {:error, {:password_reset_token, _}, _reason, _} -> + raise "Failed to create password reset token" end end diff --git a/test/pleroma/web/admin_api/controllers/user_controller_test.exs b/test/pleroma/web/admin_api/controllers/user_controller_test.exs index c2802164d..b6c19f37c 100644 --- a/test/pleroma/web/admin_api/controllers/user_controller_test.exs +++ b/test/pleroma/web/admin_api/controllers/user_controller_test.exs @@ -208,6 +208,36 @@ defmodule Pleroma.Web.AdminAPI.UserControllerTest do refute Map.has_key?(account2["data"], "password_reset_link") end + test "Password reset token failures roll back the user batch", %{conn: conn} do + with_mock Pleroma.PasswordResetToken, [:passthrough], + create_token: fn user -> + if user.nickname == "lain2" do + {:error, :forced} + else + result = passthrough([user]) + send(self(), {:token_created, user.nickname}) + result + end + end do + assert_error_sent(500, fn -> + conn + |> put_req_header("accept", "application/json") + |> put_req_header("content-type", "application/json") + |> post("/api/pleroma/admin/users", %{ + "users" => [ + %{"nickname" => "lain", "email" => "lain@example.org"}, + %{"nickname" => "lain2", "email" => "lain2@example.org"} + ] + }) + end) + end + + assert_received {:token_created, "lain"} + refute User.get_by_nickname("lain") + refute User.get_by_nickname("lain2") + assert Repo.aggregate(Pleroma.PasswordResetToken, :count) == 0 + end + test "Duplicate nicknames in a batch return a conflict", %{conn: conn} do conn = conn From 3b3aad606508ecf1801a24420143d700c4cf01c6 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Fri, 24 Jul 2026 22:45:50 +0400 Subject: [PATCH 54/90] fix(api): isolate sensitive idempotency responses --- .../admin_api/controllers/user_controller.ex | 7 + lib/pleroma/web/plugs/idempotency_plug.ex | 58 ++++++-- .../controllers/user_controller_test.exs | 45 ++++++ .../web/plugs/idempotency_plug_test.exs | 133 ++++++++++++++++++ 4 files changed, 231 insertions(+), 12 deletions(-) diff --git a/lib/pleroma/web/admin_api/controllers/user_controller.ex b/lib/pleroma/web/admin_api/controllers/user_controller.ex index 7146e4bea..c94964937 100644 --- a/lib/pleroma/web/admin_api/controllers/user_controller.ex +++ b/lib/pleroma/web/admin_api/controllers/user_controller.ex @@ -152,6 +152,13 @@ defmodule Pleroma.Web.AdminAPI.UserController do } = conn, _ ) do + conn = + if Enum.any?(users, fn user -> Map.get(user, :password) in ["", nil] end) do + put_private(conn, :skip_idempotency_cache, true) + else + conn + end + multi = users |> Enum.map(fn %{nickname: nickname, email: email} = attrs -> diff --git a/lib/pleroma/web/plugs/idempotency_plug.ex b/lib/pleroma/web/plugs/idempotency_plug.ex index a3b7af869..81983efba 100644 --- a/lib/pleroma/web/plugs/idempotency_plug.ex +++ b/lib/pleroma/web/plugs/idempotency_plug.ex @@ -6,6 +6,8 @@ defmodule Pleroma.Web.Plugs.IdempotencyPlug do import Phoenix.Controller, only: [json: 2] import Plug.Conn + alias Pleroma.Config + @behaviour Plug @cachex Pleroma.Config.get([:cachex, :provider], Cachex) @@ -27,9 +29,11 @@ defmodule Pleroma.Web.Plugs.IdempotencyPlug do def call(conn, _), do: conn def process_request(conn, key) do - case @cachex.get(:idempotency_cache, key) do + cache_key = {conn.method, conn.request_path, actor_key(conn), key} + + case @cachex.get(:idempotency_cache, cache_key) do {:ok, nil} -> - cache_resposnse(conn, key) + cache_resposnse(conn, key, cache_key) {:ok, record} -> send_cached(conn, key, record) @@ -39,18 +43,48 @@ defmodule Pleroma.Web.Plugs.IdempotencyPlug do end end - defp cache_resposnse(conn, key) do - register_before_send(conn, fn conn -> - [request_id] = get_resp_header(conn, "x-request-id") - content_type = get_content_type(conn) + defp cache_resposnse(conn, key, cache_key) do + register_before_send(conn, &maybe_cache_response(&1, key, cache_key)) + end - record = {request_id, content_type, conn.status, conn.resp_body} - {:ok, _} = @cachex.put(:idempotency_cache, key, record) + defp maybe_cache_response( + %{private: %{skip_idempotency_cache: true}} = conn, + _key, + _cache_key + ), + do: conn - conn - |> put_resp_header("idempotency-key", key) - |> put_resp_header("x-original-request-id", request_id) - end) + defp maybe_cache_response(conn, key, cache_key) do + [request_id] = get_resp_header(conn, "x-request-id") + content_type = get_content_type(conn) + + record = {request_id, content_type, conn.status, conn.resp_body} + {:ok, _} = @cachex.put(:idempotency_cache, cache_key, record) + + conn + |> put_resp_header("idempotency-key", key) + |> put_resp_header("x-original-request-id", request_id) + end + + defp actor_key(%{assigns: assigns}) do + {principal_key(assigns), token_key(assigns[:token]), staff_privileges()} + end + + defp principal_key(%{user: %{id: id} = user}) do + {:user, id, Map.get(user, :is_admin), Map.get(user, :is_moderator)} + end + + defp principal_key(%{app: %{id: id}}), do: {:app, id} + defp principal_key(_assigns), do: nil + + defp token_key(%{id: id} = token), do: {id, token |> Map.get(:scopes, []) |> Enum.sort()} + defp token_key(_token), do: nil + + defp staff_privileges do + { + Config.get([:instance, :admin_privileges], []) |> Enum.sort(), + Config.get([:instance, :moderator_privileges], []) |> Enum.sort() + } end defp send_cached(conn, key, record) do diff --git a/test/pleroma/web/admin_api/controllers/user_controller_test.exs b/test/pleroma/web/admin_api/controllers/user_controller_test.exs index b6c19f37c..32f2c1d30 100644 --- a/test/pleroma/web/admin_api/controllers/user_controller_test.exs +++ b/test/pleroma/web/admin_api/controllers/user_controller_test.exs @@ -238,6 +238,51 @@ defmodule Pleroma.Web.AdminAPI.UserControllerTest do assert Repo.aggregate(Pleroma.PasswordResetToken, :count) == 0 end + test "Password reset links are not cached by idempotency key", %{conn: conn} do + idempotency_key = Ecto.UUID.generate() + + request = fn conn -> + conn + |> put_req_header("accept", "application/json") + |> put_req_header("content-type", "application/json") + |> put_req_header("idempotency-key", idempotency_key) + |> post("/api/pleroma/admin/users", %{ + "users" => [%{"nickname" => "lain", "email" => "lain@example.org"}] + }) + end + + assert [%{"data" => %{"password_reset_link" => _link}}] = + conn |> request.() |> json_response(200) + + second_conn = request.(conn) + + assert [_error] = json_response(second_conn, 409) + refute get_resp_header(second_conn, "idempotent-replayed") == ["true"] + end + + test "Explicit-password responses remain idempotent", %{conn: conn} do + idempotency_key = Ecto.UUID.generate() + + request = fn conn -> + conn + |> put_req_header("accept", "application/json") + |> put_req_header("content-type", "application/json") + |> put_req_header("idempotency-key", idempotency_key) + |> post("/api/pleroma/admin/users", %{ + "users" => [ + %{"nickname" => "lain", "email" => "lain@example.org", "password" => "test"} + ] + }) + end + + assert [_account] = conn |> request.() |> json_response(200) + + second_conn = request.(conn) + + assert [_account] = json_response(second_conn, 200) + assert get_resp_header(second_conn, "idempotent-replayed") == ["true"] + end + test "Duplicate nicknames in a batch return a conflict", %{conn: conn} do conn = conn diff --git a/test/pleroma/web/plugs/idempotency_plug_test.exs b/test/pleroma/web/plugs/idempotency_plug_test.exs index 3b0131596..cc7580ca2 100644 --- a/test/pleroma/web/plugs/idempotency_plug_test.exs +++ b/test/pleroma/web/plugs/idempotency_plug_test.exs @@ -70,6 +70,139 @@ defmodule Pleroma.Web.Plugs.IdempotencyPlugTest do assert [^key] = Conn.get_resp_header(conn, "idempotency-key") end + test "does not cache responses marked as sensitive" do + key = Ecto.UUID.generate() + + :post + |> conn("/cofe") + |> put_req_header("idempotency-key", key) + |> Conn.put_resp_header("x-request-id", "sensitive-request") + |> Conn.put_resp_content_type("application/json") + |> IdempotencyPlug.call([]) + |> Conn.put_private(:skip_idempotency_cache, true) + |> Conn.send_resp(200, "secret") + + conn = + :post + |> conn("/cofe") + |> put_req_header("idempotency-key", key) + |> Conn.put_resp_header("x-request-id", "second-request") + |> Conn.put_resp_content_type("application/json") + |> IdempotencyPlug.call([]) + + refute conn.halted + refute conn.resp_body + end + + test "does not replay legacy unscoped cache entries" do + key = Ecto.UUID.generate() + record = {"legacy-request", "application/json", 200, "secret"} + assert {:ok, true} = Cachex.put(:idempotency_cache, key, record) + + conn = + :post + |> conn("/cofe") + |> put_req_header("idempotency-key", key) + |> Conn.put_resp_header("x-request-id", "new-request") + |> Conn.put_resp_content_type("application/json") + |> IdempotencyPlug.call([]) + + refute conn.halted + refute conn.resp_body + end + + test "scopes cached responses to the route and authenticated actor" do + key = Ecto.UUID.generate() + + :post + |> conn("/cofe") + |> Conn.assign(:user, %{id: "user-one", is_admin: false, is_moderator: false}) + |> put_req_header("idempotency-key", key) + |> Conn.put_resp_header("x-request-id", "original-request") + |> Conn.put_resp_content_type("application/json") + |> IdempotencyPlug.call([]) + |> Conn.send_resp(200, "one") + + for {path, user} <- [ + {"/tea", %{id: "user-one", is_admin: false, is_moderator: false}}, + {"/cofe", %{id: "user-two", is_admin: false, is_moderator: false}}, + {"/cofe", %{id: "user-one", is_admin: true, is_moderator: false}} + ] do + conn = + :post + |> conn(path) + |> Conn.assign(:user, user) + |> put_req_header("idempotency-key", key) + |> Conn.put_resp_header("x-request-id", Ecto.UUID.generate()) + |> Conn.put_resp_content_type("application/json") + |> IdempotencyPlug.call([]) + + refute conn.halted + refute conn.resp_body + end + end + + test "scopes cached responses to the OAuth application and token" do + key = Ecto.UUID.generate() + + :post + |> conn("/cofe") + |> Conn.assign(:app, %{id: "app-one"}) + |> Conn.assign(:token, %{id: "token-one", scopes: ["write"]}) + |> put_req_header("idempotency-key", key) + |> Conn.put_resp_header("x-request-id", "original-request") + |> Conn.put_resp_content_type("application/json") + |> IdempotencyPlug.call([]) + |> Conn.send_resp(200, "one") + + for {app, token} <- [ + {%{id: "app-two"}, %{id: "token-two", scopes: ["write"]}}, + {%{id: "app-one"}, %{id: "token-two", scopes: ["write"]}}, + {%{id: "app-one"}, %{id: "token-one", scopes: ["read"]}} + ] do + conn = + :post + |> conn("/cofe") + |> Conn.assign(:app, app) + |> Conn.assign(:token, token) + |> put_req_header("idempotency-key", key) + |> Conn.put_resp_header("x-request-id", Ecto.UUID.generate()) + |> Conn.put_resp_content_type("application/json") + |> IdempotencyPlug.call([]) + + refute conn.halted + refute conn.resp_body + end + end + + test "scopes cached responses to configured staff privileges" do + key = Ecto.UUID.generate() + clear_config([:instance, :admin_privileges], [:users_delete]) + + :post + |> conn("/cofe") + |> Conn.assign(:user, %{id: "admin", is_admin: true, is_moderator: false}) + |> put_req_header("idempotency-key", key) + |> Conn.put_resp_header("x-request-id", "original-request") + |> Conn.put_resp_content_type("application/json") + |> IdempotencyPlug.call([]) + |> Conn.send_resp(200, "one") + + clear_config([:instance, :admin_privileges], []) + + conn = + :post + |> conn("/cofe") + |> Conn.assign(:user, %{id: "admin", is_admin: true, is_moderator: false}) + |> put_req_header("idempotency-key", key) + |> Conn.put_resp_header("x-request-id", "second-request") + |> Conn.put_resp_content_type("application/json") + |> IdempotencyPlug.call([]) + + refute conn.halted + refute conn.resp_body + end + test "passes conn downstream if idempotency is not present in headers" do orig_request_id = "test4" body = "testing" From 836f4a2d69e3945ca205eb1af7d3ebfea7c3153b Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Fri, 24 Jul 2026 22:52:23 +0400 Subject: [PATCH 55/90] fix(admin-api): reject explicit empty passwords --- docs/development/API/admin_api.md | 2 +- .../web/admin_api/controllers/user_controller.ex | 16 +++++++--------- .../api_spec/operations/admin/user_operation.ex | 3 ++- .../controllers/user_controller_test.exs | 15 +++++++++++++++ 4 files changed, 25 insertions(+), 11 deletions(-) diff --git a/docs/development/API/admin_api.md b/docs/development/API/admin_api.md index ef04944af..a59804c87 100644 --- a/docs/development/API/admin_api.md +++ b/docs/development/API/admin_api.md @@ -80,7 +80,7 @@ The `/api/v1/pleroma/admin/*` path is backwards compatible with `/api/pleroma/ad { `nickname`, `email`, - `password` (optional) + `password` (optional; when provided, must not be empty) } ] - Response: Array of user objects. When a user is created without a `password`, `data` diff --git a/lib/pleroma/web/admin_api/controllers/user_controller.ex b/lib/pleroma/web/admin_api/controllers/user_controller.ex index c94964937..f0edfe4ba 100644 --- a/lib/pleroma/web/admin_api/controllers/user_controller.ex +++ b/lib/pleroma/web/admin_api/controllers/user_controller.ex @@ -153,7 +153,7 @@ defmodule Pleroma.Web.AdminAPI.UserController do _ ) do conn = - if Enum.any?(users, fn user -> Map.get(user, :password) in ["", nil] end) do + if Enum.any?(users, fn user -> not Map.has_key?(user, :password) end) do put_private(conn, :skip_idempotency_cache, true) else conn @@ -162,15 +162,13 @@ defmodule Pleroma.Web.AdminAPI.UserController do multi = users |> Enum.map(fn %{nickname: nickname, email: email} = attrs -> - passwordless? = Map.get(attrs, :password) in ["", nil] + {password, passwordless?} = + case Map.fetch(attrs, :password) do + {:ok, password} -> + {password, false} - password = - case Map.get(attrs, :password) do - password when is_binary(password) and password != "" -> - password - - _ -> - :crypto.strong_rand_bytes(16) |> Base.encode64() + :error -> + {:crypto.strong_rand_bytes(16) |> Base.encode64(), true} end user_data = %{ diff --git a/lib/pleroma/web/api_spec/operations/admin/user_operation.ex b/lib/pleroma/web/api_spec/operations/admin/user_operation.ex index 84cf789d8..7c4725f56 100644 --- a/lib/pleroma/web/api_spec/operations/admin/user_operation.ex +++ b/lib/pleroma/web/api_spec/operations/admin/user_operation.ex @@ -86,7 +86,8 @@ defmodule Pleroma.Web.ApiSpec.Admin.UserOperation do type: :string, description: "Optional. When omitted, a `password_reset_link` is returned, " <> - "allowing the user to set their own password." + "allowing the user to set their own password. When provided, " <> + "it must not be empty." } }, required: [:nickname, :email] diff --git a/test/pleroma/web/admin_api/controllers/user_controller_test.exs b/test/pleroma/web/admin_api/controllers/user_controller_test.exs index 32f2c1d30..0bea0c185 100644 --- a/test/pleroma/web/admin_api/controllers/user_controller_test.exs +++ b/test/pleroma/web/admin_api/controllers/user_controller_test.exs @@ -208,6 +208,21 @@ defmodule Pleroma.Web.AdminAPI.UserControllerTest do refute Map.has_key?(account2["data"], "password_reset_link") end + test "Explicit empty passwords return a conflict", %{conn: conn} do + conn = + conn + |> put_req_header("accept", "application/json") + |> put_req_header("content-type", "application/json") + |> post("/api/pleroma/admin/users", %{ + "users" => [ + %{"nickname" => "lain", "email" => "lain@example.org", "password" => ""} + ] + }) + + assert [%{"error" => "password can't be blank"}] = json_response(conn, 409) + refute User.get_by_nickname("lain") + end + test "Password reset token failures roll back the user batch", %{conn: conn} do with_mock Pleroma.PasswordResetToken, [:passthrough], create_token: fn user -> From d0b5a05075ec9c3566df8de90346c5969c3089f0 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Fri, 24 Jul 2026 23:14:03 +0400 Subject: [PATCH 56/90] fix(admin-api): confirm created users --- .../web/admin_api/controllers/user_controller.ex | 2 +- .../controllers/user_controller_test.exs | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/admin_api/controllers/user_controller.ex b/lib/pleroma/web/admin_api/controllers/user_controller.ex index f0edfe4ba..82ac4bd9d 100644 --- a/lib/pleroma/web/admin_api/controllers/user_controller.ex +++ b/lib/pleroma/web/admin_api/controllers/user_controller.ex @@ -180,7 +180,7 @@ defmodule Pleroma.Web.AdminAPI.UserController do bio: "." } - {User.register_changeset(%User{}, user_data, need_confirmation: false), passwordless?} + {User.register_changeset(%User{}, user_data, confirmed: true), passwordless?} end) |> Enum.reduce(Multi.new(), fn {changeset, passwordless?}, multi -> user_operation = {:user, Ecto.UUID.generate()} diff --git a/test/pleroma/web/admin_api/controllers/user_controller_test.exs b/test/pleroma/web/admin_api/controllers/user_controller_test.exs index 0bea0c185..7e05d67b7 100644 --- a/test/pleroma/web/admin_api/controllers/user_controller_test.exs +++ b/test/pleroma/web/admin_api/controllers/user_controller_test.exs @@ -176,6 +176,22 @@ defmodule Pleroma.Web.AdminAPI.UserControllerTest do assert ["lain", "lain2"] -- Enum.map(log_entry.data["subjects"], & &1["nickname"]) == [] end + test "Admin-created users bypass account confirmation", %{conn: conn} do + clear_config([:instance, :account_activation_required], true) + + conn + |> put_req_header("accept", "application/json") + |> put_req_header("content-type", "application/json") + |> post("/api/pleroma/admin/users", %{ + "users" => [ + %{"nickname" => "lain", "email" => "lain@example.org", "password" => "test"} + ] + }) + |> json_response_and_validate_schema(200) + + assert User.get_by_nickname("lain").is_confirmed + end + test "Creating without a password returns a password reset link", %{conn: conn} do accounts = conn From bb4826d8aec57391b29ad3312e34f4b5abe21383 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Fri, 24 Jul 2026 23:24:52 +0400 Subject: [PATCH 57/90] fix(admin-api): structure reset token failures --- .../admin_api/controllers/user_controller.ex | 5 ++- .../controllers/user_controller_test.exs | 31 ++++++++++++------- 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/lib/pleroma/web/admin_api/controllers/user_controller.ex b/lib/pleroma/web/admin_api/controllers/user_controller.ex index 82ac4bd9d..ab8bf73ee 100644 --- a/lib/pleroma/web/admin_api/controllers/user_controller.ex +++ b/lib/pleroma/web/admin_api/controllers/user_controller.ex @@ -250,7 +250,10 @@ defmodule Pleroma.Web.AdminAPI.UserController do |> render("create_errors.json", changesets: changesets) {:error, {:password_reset_token, _}, _reason, _} -> - raise "Failed to create password reset token" + Pleroma.Web.AdminAPI.FallbackController.call( + conn, + {:password_reset_token_error, :creation_failed} + ) end end diff --git a/test/pleroma/web/admin_api/controllers/user_controller_test.exs b/test/pleroma/web/admin_api/controllers/user_controller_test.exs index 7e05d67b7..764f5e01f 100644 --- a/test/pleroma/web/admin_api/controllers/user_controller_test.exs +++ b/test/pleroma/web/admin_api/controllers/user_controller_test.exs @@ -240,6 +240,21 @@ defmodule Pleroma.Web.AdminAPI.UserControllerTest do end test "Password reset token failures roll back the user batch", %{conn: conn} do + idempotency_key = Ecto.UUID.generate() + + request = fn conn -> + conn + |> put_req_header("accept", "application/json") + |> put_req_header("content-type", "application/json") + |> put_req_header("idempotency-key", idempotency_key) + |> post("/api/pleroma/admin/users", %{ + "users" => [ + %{"nickname" => "lain", "email" => "lain@example.org"}, + %{"nickname" => "lain2", "email" => "lain2@example.org"} + ] + }) + end + with_mock Pleroma.PasswordResetToken, [:passthrough], create_token: fn user -> if user.nickname == "lain2" do @@ -250,23 +265,17 @@ defmodule Pleroma.Web.AdminAPI.UserControllerTest do result end end do - assert_error_sent(500, fn -> - conn - |> put_req_header("accept", "application/json") - |> put_req_header("content-type", "application/json") - |> post("/api/pleroma/admin/users", %{ - "users" => [ - %{"nickname" => "lain", "email" => "lain@example.org"}, - %{"nickname" => "lain2", "email" => "lain2@example.org"} - ] - }) - end) + conn = request.(conn) + + assert json_response(conn, 500) == %{"error" => "Something went wrong"} end assert_received {:token_created, "lain"} refute User.get_by_nickname("lain") refute User.get_by_nickname("lain2") assert Repo.aggregate(Pleroma.PasswordResetToken, :count) == 0 + + assert [_account1, _account2] = conn |> request.() |> json_response(200) end test "Password reset links are not cached by idempotency key", %{conn: conn} do From 91089c93ada742ff2b709601e9f0ea217c9f5a06 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sun, 26 Jul 2026 11:38:24 +0400 Subject: [PATCH 58/90] feat(search): add resumable runtime backfill --- lib/mix/tasks/pleroma/search/indexer.ex | 58 ++++------ lib/pleroma/search/backfill.ex | 137 ++++++++++++++++++++++++ test/pleroma/search/backfill_test.exs | 79 ++++++++++++++ 3 files changed, 234 insertions(+), 40 deletions(-) create mode 100644 lib/pleroma/search/backfill.ex create mode 100644 test/pleroma/search/backfill_test.exs diff --git a/lib/mix/tasks/pleroma/search/indexer.ex b/lib/mix/tasks/pleroma/search/indexer.ex index 2a52472f9..62cbf2e50 100644 --- a/lib/mix/tasks/pleroma/search/indexer.ex +++ b/lib/mix/tasks/pleroma/search/indexer.ex @@ -4,9 +4,6 @@ defmodule Mix.Tasks.Pleroma.Search.Indexer do import Mix.Pleroma - import Ecto.Query - - alias Pleroma.Workers.SearchIndexingWorker def run(["create_index"]) do start_pleroma() @@ -35,49 +32,30 @@ defmodule Mix.Tasks.Pleroma.Search.Indexer do strict: [ chunk: :integer, limit: :integer, - step: :integer + step: :integer, + before: :string ] ) start_pleroma() - chunk_size = Keyword.get(options, :chunk, 100) - limit = Keyword.get(options, :limit, 100_000) - per_step = Keyword.get(options, :step, 1000) + result = + Pleroma.Search.Backfill.run( + options ++ + [ + on_page: fn page -> + IO.puts( + "Queued #{page.count} activities (#{page.enqueued} total), " <> + "checkpoint: #{page.next_cursor}" + ) + end + ] + ) - chunks = max(div(limit, per_step), 1) + IO.puts("Queued #{result.enqueued} activities") - 1..chunks - |> Enum.each(fn step -> - q = - from(a in Pleroma.Activity, - limit: ^per_step, - offset: ^per_step * (^step - 1), - select: [:id], - order_by: [desc: :id] - ) - - {:ok, ids} = - Pleroma.Repo.transaction(fn -> - Pleroma.Repo.stream(q, timeout: :infinity) - |> Enum.map(fn a -> - a.id - end) - end) - - IO.puts("Got #{length(ids)} activities, adding to indexer") - - ids - |> Enum.chunk_every(chunk_size) - |> Enum.each(fn chunk -> - IO.puts("Adding #{length(chunk)} activities to indexing queue") - - chunk - |> Enum.map(fn id -> - SearchIndexingWorker.new(%{"op" => "add_to_index", "activity" => id}) - end) - |> Oban.insert_all() - end) - end) + unless result.exhausted do + IO.puts("Resume with --before #{result.next_cursor}") + end end end diff --git a/lib/pleroma/search/backfill.ex b/lib/pleroma/search/backfill.ex new file mode 100644 index 000000000..1b986bb59 --- /dev/null +++ b/lib/pleroma/search/backfill.ex @@ -0,0 +1,137 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2026 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Search.Backfill do + import Ecto.Query + + alias Pleroma.Activity + alias Pleroma.Workers.SearchIndexingWorker + + @default_chunk_size 100 + @default_step_size 1_000 + @default_limit 100_000 + + @doc """ + Enqueues existing Create activities for search indexing. + + The returned `next_cursor` can be passed as `before` to resume a bounded run. + Pass `limit: :infinity` to continue until the oldest activity is reached. + """ + def run(options \\ []) do + options = + Keyword.validate!(options, + chunk: @default_chunk_size, + step: @default_step_size, + limit: @default_limit, + before: nil, + on_page: nil + ) + + chunk_size = positive_option!(options, :chunk, @default_chunk_size) + step_size = positive_option!(options, :step, @default_step_size) + limit = limit_option!(options) + before = cursor_option!(options[:before]) + on_page = callback_option!(options[:on_page]) + + do_run(before, limit, step_size, chunk_size, on_page, 0) + end + + defp do_run(before, remaining, step_size, chunk_size, on_page, enqueued) do + page_size = page_size(remaining, step_size) + ids = fetch_ids(before, page_size) + + case ids do + [] -> + result(enqueued, before, true) + + ids -> + enqueue(ids, chunk_size) + + page_count = length(ids) + next_cursor = List.last(ids) + total = enqueued + page_count + remaining = decrement(remaining, page_count) + + on_page.(%{count: page_count, enqueued: total, next_cursor: next_cursor}) + + cond do + remaining == 0 -> result(total, next_cursor, false) + page_count < page_size -> result(total, next_cursor, true) + true -> do_run(next_cursor, remaining, step_size, chunk_size, on_page, total) + end + end + end + + defp fetch_ids(before, limit) do + Activity + |> where([a], fragment("?->>'type' = 'Create'", a.data)) + |> maybe_before(before) + |> order_by([a], desc: a.id) + |> limit(^limit) + |> select([a], a.id) + |> Pleroma.Repo.all() + end + + defp maybe_before(query, nil), do: query + defp maybe_before(query, before), do: where(query, [a], a.id < ^before) + + defp enqueue(ids, chunk_size) do + ids + |> Enum.chunk_every(chunk_size) + |> Enum.each(fn chunk -> + chunk + |> Enum.map(&SearchIndexingWorker.new(%{"op" => "add_to_index", "activity" => &1})) + |> Oban.insert_all() + end) + end + + defp page_size(:infinity, step_size), do: step_size + defp page_size(remaining, step_size), do: min(remaining, step_size) + + defp decrement(:infinity, _count), do: :infinity + defp decrement(remaining, count), do: remaining - count + + defp result(enqueued, next_cursor, exhausted) do + %{enqueued: enqueued, next_cursor: next_cursor, exhausted: exhausted} + end + + defp positive_option!(options, key, default) do + case Keyword.get(options, key, default) do + value when is_integer(value) and value > 0 -> value + value -> raise ArgumentError, "#{key} must be a positive integer, got: #{inspect(value)}" + end + end + + defp limit_option!(options) do + case Keyword.fetch!(options, :limit) do + :infinity -> :infinity + value when is_integer(value) and value > 0 -> value + value -> raise ArgumentError, "limit must be a positive integer, got: #{inspect(value)}" + end + end + + defp cursor_option!(nil), do: nil + + defp cursor_option!(cursor) do + valid? = + try do + match?({:ok, <<_::binary-size(16)>>}, FlakeId.Ecto.CompatType.dump(cursor)) + rescue + _ -> false + end + + if valid? do + cursor + else + raise ArgumentError, "before must be a valid activity id, got: #{inspect(cursor)}" + end + end + + defp callback_option!(nil), do: fn _ -> :ok end + defp callback_option!(callback) when is_function(callback, 1), do: callback + + defp callback_option!(callback) do + raise ArgumentError, "on_page must be a function of arity 1, got: #{inspect(callback)}" + end +end diff --git a/test/pleroma/search/backfill_test.exs b/test/pleroma/search/backfill_test.exs new file mode 100644 index 000000000..d707524c3 --- /dev/null +++ b/test/pleroma/search/backfill_test.exs @@ -0,0 +1,79 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2026 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Search.BackfillTest do + use Pleroma.DataCase, async: false + use Oban.Testing, repo: Pleroma.Repo + + import Pleroma.Factory + + alias Pleroma.Search.Backfill + alias Pleroma.Workers.SearchIndexingWorker + + test "enqueues the exact requested number across partial pages" do + activities = insert_list(5, :note_activity) |> Enum.sort_by(& &1.id, :desc) + + assert %{enqueued: 3, exhausted: false, next_cursor: cursor} = + Backfill.run(limit: 3, step: 2, chunk: 2) + + assert search_indexing_ids() == Enum.map(Enum.take(activities, 3), & &1.id) + assert cursor == Enum.at(activities, 2).id + end + + test "resumes after the last processed activity without duplicates" do + activities = insert_list(5, :note_activity) |> Enum.sort_by(& &1.id, :desc) + + assert %{enqueued: 2, next_cursor: cursor} = Backfill.run(limit: 2, step: 1) + + Pleroma.Repo.delete_all(Oban.Job) + + assert %{enqueued: 2, next_cursor: next_cursor} = + Backfill.run(before: cursor, limit: 2, step: 1) + + assert search_indexing_ids() == + activities + |> Enum.slice(2, 2) + |> Enum.map(& &1.id) + + assert next_cursor == Enum.at(activities, 3).id + end + + test "only queues Create activities" do + create = insert(:note_activity) + insert(:like_activity, note_activity: create) + + assert %{enqueued: 1, exhausted: true, next_cursor: cursor} = Backfill.run(step: 2) + assert search_indexing_ids() == [create.id] + assert cursor == create.id + end + + test "rejects unknown options and malformed cursors before enqueueing" do + assert_raise ArgumentError, fn -> Backfill.run(limt: 1) end + assert_raise ArgumentError, fn -> Backfill.run(before: "not-an-activity-id") end + assert_raise ArgumentError, fn -> Backfill.run(on_page: :invalid) end + + assert all_enqueued(worker: SearchIndexingWorker) == [] + end + + test "does not duplicate or skip the initial activities when newer rows are inserted" do + activities = insert_list(3, :note_activity) |> Enum.sort_by(& &1.id, :desc) + + assert %{enqueued: 3, exhausted: true} = + Backfill.run( + limit: :infinity, + step: 1, + on_page: fn %{enqueued: enqueued} -> + if enqueued == 1, do: insert(:note_activity) + end + ) + + assert search_indexing_ids() == Enum.map(activities, & &1.id) + end + + defp search_indexing_ids do + all_enqueued(worker: SearchIndexingWorker) + |> Enum.sort_by(& &1.id) + |> Enum.map(& &1.args["activity"]) + end +end From 271c3a3dd8a22fad2fa4cd6d034c98e1ffc91fb4 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sun, 26 Jul 2026 11:58:31 +0400 Subject: [PATCH 59/90] fix(config): keep search startup settings static --- lib/mix/tasks/pleroma/config.ex | 9 ++++--- lib/pleroma/config/transfer_task.ex | 1 + lib/pleroma/config_db.ex | 26 +++++++++++++++++-- .../controllers/config_controller.ex | 5 +++- test/fixtures/config/static_search.secret.exs | 5 ++++ test/mix/tasks/pleroma/config_test.exs | 8 ++++++ test/pleroma/config/transfer_task_test.exs | 25 ++++++++++++++++++ test/pleroma/config_db_test.exs | 14 ++++++++++ .../controllers/config_controller_test.exs | 20 ++++++++++++-- 9 files changed, 105 insertions(+), 8 deletions(-) create mode 100644 test/fixtures/config/static_search.secret.exs diff --git a/lib/mix/tasks/pleroma/config.ex b/lib/mix/tasks/pleroma/config.ex index a10506236..81c41d967 100644 --- a/lib/mix/tasks/pleroma/config.ex +++ b/lib/mix/tasks/pleroma/config.ex @@ -335,9 +335,12 @@ defmodule Mix.Tasks.Pleroma.Config do group |> Pleroma.Config.Loader.filter_group(settings) |> Enum.each(fn {key, value} -> - {:ok, _} = ConfigDB.update_or_create(%{group: group, key: key, value: value}) - - shell_info("Settings for key #{key} migrated.") + if ConfigDB.static?(group, key) do + shell_info("Settings for key #{key} must remain in static configuration; skipping.") + else + {:ok, _} = ConfigDB.update_or_create(%{group: group, key: key, value: value}) + shell_info("Settings for key #{key} migrated.") + end end) shell_info("Settings for group #{inspect(group)} migrated.") diff --git a/lib/pleroma/config/transfer_task.ex b/lib/pleroma/config/transfer_task.ex index 3e59dc4e0..48cb158ca 100644 --- a/lib/pleroma/config/transfer_task.ex +++ b/lib/pleroma/config/transfer_task.ex @@ -47,6 +47,7 @@ defmodule Pleroma.Config.TransferTask do settings = (Repo.all(ConfigDB) ++ deleted_settings) + |> Enum.reject(&ConfigDB.static?/1) |> Enum.map(&merge_with_default/1) started_applications = Application.started_applications() diff --git a/lib/pleroma/config_db.ex b/lib/pleroma/config_db.ex index 2c3df773b..9aa52f7d6 100644 --- a/lib/pleroma/config_db.ex +++ b/lib/pleroma/config_db.ex @@ -23,6 +23,12 @@ defmodule Pleroma.ConfigDB do {:pleroma, :mrf_keyword, :replace} ] + @static_config [ + {:pleroma, Pleroma.Search}, + {:pleroma, Pleroma.Search.ParadeDB}, + {:pleroma, Pleroma.Search.ParadeDB.Repo} + ] + schema "config" do field(:key, Pleroma.EctoType.Config.Atom) field(:group, Pleroma.EctoType.Config.Atom) @@ -32,16 +38,32 @@ defmodule Pleroma.ConfigDB do timestamps() end - @spec get_all_as_keyword() :: keyword() - def get_all_as_keyword do + @spec get_all_as_keyword(keyword()) :: keyword() + def get_all_as_keyword(options \\ []) do ConfigDB |> select([c], {c.group, c.key, c.value}) |> Repo.all() + |> maybe_reject_static(options) |> Enum.reduce([], fn {group, key, value}, acc -> Keyword.update(acc, group, [{key, value}], &Keyword.merge(&1, [{key, value}])) end) end + def static?(%{group: group, key: key}), do: static?(group, key) + def static?(_config), do: false + + def static?(group, key) do + Enum.any?(@static_config, fn {static_group, static_key} -> + group in [static_group, inspect(static_group)] and key in [static_key, inspect(static_key)] + end) + end + + defp maybe_reject_static(configs, exclude_static: true) do + Enum.reject(configs, fn {group, key, _value} -> static?(group, key) end) + end + + defp maybe_reject_static(configs, _options), do: configs + @spec get_all_by_group(atom() | String.t()) :: [t()] def get_all_by_group(group) do from(c in ConfigDB, where: c.group == ^group) |> Repo.all() diff --git a/lib/pleroma/web/admin_api/controllers/config_controller.ex b/lib/pleroma/web/admin_api/controllers/config_controller.ex index 753690d45..4c76d808d 100644 --- a/lib/pleroma/web/admin_api/controllers/config_controller.ex +++ b/lib/pleroma/web/admin_api/controllers/config_controller.ex @@ -75,6 +75,7 @@ defmodule Pleroma.Web.AdminAPI.ConfigController do Pleroma.Docs.JSON.compiled_descriptions() |> Enum.filter(&allowed_config?(&1, :whitelist)) |> Enum.reject(&allowed_config?(&1, :blacklist)) + |> Enum.reject(&ConfigDB.static?/1) json(conn, translate_descriptions(descriptions)) end @@ -92,7 +93,7 @@ defmodule Pleroma.Web.AdminAPI.ConfigController do def show(conn, _params) do with :ok <- configurable_from_database() do - configs = ConfigDB.get_all_as_keyword() + configs = ConfigDB.get_all_as_keyword(exclude_static: true) merged = Config.Holder.default_config() @@ -123,6 +124,7 @@ defmodule Pleroma.Web.AdminAPI.ConfigController do end) end) |> List.flatten() + |> Enum.reject(&ConfigDB.static?/1) render(conn, "index.json", %{ configs: merged, @@ -137,6 +139,7 @@ defmodule Pleroma.Web.AdminAPI.ConfigController do configs |> Enum.filter(&allowed_config?(&1, :whitelist)) |> Enum.reject(&allowed_config?(&1, :blacklist)) + |> Enum.reject(&(ConfigDB.static?(&1) and not &1[:delete])) |> Enum.map(fn %{group: group, key: key, delete: true} = params -> ConfigDB.delete(%{group: group, key: key, subkeys: params[:subkeys]}) diff --git a/test/fixtures/config/static_search.secret.exs b/test/fixtures/config/static_search.secret.exs new file mode 100644 index 000000000..13f744184 --- /dev/null +++ b/test/fixtures/config/static_search.secret.exs @@ -0,0 +1,5 @@ +import Config + +config :pleroma, Pleroma.Search, module: Pleroma.Search.ParadeDB +config :pleroma, Pleroma.Search.ParadeDB, url: "postgres://localhost/paradedb" +config :pleroma, Pleroma.Search.ParadeDB.Repo, pool_size: 2 diff --git a/test/mix/tasks/pleroma/config_test.exs b/test/mix/tasks/pleroma/config_test.exs index 7513e958e..620bf8662 100644 --- a/test/mix/tasks/pleroma/config_test.exs +++ b/test/mix/tasks/pleroma/config_test.exs @@ -87,6 +87,14 @@ defmodule Mix.Tasks.Pleroma.ConfigTest do config = ConfigDB.get_by_params(%{group: ":pleroma", key: ":first_setting"}) assert config.value == [key: "value", key2: [Repo]] end + + test "static search settings remain in the config file" do + MixTask.migrate_to_db("test/fixtures/config/static_search.secret.exs") + + refute ConfigDB.get_by_group_and_key(:pleroma, Pleroma.Search) + refute ConfigDB.get_by_group_and_key(:pleroma, Pleroma.Search.ParadeDB) + refute ConfigDB.get_by_group_and_key(:pleroma, Pleroma.Search.ParadeDB.Repo) + end end describe "with deletion of temp file" do diff --git a/test/pleroma/config/transfer_task_test.exs b/test/pleroma/config/transfer_task_test.exs index 6295fa888..ea5378ffd 100644 --- a/test/pleroma/config/transfer_task_test.exs +++ b/test/pleroma/config/transfer_task_test.exs @@ -71,6 +71,31 @@ defmodule Pleroma.Config.TransferTaskTest do assert assets_env[:mascots] == [a: 1, b: 2] end + test "does not transfer static search configuration" do + search_config = Application.get_env(:pleroma, Pleroma.Search) + parade_config = Application.get_env(:pleroma, Pleroma.Search.ParadeDB) + + clear_config(:database_config_blacklist, []) + + insert(:config, key: Pleroma.Search, value: [module: Pleroma.Search.ParadeDB]) + + insert(:config, + key: Pleroma.Search.ParadeDB, + value: [url: "postgres://localhost/paradedb"] + ) + + insert(:config, + key: Pleroma.Search.ParadeDB.Repo, + value: [pool_size: 50] + ) + + TransferTask.start_link([]) + + assert Application.get_env(:pleroma, Pleroma.Search) == search_config + assert Application.get_env(:pleroma, Pleroma.Search.ParadeDB) == parade_config + assert Application.get_env(:pleroma, Pleroma.Search.ParadeDB.Repo)[:pool_size] == 2 + end + describe "pleroma restart" do setup do on_exit(fn -> diff --git a/test/pleroma/config_db_test.exs b/test/pleroma/config_db_test.exs index 9d70e6d54..ae17a9d34 100644 --- a/test/pleroma/config_db_test.exs +++ b/test/pleroma/config_db_test.exs @@ -36,6 +36,20 @@ defmodule Pleroma.ConfigDBTest do assert config[:goose][:webhook_url] == "https://gander.com/" end + test "get_all_as_keyword/1 excludes static search configuration" do + insert(:config, key: Pleroma.Search, value: [module: Pleroma.Search.ParadeDB]) + insert(:config, key: Pleroma.Search.ParadeDB, value: [url: "postgres://example/db"]) + insert(:config, key: Pleroma.Search.ParadeDB.Repo, value: [pool_size: 50]) + insert(:config, key: :instance, value: [name: "Pleroma"]) + + config = ConfigDB.get_all_as_keyword(exclude_static: true) + + refute config[:pleroma][Pleroma.Search] + refute config[:pleroma][Pleroma.Search.ParadeDB] + refute config[:pleroma][Pleroma.Search.ParadeDB.Repo] + assert config[:pleroma][:instance] == [name: "Pleroma"] + end + describe "update_or_create/1" do test "common" do config = insert(:config) diff --git a/test/pleroma/web/admin_api/controllers/config_controller_test.exs b/test/pleroma/web/admin_api/controllers/config_controller_test.exs index c11ece10d..e4c8d866f 100644 --- a/test/pleroma/web/admin_api/controllers/config_controller_test.exs +++ b/test/pleroma/web/admin_api/controllers/config_controller_test.exs @@ -121,6 +121,19 @@ defmodule Pleroma.Web.AdminAPI.ConfigControllerTest do end) end + test "effective configuration omits static search settings", %{conn: conn} do + insert(:config, key: Pleroma.Search, value: [module: Pleroma.Search.ParadeDB]) + insert(:config, key: Pleroma.Search.ParadeDB, value: [url: "postgres://example/db"]) + + %{"configs" => configs} = + conn + |> get("/api/pleroma/admin/config") + |> json_response_and_validate_schema(200) + + refute Enum.any?(configs, &(&1["key"] == "Pleroma.Search")) + refute Enum.any?(configs, &(&1["key"] == "Pleroma.Search.ParadeDB")) + end + test "subkeys with full update right merge", %{conn: conn} do insert(:config, key: ":emoji", @@ -1552,12 +1565,15 @@ defmodule Pleroma.Web.AdminAPI.ConfigControllerTest do assert web_endpoint["children"] end - test "all keys from description are whitelisted", %{conn: conn} do + test "descriptions exclude blacklisted keys", %{conn: conn} do conn = get(conn, "/api/pleroma/admin/config/descriptions") assert response = json_response_and_validate_schema(conn, 200) - assert length(response) == length(Pleroma.Docs.JSON.compiled_descriptions()) + refute Enum.any?(response, &(&1["key"] == "Pleroma.Search")) + refute Enum.any?(response, &(&1["key"] == "Pleroma.Search.ParadeDB")) + + assert length(response) == length(Pleroma.Docs.JSON.compiled_descriptions()) - 2 end test "filters by database configuration blacklist and whitelist", %{conn: conn} do From 13c944a04b217a534cd9ef0d0f9d8dfd65609faf Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sun, 26 Jul 2026 12:03:24 +0400 Subject: [PATCH 60/90] fix(search): index remote attachment descriptions --- lib/pleroma/search.ex | 1 + test/pleroma/search_test.exs | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/lib/pleroma/search.ex b/lib/pleroma/search.ex index cd4907026..0d9c7c6e3 100644 --- a/lib/pleroma/search.ex +++ b/lib/pleroma/search.ex @@ -78,6 +78,7 @@ defmodule Pleroma.Search do defp attachment_names(attachments) when is_list(attachments) do Enum.map(attachments, fn + %{"summary" => summary} when is_binary(summary) -> summary %{"name" => name} -> name _ -> nil end) diff --git a/test/pleroma/search_test.exs b/test/pleroma/search_test.exs index 00d8b10ed..91521b1a5 100644 --- a/test/pleroma/search_test.exs +++ b/test/pleroma/search_test.exs @@ -83,6 +83,11 @@ defmodule Pleroma.SearchTest do "summary" => "subject text", "attachment" => [ %{"type" => "Document", "name" => "image description"}, + %{ + "type" => "Document", + "name" => "remote-image.jpg", + "summary" => "remote image description" + }, %{"type" => "Document", "name" => ""} ], "published" => DateTime.utc_now() |> DateTime.to_iso8601() @@ -93,6 +98,8 @@ defmodule Pleroma.SearchTest do assert content =~ "body text" assert content =~ "subject text" assert content =~ "image description" + assert content =~ "remote image description" + refute content =~ "remote-image.jpg" end test "object_to_search_data ignores objects with invalid published dates" do From f90a52398ba8825d70db9f4594904eca02e248b5 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sun, 26 Jul 2026 12:16:04 +0400 Subject: [PATCH 61/90] ci(search): gate pinned ParadeDB integration --- .woodpecker/paradedb-integration.yaml | 64 ++++++++ .../search/parade_db_integration_test.exs | 145 +++++++++++++----- 2 files changed, 171 insertions(+), 38 deletions(-) create mode 100644 .woodpecker/paradedb-integration.yaml diff --git a/.woodpecker/paradedb-integration.yaml b/.woodpecker/paradedb-integration.yaml new file mode 100644 index 000000000..b91222837 --- /dev/null +++ b/.woodpecker/paradedb-integration.yaml @@ -0,0 +1,64 @@ +when: + - event: pull_request + path: + - ".woodpecker/**" + - "lib/pleroma/search/**" + - "test/pleroma/search/**" + - "mix.exs" + - "mix.lock" + evaluate: 'CI_COMMIT_SOURCE_BRANCH != "weblate"' + - event: push + branch: ${CI_REPO_DEFAULT_BRANCH} + path: + - ".woodpecker/**" + - "lib/pleroma/search/**" + - "test/pleroma/search/**" + - "mix.exs" + - "mix.lock" + +depends_on: + - lint + +labels: + platform: linux/amd64 + +variables: + script_file_entrypoint: &script_file_entrypoint + - /bin/sh + - -c + - 'printf "%s" "$CI_SCRIPT" | base64 -d > /tmp/ci-script.sh && /bin/sh -xe /tmp/ci-script.sh' + +steps: + paradedb-integration: + image: docker.io/elixir:1.19-otp-28-alpine + entrypoint: *script_file_entrypoint + environment: + MIX_ENV: test + VIX_COMPILATION_MODE: PLATFORM_PROVIDED_LIBVIPS + DB_HOST: postgres + DB_PORT: 5432 + PARADEDB_TEST_URL: postgres://postgres:postgres@paradedb:5432/paradedb + commands: + - apk add --no-cache build-base cmake exiftool ffmpeg vips-dev file-dev git openssl postgresql-client + - mix local.hex --force + - mix local.rebar --force + - mix deps.get + - timeout 60 sh -c 'until pg_isready -h postgres -U postgres -d pleroma_test; do sleep 1; done' || { echo "PostgreSQL did not become ready" >&2; exit 1; } + - timeout 60 sh -c 'until pg_isready -h paradedb -U postgres -d paradedb; do sleep 1; done' || { echo "ParadeDB did not become ready" >&2; exit 1; } + - mix ecto.create + - mix ecto.migrate + - mix test --include integration test/pleroma/search/parade_db_integration_test.exs + +services: + postgres: + image: docker.io/postgres:13-alpine + environment: + POSTGRES_DB: pleroma_test + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + paradedb: + image: docker.io/paradedb/paradedb:0.24.3-pg15@sha256:997d62f6c2605becb7c45878af3f8c8c698f355006b675b3ce212d4eb2d14f2e + environment: + POSTGRES_DB: paradedb + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres diff --git a/test/pleroma/search/parade_db_integration_test.exs b/test/pleroma/search/parade_db_integration_test.exs index 2687260cf..4336ecfc3 100644 --- a/test/pleroma/search/parade_db_integration_test.exs +++ b/test/pleroma/search/parade_db_integration_test.exs @@ -14,7 +14,10 @@ defmodule Pleroma.Search.ParadeDBIntegrationTest do alias Pleroma.Search.ParadeDB alias Pleroma.Web.CommonAPI - @image System.get_env("PARADEDB_TEST_IMAGE", "paradedb/paradedb:latest") + @image System.get_env( + "PARADEDB_TEST_IMAGE", + "paradedb/paradedb:0.24.3-pg15@sha256:997d62f6c2605becb7c45878af3f8c8c698f355006b675b3ce212d4eb2d14f2e" + ) @db_user "postgres" @db_password "postgres" @db_name "paradedb" @@ -26,6 +29,13 @@ defmodule Pleroma.Search.ParadeDBIntegrationTest do end setup_all do + case System.get_env("PARADEDB_TEST_URL") do + nil -> setup_docker_paradedb() + url -> setup_external_paradedb(url) + end + end + + defp setup_docker_paradedb do with :ok <- ensure_docker_available() do container = "pleroma-paradedb-it-#{System.unique_integer([:positive])}" @@ -58,49 +68,67 @@ defmodule Pleroma.Search.ParadeDBIntegrationTest do |> wait_for_port!(@docker_startup_timeout_ms) wait_for_pg_ready!(container, @docker_startup_timeout_ms) - wait_for_tcp_ready!(host_port, @docker_startup_timeout_ms) url = "postgres://#{@db_user}:#{@db_password}@127.0.0.1:#{host_port}/#{@db_name}" - - prev_env = System.get_env("PARADEDB_DATABASE_URL") - System.put_env("PARADEDB_DATABASE_URL", url) - - on_exit(fn -> - if is_nil(prev_env) do - System.delete_env("PARADEDB_DATABASE_URL") - else - System.put_env("PARADEDB_DATABASE_URL", prev_env) - end - end) - - {:ok, _pid} = start_supervised({Pleroma.Search.ParadeDB.Repo, pool_size: 5}) - - previous_table = Pleroma.Config.get([Pleroma.Search.ParadeDB, :table], nil) - table = "pleroma_search_documents_it_#{System.unique_integer([:positive])}" - Pleroma.Config.put([Pleroma.Search.ParadeDB, :table], table) - - on_exit(fn -> - if is_nil(previous_table) do - Pleroma.Config.delete([Pleroma.Search.ParadeDB, :table]) - else - Pleroma.Config.put([Pleroma.Search.ParadeDB, :table], previous_table) - end - end) - - {:ok, - %{ - container: container, - container_id: container_id, - url: url, - table: table - }} + setup_repo(url, %{container: container, container_id: container_id}) end end - test "indexes and searches against real ParadeDB" do + defp setup_external_paradedb(url) do + uri = URI.parse(url) + wait_for_tcp_ready!(uri.host, uri.port || 5432, @docker_startup_timeout_ms) + setup_repo(url, %{}) + end + + defp setup_repo(url, context) do + prev_env = System.get_env("PARADEDB_DATABASE_URL") + System.put_env("PARADEDB_DATABASE_URL", url) + + on_exit(fn -> + if is_nil(prev_env) do + System.delete_env("PARADEDB_DATABASE_URL") + else + System.put_env("PARADEDB_DATABASE_URL", prev_env) + end + end) + + {:ok, _pid} = start_supervised({Pleroma.Search.ParadeDB.Repo, pool_size: 5}) + wait_for_repo_ready!(@docker_startup_timeout_ms) + + previous_table = Pleroma.Config.get([Pleroma.Search.ParadeDB, :table], nil) + suffix = Ecto.UUID.generate() |> String.replace("-", "") + table = "pleroma_search_documents_it_#{suffix}" + Pleroma.Config.put([Pleroma.Search.ParadeDB, :table], table) + + on_exit(fn -> + if is_nil(previous_table) do + Pleroma.Config.delete([Pleroma.Search.ParadeDB, :table]) + else + Pleroma.Config.put([Pleroma.Search.ParadeDB, :table], previous_table) + end + end) + + {:ok, Map.merge(context, %{url: url, table: table})} + end + + test "indexes and searches against real ParadeDB", %{table: table} do + on_exit(fn -> + _ = + Ecto.Adapters.SQL.query( + Pleroma.Search.ParadeDB.Repo, + "DROP TABLE IF EXISTS #{table}" + ) + end) + :ok = ParadeDB.drop_index() assert :ok = ParadeDB.create_index() + assert {:ok, %{rows: [["0.24.3"]]}} = + Ecto.Adapters.SQL.query( + Pleroma.Search.ParadeDB.Repo, + "SELECT extversion FROM pg_extension WHERE extname = 'pg_search'" + ) + user = insert(:user) {:ok, a1} = @@ -118,6 +146,23 @@ defmodule Pleroma.Search.ParadeDBIntegrationTest do a2 = Activity.get_by_id_with_object(a2.id) private = Activity.get_by_id_with_object(private.id) + a1 = %{ + a1 + | object: %{ + a1.object + | data: + Map.merge(a1.object.data, %{ + "summary" => "paradedb subject phrase", + "attachment" => [ + %{ + "name" => "remote-image.jpg", + "summary" => "paradedb attachment description" + } + ] + }) + } + } + assert :ok = ParadeDB.add_to_index(a1) assert :ok = ParadeDB.add_to_index(a2) assert :ok = ParadeDB.add_to_index(private) @@ -130,6 +175,9 @@ defmodule Pleroma.Search.ParadeDBIntegrationTest do refute private.id in ids assert List.first(ids) == a2.id + assert Enum.any?(ParadeDB.search(nil, "subject phrase"), &(&1.id == a1.id)) + assert Enum.any?(ParadeDB.search(nil, "attachment description"), &(&1.id == a1.id)) + previous_fuzzy_distance = Pleroma.Config.get([Pleroma.Search.ParadeDB, :fuzzy_distance], 0) Pleroma.Config.put([Pleroma.Search.ParadeDB, :fuzzy_distance], 1) @@ -226,11 +274,11 @@ defmodule Pleroma.Search.ParadeDBIntegrationTest do wait.(wait) end - defp wait_for_tcp_ready!(port, timeout_ms) do + defp wait_for_tcp_ready!(host, port, timeout_ms) do deadline = System.monotonic_time(:millisecond) + timeout_ms wait = fn wait -> - case :gen_tcp.connect(~c"127.0.0.1", port, [:binary, active: false], 500) do + case :gen_tcp.connect(String.to_charlist(host), port, [:binary, active: false], 500) do {:ok, socket} -> :gen_tcp.close(socket) :ok @@ -248,6 +296,27 @@ defmodule Pleroma.Search.ParadeDBIntegrationTest do wait.(wait) end + defp wait_for_repo_ready!(timeout_ms) do + deadline = System.monotonic_time(:millisecond) + timeout_ms + + wait = fn wait -> + case Ecto.Adapters.SQL.query(Pleroma.Search.ParadeDB.Repo, "SELECT 1") do + {:ok, _} -> + :ok + + _ -> + if System.monotonic_time(:millisecond) > deadline do + raise "Timed out waiting for ParadeDB Repo" + else + :timer.sleep(250) + wait.(wait) + end + end + end + + wait.(wait) + end + defp docker(args) do System.cmd("docker", args, stderr_to_stdout: true) end From 9df50eea82e4141bca673750448c5e795d31fa49 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sun, 26 Jul 2026 12:26:35 +0400 Subject: [PATCH 62/90] fix: reject duplicate Signature headers --- .../activitypub-bot-rfc9421-fallback.fix | 2 +- .../activity_pub/activity_pub_controller.ex | 42 ++++++++---- .../activity_pub_controller_test.exs | 67 +++++++++++++++++++ 3 files changed, 97 insertions(+), 14 deletions(-) diff --git a/changelog.d/activitypub-bot-rfc9421-fallback.fix b/changelog.d/activitypub-bot-rfc9421-fallback.fix index 22642fee1..a04940bfe 100644 --- a/changelog.d/activitypub-bot-rfc9421-fallback.fix +++ b/changelog.d/activitypub-bot-rfc9421-fallback.fix @@ -1 +1 @@ -Reject unsupported RFC 9421 inbox signatures instead of queueing a doomed failed-signature retry, allowing clients to fall back to legacy HTTP Signatures. +Reject unsupported or ambiguous inbox signatures instead of queueing a doomed failed-signature retry, allowing clients to fall back to legacy HTTP Signatures. diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index dd644c132..38b8b8bfc 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -309,7 +309,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do } = conn, %{"nickname" => nickname} = params ) do - with {:recipient_exists, %User{} = recipient} <- + with false <- unsupported_http_message_signature?(conn), + {:recipient_exists, %User{} = recipient} <- {:recipient_exists, User.get_cached_by_nickname(nickname)}, {:sender_exists, {:ok, %User{} = actor}} <- {:sender_exists, User.get_or_fetch_by_ap_id(params["actor"])}, @@ -320,6 +321,9 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do Federator.incoming_ap_doc(params) json(conn, "ok") else + true -> + reject_unsupported_http_message_signature(conn) + {:recipient_exists, _} -> conn |> put_status(:not_found) @@ -348,15 +352,17 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do end def inbox(%{assigns: %{valid_signature: true, valid_host_header: true}} = conn, params) do - Federator.incoming_ap_doc(params) - json(conn, "ok") + if unsupported_http_message_signature?(conn) do + reject_unsupported_http_message_signature(conn) + else + Federator.incoming_ap_doc(params) + json(conn, "ok") + end end def inbox(%{assigns: %{valid_signature: false}} = conn, params) do if unsupported_http_message_signature?(conn) do - conn - |> put_status(:unauthorized) - |> json("error, unsupported HTTP Message Signature") + reject_unsupported_http_message_signature(conn) else Federator.incoming_failed_signature_ap_doc(%{ method: conn.method, @@ -387,15 +393,25 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do |> json("error, missing HTTP Signature") end - defp unsupported_http_message_signature?(conn) do - Plug.Conn.get_req_header(conn, "signature-input") != [] and - not cavage_signature?(conn) + defp reject_unsupported_http_message_signature(conn) do + conn + |> put_status(:unauthorized) + |> json("error, unsupported HTTP Message Signature") end - defp cavage_signature?(conn) do - conn - |> Plug.Conn.get_req_header("signature") - |> Enum.any?(&Regex.match?(~r/(^|,)\s*keyId\s*=/, &1)) + defp unsupported_http_message_signature?(conn) do + signature_headers = Plug.Conn.get_req_header(conn, "signature") + + multiple_signature_headers?(signature_headers) or + (Plug.Conn.get_req_header(conn, "signature-input") != [] and + not cavage_signature?(signature_headers)) + end + + defp multiple_signature_headers?([_, _ | _]), do: true + defp multiple_signature_headers?(_), do: false + + defp cavage_signature?(signature_headers) do + Enum.any?(signature_headers, &Regex.match?(~r/(^|,)\s*keyId\s*=/, &1)) end defp post_inbox_relayed_create(conn, params) do diff --git a/test/pleroma/web/activity_pub/activity_pub_controller_test.exs b/test/pleroma/web/activity_pub/activity_pub_controller_test.exs index e9a777758..0de9618d4 100644 --- a/test/pleroma/web/activity_pub/activity_pub_controller_test.exs +++ b/test/pleroma/web/activity_pub/activity_pub_controller_test.exs @@ -729,6 +729,22 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do assert Activity.get_by_ap_id(data["id"]) end + test "rejects duplicate Signature headers before processing signed inbox requests", %{ + conn: conn + } do + data = File.read!("test/fixtures/mastodon-post-activity.json") |> Jason.decode!() + + conn = + conn + |> assign_valid_signature_for_actor(data["actor"]) + |> put_req_header("content-type", "application/activity+json") + |> Map.update!(:req_headers, &[{"signature", "sig1=:fake-signature:"} | &1]) + |> post("/inbox", data) + + assert "error, unsupported HTTP Message Signature" == json_response(conn, 401) + assert [] == all_enqueued(worker: ReceiverWorker) + end + test "it inserts an incoming activity into the database" <> "even if we can't fetch the user but have it in our db", %{conn: conn} do @@ -794,6 +810,43 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do assert [] == all_enqueued(worker: SignatureRetryWorker) end + test "rejects duplicate Signature headers so clients can fall back", %{conn: conn} do + data = %{ + "type" => "Create", + "actor" => "https://activitypubbot.example/user/ok", + "id" => "https://activitypubbot.example/activities/duplicate-signature-create", + "to" => ["https://www.w3.org/ns/activitystreams#Public"], + "cc" => [], + "object" => %{ + "type" => "Note", + "id" => "https://activitypubbot.example/objects/duplicate-signature-note", + "actor" => "https://activitypubbot.example/user/ok", + "attributedTo" => "https://activitypubbot.example/user/ok", + "content" => "hello from activitypub-bot", + "published" => "2026-05-29T13:33:31Z", + "to" => ["https://www.w3.org/ns/activitystreams#Public"], + "cc" => [] + } + } + + legacy_signature = + "keyId=\"https://activitypubbot.example/user/ok#main-key\",algorithm=\"rsa-sha256\",headers=\"(request-target) host date digest content-type\",signature=\"fake-signature\"" + + Mox.expect(Pleroma.StubbedHTTPSignaturesMock, :validate_conn, fn _conn -> false end) + + conn = + conn + |> put_req_header("content-type", "application/activity+json") + |> put_req_header("date", "Fri, 29 May 2026 13:33:31 GMT") + |> put_req_header("digest", "SHA-256=fake-digest") + |> put_req_header("signature", "sig1=:fake-signature:") + |> Map.update!(:req_headers, &[{"signature", legacy_signature} | &1]) + |> post("/inbox", data) + + assert "error, unsupported HTTP Message Signature" == json_response(conn, 401) + assert [] == all_enqueued(worker: SignatureRetryWorker) + end + test "keeps failed-signature retry for legacy signatures with Signature-Input", %{conn: conn} do data = %{ "type" => "Create", @@ -1397,6 +1450,20 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do assert Activity.get_by_ap_id(data["id"]) end + test "rejects duplicate Signature headers", %{conn: conn, data: data} do + user = insert(:user) + + conn = + conn + |> assign_valid_signature_for_actor(data["actor"]) + |> put_req_header("content-type", "application/activity+json") + |> Map.update!(:req_headers, &[{"signature", "sig1=:fake-signature:"} | &1]) + |> post("/users/#{user.nickname}/inbox", data) + + assert "error, unsupported HTTP Message Signature" == json_response(conn, 401) + assert [] == all_enqueued(worker: ReceiverWorker) + end + test "it accepts messages with to as string instead of array", %{conn: conn, data: data} do user = insert(:user) From cece76e4cbce194a56847057f120550794c4f4fe Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sun, 26 Jul 2026 12:32:24 +0400 Subject: [PATCH 63/90] docs(search): document ParadeDB operations --- docs/configuration/search.md | 125 ++++++++++++++++++++++++++++++++--- 1 file changed, 117 insertions(+), 8 deletions(-) diff --git a/docs/configuration/search.md b/docs/configuration/search.md index 1d5056ddf..5820cdbd9 100644 --- a/docs/configuration/search.md +++ b/docs/configuration/search.md @@ -150,18 +150,22 @@ depends on the amount of text in posts. [ParadeDB](https://www.paradedb.com/) is a Postgres extension that provides BM25 full-text search. In Pleroma, it can be used as an external search backend by pointing Pleroma at a separate Postgres instance with the `pg_search` extension installed. +Pleroma's integration suite tests `pg_search` 0.24.3 on PostgreSQL 15. Pleroma will maintain a small `pleroma_search_documents` table in that database (via the existing search indexing queue) and run search queries against it. The indexed text includes status content, content warnings, and attachment descriptions. -To use it, set the search module to `Pleroma.Search.ParadeDB` and configure the ParadeDB database URL: -These settings must be in static/runtime config (or `PARADEDB_DATABASE_URL`) so the dedicated ParadeDB Repo is started with the application. +### Configuration + +Set the search module to `Pleroma.Search.ParadeDB` and configure the ParadeDB database URL. These settings must be in static/runtime config +(or `PARADEDB_DATABASE_URL`) so the dedicated ParadeDB Repo is started with the application. They cannot be migrated to or changed through +ConfigDB. > config :pleroma, Pleroma.Search, module: Pleroma.Search.ParadeDB > > config :pleroma, Pleroma.Search.ParadeDB, -> url: System.get_env("PARADEDB_DATABASE_URL") || "postgres://postgres:postgres@127.0.0.1:5432/paradedb", +> url: System.get_env("PARADEDB_DATABASE_URL"), > table: "pleroma_search_documents", > fuzzy_distance: 0 > @@ -169,22 +173,127 @@ These settings must be in static/runtime config (or `PARADEDB_DATABASE_URL`) so > pool_size: 2, > prepare: :unnamed -Then, create the table and BM25 index once: +When connecting over an untrusted network, enable certificate verification in the Repo configuration. For example: + +> config :pleroma, Pleroma.Search.ParadeDB.Repo, +> ssl: [ +> verify: :verify_peer, +> cacertfile: "/etc/pleroma/paradedb-ca.pem", +> server_name_indication: ~c"paradedb.example.com" +> ] + +### Initial rollout + +The `pg_search` extension must already be available in the database. Creating the extension normally requires an administrative database +role. The Pleroma runtime role requires `CONNECT` on the database, `USAGE` on the table's schema and the `pdb` schema, and `SELECT`, `INSERT`, `UPDATE`, +and `DELETE` on the generated table. For example, after creating the table as its owner: + +```sql +GRANT CONNECT ON DATABASE paradedb TO pleroma_search; +GRANT USAGE ON SCHEMA public, pdb TO pleroma_search; +GRANT SELECT, INSERT, UPDATE, DELETE ON TABLE pleroma_search_documents TO pleroma_search; +``` + +Use this rollout order to prevent jobs from being handled by the previous backend: + +1. Install `pg_search` 0.24.3 on PostgreSQL 15 or another deliberately tested combination. +2. Stop Pleroma so no search indexing jobs are running. +3. Put the ParadeDB backend in static configuration. +4. Create the table and BM25 index in a one-shot offline process using `PARADEDB_DATABASE_URL` with the table owner/setup credential. +5. Apply the runtime grants shown above, set the service's `PARADEDB_DATABASE_URL` to the restricted runtime credential, and start Pleroma. +6. Start the initial backfill and monitor the `search_indexing` queue. + +Create the table and BM25 index once with: + +=== "OTP" + ```sh + PARADEDB_DATABASE_URL='OWNER_URL' PLEROMA_CTL_RPC_DISABLED=true \ + ./bin/pleroma_ctl search.indexer create_index + ``` === "From Source" ```sh - mix pleroma.search.indexer create_index + PARADEDB_DATABASE_URL='OWNER_URL' mix pleroma.search.indexer create_index ``` -After that, run an initial indexing pass to backfill existing posts: +The task currently prints an error without returning a failing process status. Verify that it prints `Index created` before continuing. + +### Backfill + +Run an initial indexing pass to enqueue existing posts: + +=== "OTP" + ```sh + ./bin/pleroma_ctl search.indexer index + ``` === "From Source" ```sh - mix pleroma.search.indexer index + PARADEDB_DATABASE_URL='RUNTIME_URL' mix pleroma.search.indexer index ``` +The command enqueues up to 100,000 Create activities by default. It uses keyset pagination, so new activities arriving during the backfill +do not shift the remaining pages. The defaults and accepted options are: + +* `--limit`: maximum activities to enqueue in this run; defaults to `100000` and must be positive +* `--step`: activities fetched from Pleroma's database per page; defaults to `1000` and must be positive +* `--chunk`: jobs inserted into Oban per batch; defaults to `100` and must be positive +* `--before`: resume before the checkpoint activity ID printed by an earlier run; the cursor is exclusive + +The checkpoint is the last activity enqueued, not confirmation that its job succeeded. Repeat the command with its last `--before` +checkpoint only when it reports that more activities remain. Inspect failed or discarded Oban jobs as well as waiting jobs before declaring +the backfill complete. + +The same operation can be run from an attached console while Pleroma is online: + +```elixir +Pleroma.Search.Backfill.run(limit: 10_000) +``` + +The returned `next_cursor` can be supplied as `before` when `exhausted` is `false`. The attached API also accepts `limit: :infinity`, but this +can create an unbounded number of Oban rows; bounded runs are safer on large instances. An `on_page` callback may be supplied for progress +reporting. + Note: Like the other external search backends, only public/unlisted Notes are indexed. Set `fuzzy_distance` to `1` or `2` to allow typo-tolerant matching (`2` is the maximum ParadeDB supports). -Indexing runs via the Oban `search_indexing` queue, so ensure that queue is enabled (not paused) in your Oban configuration. +Indexing runs via the Oban `search_indexing` queue. Its stock configuration starts paused so the search health monitor can control it; verify +that the queue is running before backfilling. A manually managed queue can be enabled with `search_indexing: 10` in the Oban `queues` +configuration instead of a `paused: true` entry. +Monitor that queue until the backfill jobs have drained before evaluating search completeness. + +### Rebuilding + +Stop Pleroma, or otherwise ensure no search indexing jobs are running, before rebuilding. While ParadeDB is still the configured backend, +run `drop_index`, then `create_index` with the table owner/setup credential. Reapply runtime grants after recreating the table, restart +Pleroma, and begin a new backfill. The DDL tasks print failures without returning a failing process status, so verify the exact `Index +dropped` and `Index created` success messages. Dropping the table removes only derived search data; Pleroma activities remain in the primary +database. + +### Rolling back + +To retain the ParadeDB table for a later retry, configure `Pleroma.Search.DatabaseSearch` (or the previous external backend) in static config +and restart Pleroma. + +Before switching backends, inspect the `search_indexing` queue. Drain or cancel pending backfill jobs, or explicitly accept that they will run +against the newly configured backend after restart. + +To remove the table during rollback, first stop Pleroma and run `drop_index` with the table owner/setup credential while ParadeDB is still +configured: + +=== "OTP" + ```sh + PARADEDB_DATABASE_URL='OWNER_URL' PLEROMA_CTL_RPC_DISABLED=true \ + ./bin/pleroma_ctl search.indexer drop_index + ``` + +=== "From Source" + ```sh + PARADEDB_DATABASE_URL='OWNER_URL' mix pleroma.search.indexer drop_index + ``` + +Verify that the task prints `Index dropped`. Then change the backend in static config and restart. The ParadeDB environment variable may be +removed after removing the ParadeDB stanza or switching to a backend that does not start its Repo. After switching away from ParadeDB, the +generic `drop_index` task targets the newly configured backend and will not remove the ParadeDB table; remove it directly with the database +owner instead. From ce5c1b4c15ef89b077f62cde6f75379af68ac601 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sun, 26 Jul 2026 12:35:09 +0400 Subject: [PATCH 64/90] fix(activitypub): harden object normalization --- changelog.d/fedidev-federation-compat.add | 1 + lib/pleroma/object/containment.ex | 6 +- .../object_validators/tag_validator.ex | 32 +++- .../web/activity_pub/transmogrifier.ex | 146 ++++++++++++------ test/pleroma/object/containment_test.exs | 5 + .../fedidev_fun_attachments_test.exs | 33 +++- .../transmogrifier/fedidev_fun_emoji_test.exs | 42 ++++- .../fedidev_fun_object_type_list_test.exs | 11 ++ .../fedidev_fun_tag_inference_test.exs | 47 ++++++ .../transmogrifier/note_handling_test.exs | 38 ++++- .../user_update_handling_test.exs | 19 +++ 11 files changed, 315 insertions(+), 65 deletions(-) create mode 100644 changelog.d/fedidev-federation-compat.add diff --git a/changelog.d/fedidev-federation-compat.add b/changelog.d/fedidev-federation-compat.add new file mode 100644 index 000000000..bb29925f9 --- /dev/null +++ b/changelog.d/fedidev-federation-compat.add @@ -0,0 +1 @@ +Accept more real-world ActivityPub object shapes collected by fedidev.fun diff --git a/lib/pleroma/object/containment.ex b/lib/pleroma/object/containment.ex index b90d68636..c4cc1f014 100644 --- a/lib/pleroma/object/containment.ex +++ b/lib/pleroma/object/containment.ex @@ -79,9 +79,11 @@ defmodule Pleroma.Object.Containment do def contain_origin(id, %{"actor" => _actor} = params) do id_uri = URI.parse(id) - actor_uri = URI.parse(get_actor(params)) - compare_uris(actor_uri, id_uri) + case get_actor(params) do + actor when is_binary(actor) -> compare_uris(URI.parse(actor), id_uri) + _ -> :error + end end def contain_origin(id, %{"attributedTo" => actor} = params), diff --git a/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex b/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex index cdab8d78d..0659c93d7 100644 --- a/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex @@ -12,6 +12,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.TagValidator do require Pleroma.Constants @primary_key false + @tag_types ~w[Mention Hashtag Emoji Link] + embedded_schema do # Common field(:type, :string) @@ -49,7 +51,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.TagValidator do |> validate_required([:type, :href]) end - def changeset(struct, %{"type" => "Hashtag", "name" => name} = data) do + def changeset(struct, %{"type" => "Hashtag", "name" => name} = data) when is_binary(name) do name = String.downcase(name) data = Map.put(data, "name", name) @@ -58,7 +60,13 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.TagValidator do |> validate_required([:type, :name]) end - def changeset(struct, %{"type" => "Emoji"} = data) do + def changeset(struct, %{"type" => "Hashtag"} = data) do + struct + |> cast(data, []) + |> Map.put(:action, :ignore) + end + + def changeset(struct, %{"type" => "Emoji", "name" => name} = data) when is_binary(name) do data = data |> Map.put("name", String.trim(data["name"], ":")) @@ -84,6 +92,12 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.TagValidator do end end + def changeset(struct, %{"type" => "Emoji"} = data) do + struct + |> cast(data, []) + |> Map.put(:action, :ignore) + end + def changeset(struct, %{"type" => "Link"} = data) do struct |> cast(data, [:type, :name, :mediaType, :href]) @@ -91,6 +105,10 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.TagValidator do |> validate_required([:type, :href, :mediaType]) end + def changeset(struct, %{"type" => types} = data) when is_list(types) do + changeset(struct, infer_type(data)) + end + def changeset(struct, %{"type" => _} = data) do struct |> cast(data, []) @@ -98,7 +116,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.TagValidator do end def changeset(struct, data) when is_map(data) do - data = infer_type(data) + data = normalize(data) if Map.has_key?(data, "type") do changeset(struct, data) @@ -109,6 +127,14 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.TagValidator do end end + def normalize(data) when is_map(data), do: infer_type(data) + + defp infer_type(%{"type" => types} = data) when is_list(types) do + type = Enum.find(types, &(&1 in @tag_types)) || Enum.find(types, &is_binary/1) + + if type, do: Map.put(data, "type", type), else: Map.delete(data, "type") + end + defp infer_type(%{"type" => _} = data), do: data defp infer_type(%{"name" => "#" <> _} = data), do: Map.put(data, "type", "Hashtag") defp infer_type(%{"name" => "@" <> _} = data), do: Map.put(data, "type", "Mention") diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 61c585daa..33ab62ca3 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -19,6 +19,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do alias Pleroma.Web.ActivityPub.Builder alias Pleroma.Web.ActivityPub.ObjectValidator alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes + alias Pleroma.Web.ActivityPub.ObjectValidators.TagValidator alias Pleroma.Web.ActivityPub.Pipeline alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.ActivityPub.Visibility @@ -28,6 +29,9 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do require Pleroma.Constants + @supported_types Pleroma.Constants.activity_types() ++ + Pleroma.Constants.object_types() ++ Pleroma.Constants.actor_types() + @doc """ Modifies an incoming AP object (mastodon format) to our internal format. """ @@ -301,58 +305,66 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def fix_attachments(%{"attachment" => attachment} = object) when is_list(attachment) do attachments = - Enum.map(attachment, fn data -> - url = - cond do - is_list(data["url"]) -> List.first(data["url"]) - is_map(data["url"]) -> data["url"] - true -> nil - end + Enum.map(attachment, fn + data when is_map(data) -> + url = + cond do + is_list(data["url"]) -> List.first(data["url"]) + is_map(data["url"]) -> data["url"] + true -> nil + end - media_type = - cond do - is_map(url) && is_bitstring(url["mediaType"]) && - url["mediaType"] =~ Pleroma.Constants.mime_regex() -> - url["mediaType"] + url_data = if is_map(url), do: url, else: %{} - is_bitstring(data["mediaType"]) && data["mediaType"] =~ Pleroma.Constants.mime_regex() -> - data["mediaType"] + media_type = + cond do + is_bitstring(url_data["mediaType"]) && + url_data["mediaType"] =~ Pleroma.Constants.mime_regex() -> + url_data["mediaType"] - is_bitstring(data["mimeType"]) && data["mimeType"] =~ Pleroma.Constants.mime_regex() -> - data["mimeType"] + is_bitstring(data["mediaType"]) && + data["mediaType"] =~ Pleroma.Constants.mime_regex() -> + data["mediaType"] - true -> - nil - end + is_bitstring(data["mimeType"]) && data["mimeType"] =~ Pleroma.Constants.mime_regex() -> + data["mimeType"] - href = - cond do - is_map(url) && is_binary(url["href"]) -> url["href"] - is_binary(data["url"]) -> data["url"] - is_binary(data["href"]) -> data["href"] - true -> nil - end + true -> + nil + end + + href = + cond do + is_binary(url) -> url + is_binary(url_data["href"]) -> url_data["href"] + is_binary(data["url"]) -> data["url"] + is_binary(data["href"]) -> data["href"] + true -> nil + end + + if href && valid_http_url?(href) do + attachment_url = + %{ + "href" => href, + "type" => Map.get(url_data, "type", "Link") + } + |> Maps.put_if_present("mediaType", media_type) + |> Maps.put_if_present("width", url_data["width"] || data["width"]) + |> Maps.put_if_present("height", url_data["height"] || data["height"]) - if href && valid_http_url?(href) do - attachment_url = %{ - "href" => href, - "type" => Map.get(url || %{}, "type", "Link") + "url" => [attachment_url], + "type" => data["type"] || "Document" } |> Maps.put_if_present("mediaType", media_type) - |> Maps.put_if_present("width", (url || %{})["width"] || data["width"]) - |> Maps.put_if_present("height", (url || %{})["height"] || data["height"]) + |> Maps.put_if_present("name", data["name"]) + |> Maps.put_if_present("blurhash", data["blurhash"]) + else + nil + end - %{ - "url" => [attachment_url], - "type" => data["type"] || "Document" - } - |> Maps.put_if_present("mediaType", media_type) - |> Maps.put_if_present("name", data["name"]) - |> Maps.put_if_present("blurhash", data["blurhash"]) - else + _ -> nil - end end) |> Enum.filter(& &1) @@ -404,14 +416,23 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do defp emoji_icon_url(_), do: nil + defp valid_emoji_icon_url(icon) do + case emoji_icon_url(icon) do + url when is_binary(url) -> if valid_http_url?(url), do: url + _ -> nil + end + end + def fix_emoji(%{"tag" => tags} = object) when is_list(tags) do emoji = tags - |> Enum.filter(fn data -> is_map(data) and data["type"] == "Emoji" and data["icon"] end) + |> Enum.filter(fn data -> + is_map(data) and data["type"] == "Emoji" and is_binary(data["name"]) and data["icon"] + end) |> Enum.reduce(%{}, fn data, mapping -> name = String.trim(data["name"], ":") - case emoji_icon_url(data["icon"]) do + case valid_emoji_icon_url(data["icon"]) do url when is_binary(url) -> Map.put(mapping, name, url) _ -> mapping end @@ -420,10 +441,11 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do Map.put(object, "emoji", emoji) end - def fix_emoji(%{"tag" => %{"type" => "Emoji"} = tag} = object) do - name = String.trim(tag["name"], ":") + def fix_emoji(%{"tag" => %{"type" => "Emoji", "name" => name} = tag} = object) + when is_binary(name) do + name = String.trim(name, ":") - with url when is_binary(url) <- emoji_icon_url(tag["icon"]) do + with url when is_binary(url) <- valid_emoji_icon_url(tag["icon"]) do Map.put(object, "emoji", %{name => url}) else _ -> object @@ -435,7 +457,9 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def fix_tag(%{"tag" => tag} = object) when is_list(tag) do tags = tag - |> Enum.filter(fn data -> data["type"] == "Hashtag" and data["name"] end) + |> Enum.filter(fn data -> + is_map(data) and data["type"] == "Hashtag" and is_binary(data["name"]) + end) |> Enum.map(fn %{"name" => "#" <> hashtag} -> String.downcase(hashtag) %{"name" => hashtag} -> String.downcase(hashtag) @@ -502,10 +526,32 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do |> handle_incoming_normalized(options) end - defp fix_type_as_string(%{"type" => [type | _]} = data) when is_binary(type), - do: Map.put(data, "type", type) + defp fix_type_as_string(data) do + data + |> normalize_type() + |> normalize_tag_types() + end - defp fix_type_as_string(data), do: data + defp normalize_type(%{"type" => types} = data) when is_list(types) do + type = Enum.find(types, &(&1 in @supported_types)) || Enum.find(types, &is_binary/1) + + if type, do: Map.put(data, "type", type), else: data + end + + defp normalize_type(data), do: data + + defp normalize_tag_types(%{"tag" => tags} = data) when is_list(tags) do + Map.put(data, "tag", Enum.map(tags, &normalize_tag_type/1)) + end + + defp normalize_tag_types(%{"tag" => tag} = data) when is_map(tag) do + Map.put(data, "tag", normalize_tag_type(tag)) + end + + defp normalize_tag_types(data), do: data + + defp normalize_tag_type(tag) when is_map(tag), do: TagValidator.normalize(tag) + defp normalize_tag_type(tag), do: tag # Flag objects are placed ahead of the ID check because Mastodon 2.8 and earlier send them # with nil ID. diff --git a/test/pleroma/object/containment_test.exs b/test/pleroma/object/containment_test.exs index cf906ef47..5560dfce5 100644 --- a/test/pleroma/object/containment_test.exs +++ b/test/pleroma/object/containment_test.exs @@ -27,6 +27,11 @@ defmodule Pleroma.Object.ContainmentTest do }) end + test "rejects an empty actor list without crashing" do + assert :error == + Containment.contain_origin("https://example.com/objects/1", %{"actor" => []}) + end + test "contain_origin_from_id() catches obvious spoofing attempts" do data = %{ "id" => "http://example.com/~alyssa/activities/1234.json" diff --git a/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_attachments_test.exs b/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_attachments_test.exs index a13912b08..9d15974fc 100644 --- a/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_attachments_test.exs +++ b/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_attachments_test.exs @@ -36,8 +36,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.FedidevFunAttachmentsTest do end end - defp ingest_fixture!(rel_path) do - data = rel_path |> File.read!() |> Jason.decode!() + defp ingest_data!(data) do ensure_remote_actor!(data["actor"]) assert {:ok, activity} = Transmogrifier.handle_incoming(data) @@ -46,6 +45,13 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.FedidevFunAttachmentsTest do {activity, object} end + defp ingest_fixture!(rel_path) do + rel_path + |> File.read!() + |> Jason.decode!() + |> ingest_data!() + end + test "drops non-http(s) attachment href" do {_activity, object} = ingest_fixture!("test/fixtures/fedidev.fun/attachment_payto_link.json") assert object.data["attachment"] == [] @@ -66,4 +72,27 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.FedidevFunAttachmentsTest do assert href == "https://fedidev.fun/images/007.png" end + + test "drops malformed attachment list entries" do + data = File.read!("test/fixtures/fedidev.fun/attachment_url_list.json") |> Jason.decode!() + data = update_in(data, ["object", "attachment"], &["malformed" | List.wrap(&1)]) + + {_activity, object} = ingest_data!(data) + + assert length(object.data["attachment"]) == 1 + end + + test "accepts a URI string in an attachment url list" do + data = File.read!("test/fixtures/fedidev.fun/attachment_url_list.json") |> Jason.decode!() + + data = + put_in(data, ["object", "attachment", "url"], [ + "https://fedidev.fun/images/007.png" + ]) + + {_activity, object} = ingest_data!(data) + + assert [%{"url" => [%{"href" => "https://fedidev.fun/images/007.png"}]}] = + object.data["attachment"] + end end diff --git a/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_emoji_test.exs b/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_emoji_test.exs index 56c305352..62e30f5af 100644 --- a/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_emoji_test.exs +++ b/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_emoji_test.exs @@ -36,8 +36,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.FedidevFunEmojiTest do end end - defp ingest_fixture!(rel_path) do - data = rel_path |> File.read!() |> Jason.decode!() + defp ingest_data!(data) do ensure_remote_actor!(data["actor"]) assert {:ok, activity} = Transmogrifier.handle_incoming(data) @@ -46,6 +45,13 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.FedidevFunEmojiTest do {activity, object} end + defp ingest_fixture!(rel_path) do + rel_path + |> File.read!() + |> Jason.decode!() + |> ingest_data!() + end + test "ingests Emoji tag with string icon" do {_activity, object} = ingest_fixture!("test/fixtures/fedidev.fun/emoji_icon_string.json") assert object.data["emoji"]["cow"] == "https://fedidev.fun/static/cow.png" @@ -62,4 +68,36 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.FedidevFunEmojiTest do {_activity, object} = ingest_fixture!("test/fixtures/fedidev.fun/emoji_missing_icon.json") assert object.data["emoji"] == %{} end + + test "ignores Emoji tag missing a string name" do + data = File.read!("test/fixtures/fedidev.fun/emoji_icon_string.json") |> Jason.decode!() + data = put_in(data, ["object", "tag", Access.at(0), "name"], nil) + + {_activity, object} = ingest_data!(data) + + assert object.data["emoji"] == %{} + end + + test "ignores Emoji tag with a non-http icon" do + data = File.read!("test/fixtures/fedidev.fun/emoji_icon_string.json") |> Jason.decode!() + data = put_in(data, ["object", "tag", Access.at(0), "icon"], "file:///tmp/cow.png") + + {_activity, object} = ingest_data!(data) + + assert object.data["emoji"] == %{} + end + + test "normalizes a list-typed Emoji tag before building the emoji map" do + data = File.read!("test/fixtures/fedidev.fun/emoji_icon_string.json") |> Jason.decode!() + + data = + put_in(data, ["object", "tag", Access.at(0), "type"], [ + "https://example.com/ns#CustomTag", + "Emoji" + ]) + + {_activity, object} = ingest_data!(data) + + assert object.data["emoji"]["cow"] == "https://fedidev.fun/static/cow.png" + end end diff --git a/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_object_type_list_test.exs b/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_object_type_list_test.exs index 619e6f477..b169dc181 100644 --- a/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_object_type_list_test.exs +++ b/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_object_type_list_test.exs @@ -50,4 +50,15 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.FedidevFunObjectTypeListTest do {_activity, object} = ingest_fixture!("test/fixtures/fedidev.fun/object_type_list.json") assert object.data["type"] == "Note" end + + test "selects a supported type after an extension type" do + data = File.read!("test/fixtures/fedidev.fun/object_type_list.json") |> Jason.decode!() + data = put_in(data, ["object", "type"], ["https://example.com/ns#CustomType", "Note"]) + ensure_remote_actor!(data["actor"]) + + assert {:ok, activity} = Transmogrifier.handle_incoming(data) + object = Object.normalize(activity.data["object"], fetch: false) + + assert object.data["type"] == "Note" + end end diff --git a/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_tag_inference_test.exs b/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_tag_inference_test.exs index ef7feb8ad..0650dd38c 100644 --- a/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_tag_inference_test.exs +++ b/test/pleroma/web/activity_pub/transmogrifier/fedidev_fun_tag_inference_test.exs @@ -62,4 +62,51 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.FedidevFunTagInferenceTest do _ -> false end) end + + test "normalizes a list-typed tag type" do + data = + File.read!("test/fixtures/fedidev.fun/tag_hashtag_missing_type.json") |> Jason.decode!() + + data = put_in(data, ["object", "tag", Access.at(0), "type"], ["Hashtag"]) + ensure_remote_actor!(data["actor"]) + + assert {:ok, activity} = Transmogrifier.handle_incoming(data) + object = Object.normalize(activity.data["object"], fetch: false) + + assert "test" in Object.hashtags(object) + end + + test "ignores a list-typed Hashtag without a string name" do + data = + File.read!("test/fixtures/fedidev.fun/tag_hashtag_missing_type.json") |> Jason.decode!() + + data = put_in(data, ["object", "tag", Access.at(0), "type"], ["Hashtag"]) + data = put_in(data, ["object", "tag", Access.at(0), "name"], nil) + ensure_remote_actor!(data["actor"]) + + assert {:ok, activity} = Transmogrifier.handle_incoming(data) + object = Object.normalize(activity.data["object"], fetch: false) + + assert Object.hashtags(object) == [] + end + + test "infers a Link before deriving quoteUrl" do + data = + File.read!("test/fixtures/fedidev.fun/tag_hashtag_missing_type.json") |> Jason.decode!() + + data = + put_in(data, ["object", "tag"], [ + %{ + "href" => "https://fedidev.fun/assets/note1.jsonap", + "mediaType" => "application/activity+json" + } + ]) + + ensure_remote_actor!(data["actor"]) + + assert {:ok, activity} = Transmogrifier.handle_incoming(data) + object = Object.normalize(activity.data["object"], fetch: false) + + assert object.data["quoteUrl"] == "https://fedidev.fun/assets/note1.jsonap" + end end diff --git a/test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs b/test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs index 4cc199550..793293abd 100644 --- a/test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs +++ b/test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs @@ -671,13 +671,21 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.NoteHandlingTest do test "returns object with emoji when object contains list tags" do assert Transmogrifier.fix_emoji(%{ "tag" => [ - %{"type" => "Emoji", "name" => ":bib:", "icon" => %{"url" => "/test"}}, + %{ + "type" => "Emoji", + "name" => ":bib:", + "icon" => %{"url" => "https://example.com/test"} + }, %{"type" => "Hashtag"} ] }) == %{ - "emoji" => %{"bib" => "/test"}, + "emoji" => %{"bib" => "https://example.com/test"}, "tag" => [ - %{"icon" => %{"url" => "/test"}, "name" => ":bib:", "type" => "Emoji"}, + %{ + "icon" => %{"url" => "https://example.com/test"}, + "name" => ":bib:", + "type" => "Emoji" + }, %{"type" => "Hashtag"} ] } @@ -685,10 +693,28 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.NoteHandlingTest do test "returns object with emoji when object contains map tag" do assert Transmogrifier.fix_emoji(%{ - "tag" => %{"type" => "Emoji", "name" => ":bib:", "icon" => %{"url" => "/test"}} + "tag" => %{ + "type" => "Emoji", + "name" => ":bib:", + "icon" => %{"url" => "https://example.com/test"} + } }) == %{ - "emoji" => %{"bib" => "/test"}, - "tag" => %{"icon" => %{"url" => "/test"}, "name" => ":bib:", "type" => "Emoji"} + "emoji" => %{"bib" => "https://example.com/test"}, + "tag" => %{ + "icon" => %{"url" => "https://example.com/test"}, + "name" => ":bib:", + "type" => "Emoji" + } + } + end + end + + describe "fix_tag/1" do + test "ignores malformed Hashtag names and tag entries" do + tag = %{"type" => "Hashtag", "name" => 123} + + assert Transmogrifier.fix_tag(%{"tag" => [tag, "malformed"]}) == %{ + "tag" => [tag, "malformed"] } end end diff --git a/test/pleroma/web/activity_pub/transmogrifier/user_update_handling_test.exs b/test/pleroma/web/activity_pub/transmogrifier/user_update_handling_test.exs index 851c60850..cf995e143 100644 --- a/test/pleroma/web/activity_pub/transmogrifier/user_update_handling_test.exs +++ b/test/pleroma/web/activity_pub/transmogrifier/user_update_handling_test.exs @@ -50,6 +50,25 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.UserUpdateHandlingTest do assert user.bio == "

Some bio

" end + test "it selects an actor type after an extension type" do + user = insert(:user, local: false) + update_data = File.read!("test/fixtures/mastodon-update.json") |> Jason.decode!() + + object = + update_data["object"] + |> Map.put("actor", user.ap_id) + |> Map.put("id", user.ap_id) + |> Map.put("type", ["https://example.com/ns#CustomActor", "Person"]) + + update_data = + update_data + |> Map.put("actor", user.ap_id) + |> Map.put("object", object) + + assert {:ok, %Activity{local: false}} = Transmogrifier.handle_incoming(update_data) + assert User.get_cached_by_ap_id(user.ap_id).name == "gargle" + end + test "it works with alsoKnownAs" do %{ap_id: actor} = insert(:user, local: false) From 6e73d043ef87c4d0383b0e8d509a4a256f44f3eb Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sun, 26 Jul 2026 12:40:38 +0400 Subject: [PATCH 65/90] test(config): exclude static fixture from test loader --- .../config/{static_search.secret.exs => static_search.config} | 0 test/mix/tasks/pleroma/config_test.exs | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename test/fixtures/config/{static_search.secret.exs => static_search.config} (100%) diff --git a/test/fixtures/config/static_search.secret.exs b/test/fixtures/config/static_search.config similarity index 100% rename from test/fixtures/config/static_search.secret.exs rename to test/fixtures/config/static_search.config diff --git a/test/mix/tasks/pleroma/config_test.exs b/test/mix/tasks/pleroma/config_test.exs index 620bf8662..786d4aac2 100644 --- a/test/mix/tasks/pleroma/config_test.exs +++ b/test/mix/tasks/pleroma/config_test.exs @@ -89,7 +89,7 @@ defmodule Mix.Tasks.Pleroma.ConfigTest do end test "static search settings remain in the config file" do - MixTask.migrate_to_db("test/fixtures/config/static_search.secret.exs") + MixTask.migrate_to_db("test/fixtures/config/static_search.config") refute ConfigDB.get_by_group_and_key(:pleroma, Pleroma.Search) refute ConfigDB.get_by_group_and_key(:pleroma, Pleroma.Search.ParadeDB) From 458a15bf9f7d6eaaf44ea6cbe6c9f2a58b8502d1 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sun, 26 Jul 2026 13:24:03 +0400 Subject: [PATCH 66/90] fix(mastodon-api): harden admin moderation actions --- lib/pleroma/web/activity_pub/activity_pub.ex | 9 ++ .../mastodon_admin/account_operation.ex | 10 +- .../mastodon_admin/report_operation.ex | 43 ++++++- lib/pleroma/web/common_api.ex | 4 +- .../admin/controllers/account_controller.ex | 53 +++++++-- .../admin/controllers/report_controller.ex | 33 +++++- .../mastodon_api/admin/views/report_view.ex | 29 +++-- lib/pleroma/web/router.ex | 15 ++- .../controllers/account_controller_test.exs | 68 ++++++++++- .../controllers/report_controller_test.exs | 111 ++++++++++++++++++ 10 files changed, 344 insertions(+), 31 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 10d3fe063..cdad6a49b 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -1016,6 +1016,14 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp restrict_state(query, _), do: query + defp restrict_report_target(query, %{report_target_id: actor_id}) do + from(activity in query, + where: fragment("?->'object'->>0 = ?", activity.data, ^actor_id) + ) + end + + defp restrict_report_target(query, _), do: query + defp restrict_assigned_account(query, %{assigned_account: assigned_account}) do from(activity in query, where: fragment("?->>'assigned_account' = ?", activity.data, ^assigned_account) @@ -1514,6 +1522,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do |> restrict_actor(opts) |> restrict_type(opts) |> restrict_state(opts) + |> restrict_report_target(opts) |> restrict_assigned_account(opts) |> restrict_favorited_by(opts) |> restrict_blocked(restrict_blocked_opts) diff --git a/lib/pleroma/web/api_spec/operations/mastodon_admin/account_operation.ex b/lib/pleroma/web/api_spec/operations/mastodon_admin/account_operation.ex index ad79d43d7..3b39cbe2a 100644 --- a/lib/pleroma/web/api_spec/operations/mastodon_admin/account_operation.ex +++ b/lib/pleroma/web/api_spec/operations/mastodon_admin/account_operation.ex @@ -216,20 +216,24 @@ defmodule Pleroma.Web.ApiSpec.MastodonAdmin.AccountOperation do properties: %{ type: %Schema{ type: :string, - enum: ["none", "disable", "sensitive", "silence", "suspend"] + enum: ["none", "disable"] }, report_id: %Schema{ type: :string, nullable: true, description: "ID of an associated report that caused this action to be taken" } - } + }, + required: [:type] }, required: true ), responses: %{ 204 => no_content_response(), - 401 => Operation.response("Error", "application/json", ApiError) + 400 => Operation.response("Error", "application/json", ApiError), + 401 => Operation.response("Error", "application/json", ApiError), + 403 => Operation.response("Error", "application/json", ApiError), + 404 => Operation.response("Error", "application/json", ApiError) } } end diff --git a/lib/pleroma/web/api_spec/operations/mastodon_admin/report_operation.ex b/lib/pleroma/web/api_spec/operations/mastodon_admin/report_operation.ex index f59db9966..7e0bcc24b 100644 --- a/lib/pleroma/web/api_spec/operations/mastodon_admin/report_operation.ex +++ b/lib/pleroma/web/api_spec/operations/mastodon_admin/report_operation.ex @@ -34,7 +34,7 @@ defmodule Pleroma.Web.ApiSpec.MastodonAdmin.ReportOperation do :target_account_id, :query, :string, - "Filter by report target account id (not implemented)" + "Filter by report target account id" ) ] ++ pagination_params(), @@ -81,7 +81,8 @@ defmodule Pleroma.Web.ApiSpec.MastodonAdmin.ReportOperation do responses: %{ 200 => Operation.response("Report", "application/json", report()), 400 => Operation.response("Error", "application/json", ApiError), - 401 => Operation.response("Error", "application/json", ApiError) + 401 => Operation.response("Error", "application/json", ApiError), + 404 => Operation.response("Error", "application/json", ApiError) } } end @@ -99,11 +100,43 @@ defmodule Pleroma.Web.ApiSpec.MastodonAdmin.ReportOperation do responses: %{ 200 => Operation.response("Report", "application/json", report()), 400 => Operation.response("Error", "application/json", ApiError), - 401 => Operation.response("Error", "application/json", ApiError) + 401 => Operation.response("Error", "application/json", ApiError), + 404 => Operation.response("Error", "application/json", ApiError) } } end + def assign_to_self_operation do + %Operation{ + tags: ["Report management (Mastodon API)"], + summary: "Assign report to self", + operationId: "MastodonAdmin.ReportController.assign_to_self", + security: [%{"oAuth" => ["admin:write:reports"]}], + parameters: [Operation.parameter(:id, :path, :string, "ID of the report")], + responses: report_update_responses() + } + end + + def unassign_operation do + %Operation{ + tags: ["Report management (Mastodon API)"], + summary: "Unassign report", + operationId: "MastodonAdmin.ReportController.unassign", + security: [%{"oAuth" => ["admin:write:reports"]}], + parameters: [Operation.parameter(:id, :path, :string, "ID of the report")], + responses: report_update_responses() + } + end + + defp report_update_responses do + %{ + 200 => Operation.response("Report", "application/json", report()), + 400 => Operation.response("Error", "application/json", ApiError), + 401 => Operation.response("Error", "application/json", ApiError), + 404 => Operation.response("Error", "application/json", ApiError) + } + end + defp report do %Schema{ title: "Admin::Report", @@ -112,7 +145,7 @@ defmodule Pleroma.Web.ApiSpec.MastodonAdmin.ReportOperation do id: FlakeID, action_taken: %Schema{type: :boolean}, action_taken_at: %Schema{type: :string, format: "date-time", nullable: true}, - category: %Schema{type: :string, enum: ["spam", "violation", "other"]}, + category: %Schema{type: :string, enum: ["spam", "legal", "violation", "other"]}, comment: %Schema{type: :string, nullable: true}, forwarded: %Schema{type: :boolean, nullable: true}, created_at: %Schema{type: :string, format: "date-time"}, @@ -136,7 +169,7 @@ defmodule Pleroma.Web.ApiSpec.MastodonAdmin.ReportOperation do items: %Schema{ type: :object, properties: %{ - id: %Schema{type: :integer}, + id: %Schema{type: :string}, text: %Schema{type: :string} } } diff --git a/lib/pleroma/web/common_api.ex b/lib/pleroma/web/common_api.ex index cb9d521b3..220a8a228 100644 --- a/lib/pleroma/web/common_api.ex +++ b/lib/pleroma/web/common_api.ex @@ -700,7 +700,7 @@ defmodule Pleroma.Web.CommonAPI do end def update_report_state(activity_id, state) do - with %Activity{} = activity <- Activity.get_by_id(activity_id, filter: []) do + with %Activity{} = activity <- Activity.get_report(activity_id) do Utils.update_report_state(activity, state) else nil -> {:error, :not_found} @@ -715,7 +715,7 @@ defmodule Pleroma.Web.CommonAPI do end def assign_report_to_account(activity_id, user) do - with %Activity{} = activity <- Activity.get_by_id(activity_id) do + with %Activity{} = activity <- Activity.get_report(activity_id) do Utils.assign_report_to_account(activity, user) else nil -> {:error, :not_found} diff --git a/lib/pleroma/web/mastodon_api/admin/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/admin/controllers/account_controller.ex index e96566774..b4e038f0e 100644 --- a/lib/pleroma/web/mastodon_api/admin/controllers/account_controller.ex +++ b/lib/pleroma/web/mastodon_api/admin/controllers/account_controller.ex @@ -33,6 +33,8 @@ defmodule Pleroma.Web.MastodonAPI.Admin.AccountController do when action in [:index, :index2, :show] ) + action_fallback(Pleroma.Web.MastodonAPI.FallbackController) + plug( OAuthScopesPlug, %{scopes: ["admin:write:accounts"]} @@ -92,11 +94,18 @@ defmodule Pleroma.Web.MastodonAPI.Admin.AccountController do conn, _params ) do - {:ok, _user} = handle_account_action(user, admin, type) + with :ok <- validate_account_action(user, type), + {:ok, report} <- associated_report(user, body_params), + :ok <- resolve_report(admin, report), + {:ok, _user} <- handle_account_action(user, admin, type) do + json_response(conn, :no_content, "") + else + {:error, :not_found} -> + {:error, :not_found} - resolve_report(admin, body_params) - - json_response(conn, :no_content, "") + {:error, error} -> + json_response(conn, :bad_request, %{error: error}) + end end def delete(%{assigns: %{user: admin, account: user}} = conn, _params) do @@ -161,10 +170,21 @@ defmodule Pleroma.Web.MastodonAPI.Admin.AccountController do User.set_activation(user, false) end - defp handle_account_action(user, _admin, _type) do + defp handle_account_action(%User{}, _admin, "disable") do + {:error, "Only local accounts can be disabled"} + end + + defp handle_account_action(user, _admin, "none") do {:ok, user} end + defp validate_account_action(%User{local: true}, "disable"), do: :ok + + defp validate_account_action(%User{}, "disable"), + do: {:error, "Only local accounts can be disabled"} + + defp validate_account_action(%User{}, "none"), do: :ok + defp build_criteria(params) do %{} |> maybe_filter_local(params) @@ -251,7 +271,23 @@ defmodule Pleroma.Web.MastodonAPI.Admin.AccountController do defp unquote(:"maybe_filter_#{filter_param}")(criteria, _params), do: criteria end - defp resolve_report(admin, %{report_id: id}) do + defp associated_report(user, %{report_id: id}) do + case Activity.get_report(id) do + %Activity{data: %{"object" => [target_ap_id | _]}} = report + when target_ap_id == user.ap_id -> + {:ok, report} + + %Activity{} -> + {:error, "Report does not target this account"} + + nil -> + {:error, :not_found} + end + end + + defp associated_report(_user, _params), do: {:ok, nil} + + defp resolve_report(admin, %Activity{id: id}) do with {:ok, activity} <- CommonAPI.update_report_state(id, "resolved"), report <- Activity.get_by_id_with_user_actor(activity.id) do ModerationLog.insert_log(%{ @@ -260,9 +296,10 @@ defmodule Pleroma.Web.MastodonAPI.Admin.AccountController do subject: activity, subject_actor: report.user_actor }) + + :ok end end - defp resolve_report(_admin, _params) do - end + defp resolve_report(_admin, nil), do: :ok end diff --git a/lib/pleroma/web/mastodon_api/admin/controllers/report_controller.ex b/lib/pleroma/web/mastodon_api/admin/controllers/report_controller.ex index 5dbf19aba..062b90fc6 100644 --- a/lib/pleroma/web/mastodon_api/admin/controllers/report_controller.ex +++ b/lib/pleroma/web/mastodon_api/admin/controllers/report_controller.ex @@ -26,6 +26,13 @@ defmodule Pleroma.Web.MastodonAPI.Admin.ReportController do plug(OAuthScopesPlug, %{scopes: ["admin:write:reports"]} when action in [:resolve, :reopen]) + plug( + OAuthScopesPlug, + %{scopes: ["admin:write:reports"]} when action in [:assign_to_self, :unassign] + ) + + action_fallback(Pleroma.Web.MastodonAPI.FallbackController) + defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.MastodonAdmin.ReportOperation def index(conn, params) do @@ -37,6 +44,7 @@ defmodule Pleroma.Web.MastodonAPI.Admin.ReportController do |> Map.put(:total, true) |> restrict_state(params) |> restrict_actor(params) + |> restrict_target(params) reports = ActivityPub.fetch_activities_query([], opts) @@ -67,6 +75,9 @@ defmodule Pleroma.Web.MastodonAPI.Admin.ReportController do render(conn, "show.json", Report.extract_report_info(report)) else + {:error, :not_found} -> + {:error, :not_found} + {:error, error} -> json_response(conn, :bad_request, %{error: error}) end @@ -84,6 +95,9 @@ defmodule Pleroma.Web.MastodonAPI.Admin.ReportController do render(conn, "show.json", Report.extract_report_info(report)) else + {:error, :not_found} -> + {:error, :not_found} + {:error, error} -> json_response(conn, :bad_request, %{error: error}) end @@ -102,6 +116,9 @@ defmodule Pleroma.Web.MastodonAPI.Admin.ReportController do render(conn, "show.json", Report.extract_report_info(report)) else + {:error, :not_found} -> + {:error, :not_found} + {:error, error} -> json_response(conn, :bad_request, %{error: error}) end @@ -119,14 +136,17 @@ defmodule Pleroma.Web.MastodonAPI.Admin.ReportController do render(conn, "show.json", Report.extract_report_info(report)) else + {:error, :not_found} -> + {:error, :not_found} + {:error, error} -> json_response(conn, :bad_request, %{error: error}) end end - defp restrict_state(opts, %{resolved: true}), do: Map.put(opts, :state, "resolved") + defp restrict_state(opts, %{resolved: true, unresolved: true}), do: opts - defp restrict_state(opts, %{resolved: _}), do: Map.put(opts, :state, "resolved") + defp restrict_state(opts, %{resolved: true}), do: Map.put(opts, :state, "resolved") defp restrict_state(opts, _params), do: Map.put(opts, :state, "open") @@ -139,4 +159,13 @@ defmodule Pleroma.Web.MastodonAPI.Admin.ReportController do end defp restrict_actor(opts, _params), do: opts + + defp restrict_target(opts, %{target_account_id: actor}) do + case User.get_by_id(actor) do + %User{ap_id: ap_id} -> Map.put(opts, :report_target_id, ap_id) + _ -> Map.put(opts, :report_target_id, actor) + end + end + + defp restrict_target(opts, _params), do: opts end diff --git a/lib/pleroma/web/mastodon_api/admin/views/report_view.ex b/lib/pleroma/web/mastodon_api/admin/views/report_view.ex index 664f58b03..088824d03 100644 --- a/lib/pleroma/web/mastodon_api/admin/views/report_view.ex +++ b/lib/pleroma/web/mastodon_api/admin/views/report_view.ex @@ -6,6 +6,7 @@ defmodule Pleroma.Web.MastodonAPI.Admin.ReportView do use Pleroma.Web, :view alias Pleroma.HTML + alias Pleroma.Rule alias Pleroma.Web.AdminAPI.Report alias Pleroma.Web.CommonAPI.Utils alias Pleroma.Web.MastodonAPI.Admin.AccountView @@ -21,9 +22,12 @@ defmodule Pleroma.Web.MastodonAPI.Admin.ReportView do report: report, user: account, account: target_account, - statuses: statuses + statuses: statuses, + assigned_account: assigned_account }) do created_at = Utils.to_masto_date(report.data["published"]) + updated_at = Utils.to_masto_date(report.updated_at) + action_taken = report.data["state"] != "open" content = unless is_nil(report.data["content"]) do @@ -34,23 +38,34 @@ defmodule Pleroma.Web.MastodonAPI.Admin.ReportView do %{ id: report.id, - action_taken: report.data["state"] != "open", - action_taken_at: nil, + action_taken: action_taken, + action_taken_at: if(action_taken, do: updated_at), category: "other", comment: content, - forwarded: nil, + forwarded: false, created_at: created_at, - updated_at: created_at, + updated_at: updated_at, account: AccountView.render("show.json", %{user: account}), target_account: AccountView.render("show.json", %{user: target_account}), - assigned_account: nil, + assigned_account: render_account(assigned_account), action_taken_by_account: nil, statuses: StatusView.render("index.json", %{ activities: statuses, as: :activity }), - rules: [] + rules: rules(report.data["rules"]) } end + + defp render_account(nil), do: nil + defp render_account(account), do: AccountView.render("show.json", %{user: account}) + + defp rules(nil), do: [] + + defp rules(rule_ids) do + rule_ids + |> Rule.get() + |> Enum.map(&%{id: to_string(&1.id), text: &1.text}) + end end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index aa0ec4817..27229ed41 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -137,6 +137,12 @@ defmodule Pleroma.Web.Router do plug(Pleroma.Web.Plugs.EnsurePrivilegedPlug, :users_manage_activation_state) end + pipeline :require_privileged_role_account_actions do + plug(:admin_api) + plug(Pleroma.Web.Plugs.EnsurePrivilegedPlug, :users_manage_activation_state) + plug(Pleroma.Web.Plugs.EnsurePrivilegedPlug, :reports_manage_reports) + end + pipeline :require_privileged_role_users_manage_invites do plug(:admin_api) plug(Pleroma.Web.Plugs.EnsurePrivilegedPlug, :users_manage_invites) @@ -443,10 +449,15 @@ defmodule Pleroma.Web.Router do scope "/api/v1/admin", Pleroma.Web.MastodonAPI.Admin do pipe_through([:require_privileged_role_users_manage_activation_state]) - post("/accounts/:id/action", AccountController, :account_action) post("/accounts/:id/enable", AccountController, :enable) end + scope "/api/v1/admin", Pleroma.Web.MastodonAPI.Admin do + pipe_through([:require_privileged_role_account_actions]) + + post("/accounts/:id/action", AccountController, :account_action) + end + # Mastodon AdminAPI: admins and mods (staff) can perform these actions (if privileged by role) scope "/api/v1/admin", Pleroma.Web.MastodonAPI.Admin do pipe_through([:require_privileged_role_users_manage_invites]) @@ -462,6 +473,8 @@ defmodule Pleroma.Web.Router do get("/reports/:id", ReportController, :show) post("/reports/:id/resolve", ReportController, :resolve) post("/reports/:id/reopen", ReportController, :reopen) + post("/reports/:id/assign_to_self", ReportController, :assign_to_self) + post("/reports/:id/unassign", ReportController, :unassign) end # Mastodon AdminAPI: admins and mods (staff) can perform these actions (if privileged by role) diff --git a/test/pleroma/web/mastodon_api/admin/controllers/account_controller_test.exs b/test/pleroma/web/mastodon_api/admin/controllers/account_controller_test.exs index 113b242dd..38f1189cd 100644 --- a/test/pleroma/web/mastodon_api/admin/controllers/account_controller_test.exs +++ b/test/pleroma/web/mastodon_api/admin/controllers/account_controller_test.exs @@ -183,6 +183,20 @@ defmodule Pleroma.Web.MastodonAPI.Admin.AccountControllerTest do assert %{is_active: false} = user end + test "does not disable a remote account", %{conn: conn} do + %{id: id} = user = insert(:user, local: false) + + assert %{"error" => "Only local accounts can be disabled"} = + conn + |> put_req_header("content-type", "application/json") + |> post("/api/v1/admin/accounts/#{id}/action", %{ + "type" => "disable" + }) + |> json_response_and_validate_schema(400) + + assert %{is_active: true} = Repo.reload!(user) + end + test "perform action with assigned report", %{conn: conn} do [reporter, target_user] = insert_pair(:user) @@ -191,11 +205,9 @@ defmodule Pleroma.Web.MastodonAPI.Admin.AccountControllerTest do account_id: target_user.id }) - %{id: id} = insert(:user) - conn |> put_req_header("content-type", "application/json") - |> post("/api/v1/admin/accounts/#{id}/action", %{ + |> post("/api/v1/admin/accounts/#{target_user.id}/action", %{ "type" => "none", "report_id" => report_id }) @@ -205,6 +217,56 @@ defmodule Pleroma.Web.MastodonAPI.Admin.AccountControllerTest do assert %{data: %{"state" => "resolved"}} = report end + + test "requires the report management privilege", %{conn: conn} do + clear_config([:instance, :admin_privileges], [:users_manage_activation_state]) + + %{id: id} = insert(:user) + + assert %{"error" => _} = + conn + |> put_req_header("content-type", "application/json") + |> post("/api/v1/admin/accounts/#{id}/action", %{"type" => "none"}) + |> json_response(403) + end + + test "does not resolve a report for another account", %{conn: conn} do + [reporter, target_user, other_user] = insert_list(3, :user) + + {:ok, report} = + CommonAPI.report(reporter, %{ + account_id: target_user.id + }) + + assert %{"error" => _} = + conn + |> put_req_header("content-type", "application/json") + |> post("/api/v1/admin/accounts/#{other_user.id}/action", %{ + "type" => "none", + "report_id" => report.id + }) + |> json_response_and_validate_schema(400) + + assert %{data: %{"state" => "open"}} = Repo.reload!(report) + end + + test "requires an action type", %{conn: conn} do + %{id: id} = insert(:user) + + conn + |> put_req_header("content-type", "application/json") + |> post("/api/v1/admin/accounts/#{id}/action", %{}) + |> json_response_and_validate_schema(400) + end + + test "rejects unsupported action types", %{conn: conn} do + %{id: id} = insert(:user) + + conn + |> put_req_header("content-type", "application/json") + |> post("/api/v1/admin/accounts/#{id}/action", %{"type" => "sensitive"}) + |> json_response_and_validate_schema(400) + end end describe "POST /api/v1/admin/accounts/:id/enable" do diff --git a/test/pleroma/web/mastodon_api/admin/controllers/report_controller_test.exs b/test/pleroma/web/mastodon_api/admin/controllers/report_controller_test.exs index 702dcf1db..2ea5e9d92 100644 --- a/test/pleroma/web/mastodon_api/admin/controllers/report_controller_test.exs +++ b/test/pleroma/web/mastodon_api/admin/controllers/report_controller_test.exs @@ -8,6 +8,9 @@ defmodule Pleroma.Web.MastodonAPI.Admin.ReportControllerTest do import Pleroma.Factory + alias Pleroma.Activity + alias Pleroma.Repo + alias Pleroma.Rule alias Pleroma.Web.CommonAPI setup_all do @@ -58,6 +61,11 @@ defmodule Pleroma.Web.MastodonAPI.Admin.ReportControllerTest do |> get("/api/v1/admin/reports") |> json_response_and_validate_schema(200) + assert [%{"id" => ^report_id1}] = + conn + |> get("/api/v1/admin/reports?resolved=false") + |> json_response_and_validate_schema(200) + assert [%{"id" => ^report_id2}, %{"id" => ^report_id1}] = conn |> get("/api/v1/admin/reports?resolved=true&unresolved=true") @@ -68,6 +76,25 @@ defmodule Pleroma.Web.MastodonAPI.Admin.ReportControllerTest do |> get("/api/v1/admin/reports?resolved=true") |> json_response_and_validate_schema(200) end + + test "get reports by target account", %{conn: conn} do + [reporter, target_user, other_target] = insert_list(3, :user) + + {:ok, %{id: report_id}} = + CommonAPI.report(reporter, %{ + account_id: target_user.id + }) + + {:ok, _report} = + CommonAPI.report(reporter, %{ + account_id: other_target.id + }) + + assert [%{"id" => ^report_id}] = + conn + |> get("/api/v1/admin/reports?target_account_id=#{target_user.id}") + |> json_response_and_validate_schema(200) + end end describe "GET /api/v1/admin/reports/:id" do @@ -86,6 +113,34 @@ defmodule Pleroma.Web.MastodonAPI.Admin.ReportControllerTest do |> get("/api/v1/admin/reports/#{report_id}") |> json_response_and_validate_schema(200) end + + test "returns report rules", %{conn: conn} do + [reporter, target_user] = insert_pair(:user) + rule = Rule.create(%{text: "Be excellent to each other"}) + + {:ok, %{id: report_id}} = + CommonAPI.report(reporter, %{ + account_id: target_user.id, + rule_ids: [rule.id] + }) + + assert %{ + "rules" => [ + %{"id" => rule_id, "text" => "Be excellent to each other"} + ] + } = + conn + |> get("/api/v1/admin/reports/#{report_id}") + |> json_response_and_validate_schema(200) + + assert rule_id == to_string(rule.id) + end + + test "returns 404 for a missing report", %{conn: conn} do + conn + |> get("/api/v1/admin/reports/0") + |> json_response_and_validate_schema(404) + end end describe "POST /api/v1/admin/reports/:id/resolve" do @@ -104,6 +159,16 @@ defmodule Pleroma.Web.MastodonAPI.Admin.ReportControllerTest do |> post("/api/v1/admin/reports/#{report_id}/resolve") |> json_response_and_validate_schema(200) end + + test "does not mutate a non-report activity", %{conn: conn} do + activity = insert(:note_activity) + + conn + |> post("/api/v1/admin/reports/#{activity.id}/resolve") + |> json_response_and_validate_schema(404) + + refute Repo.reload!(activity).data["state"] + end end describe "POST /api/v1/admin/reports/:id/reopen" do @@ -124,5 +189,51 @@ defmodule Pleroma.Web.MastodonAPI.Admin.ReportControllerTest do |> post("/api/v1/admin/reports/#{report_id}/reopen") |> json_response_and_validate_schema(200) end + + test "does not mutate a non-report activity", %{conn: conn} do + activity = insert(:note_activity) + + conn + |> post("/api/v1/admin/reports/#{activity.id}/reopen") + |> json_response_and_validate_schema(404) + + refute Repo.reload!(activity).data["state"] + end + end + + describe "POST /api/v1/admin/reports/:id/assign_to_self" do + test "assign and unassign a report", %{admin: admin, conn: conn} do + [reporter, target_user] = insert_pair(:user) + + {:ok, report} = + CommonAPI.report(reporter, %{ + account_id: target_user.id + }) + + assert %{"assigned_account" => %{"id" => admin_id}} = + conn + |> post("/api/v1/admin/reports/#{report.id}/assign_to_self") + |> json_response_and_validate_schema(200) + + assert admin_id == admin.id + assert Repo.reload!(report).data["assigned_account"] == admin.id + + assert %{"assigned_account" => nil} = + conn + |> post("/api/v1/admin/reports/#{report.id}/unassign") + |> json_response_and_validate_schema(200) + + refute Repo.reload!(report).data["assigned_account"] + end + + test "does not assign a non-report activity", %{conn: conn} do + %Activity{id: id} = activity = insert(:note_activity) + + conn + |> post("/api/v1/admin/reports/#{id}/assign_to_self") + |> json_response_and_validate_schema(404) + + refute Repo.reload!(activity).data["assigned_account"] + end end end From 55c20c0fe8381529315740f55ed101e31ece263e Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sun, 26 Jul 2026 13:54:28 +0400 Subject: [PATCH 67/90] test(mastodon-api): avoid relationship expiry races --- .../web/mastodon_api/views/account_view_test.exs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/test/pleroma/web/mastodon_api/views/account_view_test.exs b/test/pleroma/web/mastodon_api/views/account_view_test.exs index f34a801c1..552f95994 100644 --- a/test/pleroma/web/mastodon_api/views/account_view_test.exs +++ b/test/pleroma/web/mastodon_api/views/account_view_test.exs @@ -545,11 +545,9 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do test "represent a relationship for the blocking and blocked user with expiry" do user = insert(:user) other_user = insert(:user) - date = DateTime.utc_now() |> DateTime.add(24 * 60 * 60) |> DateTime.truncate(:second) - {:ok, user, other_user} = User.follow(user, other_user) {:ok, _subscription} = User.subscribe(user, other_user) - {:ok, _user_relationship} = User.block(user, other_user, %{duration: 24 * 60 * 60}) + {:ok, user_relationship} = User.block(user, other_user, %{duration: 24 * 60 * 60}) {:ok, _user_relationship} = User.block(other_user, user) expected = @@ -558,7 +556,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do %{ following: false, blocking: true, - block_expires_at: date, + block_expires_at: user_relationship.expires_at, blocked_by: true, id: to_string(other_user.id) } @@ -570,17 +568,19 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do test "represent a relationship for the muting user with expiry" do user = insert(:user) other_user = insert(:user) - date = DateTime.utc_now() |> DateTime.add(24 * 60 * 60) |> DateTime.truncate(:second) - {:ok, _user_relationship} = + {:ok, user_relationships} = User.mute(user, other_user, %{notifications: true, duration: 24 * 60 * 60}) + user_relationship = + Enum.find(user_relationships, &(&1.relationship_type == :mute)) + expected = Map.merge( @blank_response, %{ muting: true, - mute_expires_at: date, + mute_expires_at: user_relationship.expires_at, muting_notifications: true, id: to_string(other_user.id) } From c591ca518c39eb71c41fb7b956d542bb3c4b793f Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sun, 26 Jul 2026 13:55:57 +0400 Subject: [PATCH 68/90] fix(mastodon-api): finish preview card compatibility --- .../web/api_spec/schemas/chat_message.ex | 33 +----- .../web/api_spec/schemas/preview_card.ex | 109 ++++++++++++++++++ lib/pleroma/web/api_spec/schemas/status.ex | 76 +----------- .../web/mastodon_api/views/status_view.ex | 36 ++++-- .../controllers/status_controller_test.exs | 13 +++ .../mastodon_api/views/status_view_test.exs | 22 +++- 6 files changed, 174 insertions(+), 115 deletions(-) create mode 100644 lib/pleroma/web/api_spec/schemas/preview_card.ex diff --git a/lib/pleroma/web/api_spec/schemas/chat_message.ex b/lib/pleroma/web/api_spec/schemas/chat_message.ex index 57f7890e5..4c18063be 100644 --- a/lib/pleroma/web/api_spec/schemas/chat_message.ex +++ b/lib/pleroma/web/api_spec/schemas/chat_message.ex @@ -5,6 +5,7 @@ defmodule Pleroma.Web.ApiSpec.Schemas.ChatMessage do alias OpenApiSpex.Schema alias Pleroma.Web.ApiSpec.Schemas.Emoji + alias Pleroma.Web.ApiSpec.Schemas.PreviewCard require OpenApiSpex @@ -22,36 +23,8 @@ defmodule Pleroma.Web.ApiSpec.Schemas.ChatMessage do emojis: %Schema{type: :array, items: Emoji}, attachment: %Schema{type: :object, nullable: true}, card: %Schema{ - type: :object, - nullable: true, - description: "Preview card for links included within status content", - required: [:url, :title, :description, :type], - properties: %{ - type: %Schema{ - type: :string, - enum: ["link", "photo", "video", "rich"], - description: "The type of the preview card" - }, - provider_name: %Schema{ - type: :string, - nullable: true, - description: "The provider of the original resource" - }, - provider_url: %Schema{ - type: :string, - format: :uri, - description: "A link to the provider of the original resource" - }, - url: %Schema{type: :string, format: :uri, description: "Location of linked resource"}, - image: %Schema{ - type: :string, - nullable: true, - format: :uri, - description: "Preview thumbnail" - }, - title: %Schema{type: :string, description: "Title of linked resource"}, - description: %Schema{type: :string, description: "Description of preview"} - } + allOf: [PreviewCard], + nullable: true }, unread: %Schema{type: :boolean, description: "Whether a message has been marked as read."} }, diff --git a/lib/pleroma/web/api_spec/schemas/preview_card.ex b/lib/pleroma/web/api_spec/schemas/preview_card.ex new file mode 100644 index 000000000..bb4432e18 --- /dev/null +++ b/lib/pleroma/web/api_spec/schemas/preview_card.ex @@ -0,0 +1,109 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ApiSpec.Schemas.PreviewCard do + alias OpenApiSpex.Schema + alias Pleroma.Web.ApiSpec.Schemas.Account + + require OpenApiSpex + + OpenApiSpex.schema(%{ + title: "PreviewCard", + description: "Preview card for a link included within status or chat message content", + type: :object, + required: [ + :url, + :title, + :description, + :language, + :type, + :authors, + :author_name, + :author_url, + :provider_name, + :provider_url, + :html, + :width, + :height, + :image, + :image_description, + :embed_url, + :blurhash, + :published_at + ], + properties: %{ + url: %Schema{type: :string, format: :uri, description: "Location of linked resource"}, + title: %Schema{type: :string, description: "Title of linked resource"}, + description: %Schema{type: :string, description: "Description of preview"}, + language: %Schema{ + type: :string, + nullable: true, + description: "Language of the linked resource" + }, + type: %Schema{ + type: :string, + enum: ["link", "photo", "video", "rich"], + description: "The type of the preview card" + }, + authors: %Schema{ + type: :array, + description: "Fediverse accounts of the authors of the original resource", + items: %Schema{ + type: :object, + required: [:name, :url, :account], + properties: %{ + name: %Schema{type: :string}, + url: %Schema{type: :string}, + account: %Schema{allOf: [Account], nullable: true} + } + } + }, + author_name: %Schema{type: :string, description: "Author of the original resource"}, + author_url: %Schema{ + type: :string, + description: "Link to the author of the original resource" + }, + provider_name: %Schema{ + type: :string, + description: "The provider of the original resource" + }, + provider_url: %Schema{ + type: :string, + description: "A link to the provider of the original resource" + }, + html: %Schema{ + type: :string, + format: :html, + description: "HTML to be used for generating the preview card" + }, + width: %Schema{type: :integer, description: "Width of preview, in pixels"}, + height: %Schema{type: :integer, description: "Height of preview, in pixels"}, + image: %Schema{ + type: :string, + nullable: true, + format: :uri, + description: "Preview thumbnail" + }, + image_description: %Schema{ + type: :string, + description: "Alternate text that describes what is in the thumbnail" + }, + embed_url: %Schema{ + type: :string, + description: "Used for photo embeds, instead of custom `html`" + }, + blurhash: %Schema{ + type: :string, + nullable: true, + description: + "A hash computed by the [BlurHash algorithm](https://github.com/woltapp/blurhash), for generating colorful preview thumbnails when media has not been downloaded yet." + }, + published_at: %Schema{ + type: :string, + nullable: true, + description: "Publication date of the linked resource" + } + } + }) +end diff --git a/lib/pleroma/web/api_spec/schemas/status.ex b/lib/pleroma/web/api_spec/schemas/status.ex index 893988d18..63e9eebf0 100644 --- a/lib/pleroma/web/api_spec/schemas/status.ex +++ b/lib/pleroma/web/api_spec/schemas/status.ex @@ -9,6 +9,7 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Status do alias Pleroma.Web.ApiSpec.Schemas.Emoji alias Pleroma.Web.ApiSpec.Schemas.FlakeID alias Pleroma.Web.ApiSpec.Schemas.Poll + alias Pleroma.Web.ApiSpec.Schemas.PreviewCard alias Pleroma.Web.ApiSpec.Schemas.Tag alias Pleroma.Web.ApiSpec.Schemas.VisibilityScope @@ -31,79 +32,8 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Status do }, bookmarked: %Schema{type: :boolean, description: "Have you bookmarked this status?"}, card: %Schema{ - type: :object, - nullable: true, - description: "Preview card for links included within status content", - # Yeah, it's effectivelly all required in MastoAPI so far - required: [ - :url, - :title, - :description, - :type, - :author_name, - :author_url, - :provider_name, - :provider_url, - :html, - :width, - :height, - :image, - :embed_url, - :blurhash - ], - properties: %{ - url: %Schema{type: :string, format: :uri, description: "Location of linked resource"}, - title: %Schema{type: :string, description: "Title of linked resource"}, - description: %Schema{type: :string, description: "Description of preview"}, - type: %Schema{ - type: :string, - enum: ["link", "photo", "video", "rich"], - description: "The type of the preview card" - }, - author_name: %Schema{type: :string, description: "author of the original resource"}, - author_url: %Schema{ - type: :string, - format: :uri, - description: "link to the author of the original resource" - }, - provider_name: %Schema{ - type: :string, - description: "The provider of the original resource" - }, - provider_url: %Schema{ - type: :string, - format: :uri, - description: "A link to the provider of the original resource" - }, - html: %Schema{ - type: :string, - format: :html, - description: "HTML to be used for generating the preview card" - }, - width: %Schema{type: :integer, description: "Width of preview, in pixels"}, - height: %Schema{type: :integer, description: "Height of preview, in pixels"}, - image: %Schema{ - type: :string, - nullable: true, - format: :uri, - description: "Preview thumbnail" - }, - image_description: %Schema{ - type: :string, - description: "Alternate text that describes what is in the thumbnail" - }, - embed_url: %Schema{ - type: :string, - format: :uri, - description: "Used for photo embeds, instead of custom `html`" - }, - blurhash: %Schema{ - type: :string, - nullable: true, - description: - "A hash computed by the (BlurHash algorithm)[https://github.com/woltapp/blurhash], for generating colorful preview thumbnails when media has not been downloaded yet." - } - } + allOf: [PreviewCard], + nullable: true }, content: %Schema{type: :string, format: :html, description: "HTML-encoded status content"}, text: %Schema{ diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index b78494c28..356e8fce5 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -587,21 +587,25 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do video_url = proxied_url(rich_media["video"], page_url_data) %{ - type: "link", - provider_name: page_url_data.host, - provider_url: page_url_data.scheme <> "://" <> page_url_data.host, + type: card_type(rich_media["type"]), + provider_name: rich_media["provider_name"] || page_url_data.host, + provider_url: + rich_media["provider_url"] || page_url_data.scheme <> "://" <> page_url_data.host, url: page_url, image: image_url, image_description: rich_media["image:alt"] || "", title: rich_media["title"] || "", description: rich_media["description"] || "", - author_name: "", - author_url: "", - html: "", - width: 0, - height: 0, - embed_url: "", + language: nil, + authors: [], + author_name: rich_media["author_name"] || "", + author_url: rich_media["author_url"] || "", + html: rich_media["html"] || "", + width: card_dimension(rich_media["width"]), + height: card_dimension(rich_media["height"]), + embed_url: rich_media["embed_url"] || "", blurhash: nil, + published_at: nil, pleroma: %{ opengraph: rich_media @@ -706,6 +710,20 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do %{content: content, detected_source_language: detected_source_language, provider: provider} end + defp card_type(type) when type in ["link", "photo", "video", "rich"], do: type + defp card_type(_), do: "link" + + defp card_dimension(value) when is_integer(value) and value >= 0, do: value + + defp card_dimension(value) when is_binary(value) do + case Integer.parse(value) do + {dimension, ""} when dimension >= 0 -> dimension + _ -> 0 + end + end + + defp card_dimension(_), do: 0 + def get_reply_to(activity, %{replied_to_activities: replied_to_activities}) do object = Object.normalize(activity, fetch: false) diff --git a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs index c70ec29c4..6ee79689d 100644 --- a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs @@ -2430,6 +2430,9 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do describe "cards" do setup do + clear_config([:rich_media, :enabled], true) + Mox.stub_with(Pleroma.CachexMock, Pleroma.NullCache) + Pleroma.StaticStubbedConfigMock |> stub(:get, fn [:rich_media, :enabled] -> true @@ -2443,6 +2446,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) {:ok, activity} = CommonAPI.post(user, %{status: "https://example.com/ogp"}) + ObanHelpers.perform_all() card_data = %{ "image" => "http://ia.media-imdb.com/images/rock.jpg", @@ -2465,10 +2469,14 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do }, "author_name" => "", "author_url" => "", + "authors" => [], "blurhash" => nil, "embed_url" => "", "height" => 0, "html" => "", + "image_description" => "", + "language" => nil, + "published_at" => nil, "width" => 0 } @@ -2495,6 +2503,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) {:ok, activity} = CommonAPI.post(user, %{status: "https://example.com/ogp-missing-data"}) + ObanHelpers.perform_all() response = conn @@ -2518,10 +2527,14 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do }, "author_name" => "", "author_url" => "", + "authors" => [], "blurhash" => nil, "embed_url" => "", "height" => 0, "html" => "", + "image_description" => "", + "language" => nil, + "published_at" => nil, "width" => 0 } end diff --git a/test/pleroma/web/mastodon_api/views/status_view_test.exs b/test/pleroma/web/mastodon_api/views/status_view_test.exs index faeca050e..73adb196a 100644 --- a/test/pleroma/web/mastodon_api/views/status_view_test.exs +++ b/test/pleroma/web/mastodon_api/views/status_view_test.exs @@ -845,15 +845,31 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do fields = %{ "url" => page_url, - "site_name" => "Example site name", + "type" => "photo", "title" => "Example website", "image" => page_url <> "/example.jpg", - "description" => "Example description" + "description" => "Example description", + "author_name" => "Example author", + "author_url" => page_url <> "/author", + "provider_name" => "Example provider", + "provider_url" => page_url <> "/provider", + "html" => "", + "width" => "1024", + "height" => 768 } {:ok, card} = Card.create(page_url, fields) - assert match?(%{provider_name: "example.com"}, StatusView.render("card.json", card)) + assert %{ + type: "photo", + author_name: "Example author", + author_url: "https://example.com/author", + provider_name: "Example provider", + provider_url: "https://example.com/provider", + html: "", + width: 1024, + height: 768 + } = StatusView.render("card.json", card) end test "a rich media card has all media proxied" do From 2b32e3cf1fa89eda4bd7843455de5c885b732316 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sun, 26 Jul 2026 18:12:49 +0400 Subject: [PATCH 69/90] fix(mastodon-api): restrict unauthenticated account statuses --- ...trict-unauthenticated-account-statuses.fix | 1 + lib/pleroma/web/activity_pub/activity_pub.ex | 38 +++++++ .../controllers/account_controller.ex | 1 + .../controllers/account_controller_test.exs | 106 ++++++++++++++++++ 4 files changed, 146 insertions(+) create mode 100644 changelog.d/restrict-unauthenticated-account-statuses.fix diff --git a/changelog.d/restrict-unauthenticated-account-statuses.fix b/changelog.d/restrict-unauthenticated-account-statuses.fix new file mode 100644 index 000000000..a6ae013fd --- /dev/null +++ b/changelog.d/restrict-unauthenticated-account-statuses.fix @@ -0,0 +1 @@ +Fixed account status and media views exposing activities restricted from unauthenticated users. diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index cdad6a49b..722bf8b60 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -1357,6 +1357,43 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp restrict_unauthenticated(query, _), do: query + defp restrict_unauthenticated_object(query, nil) do + local = Config.restrict_unauthenticated_access?(:activities, :local) + remote = Config.restrict_unauthenticated_access?(:activities, :remote) + local_object_pattern = Pleroma.Web.Endpoint.url() <> "/%" + + cond do + local and remote -> + from([object: object] in query, where: false) + + local -> + from([object: object] in query, + where: not fragment("(?->>'id') LIKE ?", object.data, ^local_object_pattern) + ) + + remote -> + from([object: object] in query, + where: fragment("(?->>'id') LIKE ?", object.data, ^local_object_pattern) + ) + + true -> + query + end + end + + defp restrict_unauthenticated_object(query, _), do: query + + defp maybe_restrict_unauthenticated( + query, + %{restrict_unauthenticated: true, user: user} + ) do + query + |> restrict_unauthenticated(user) + |> restrict_unauthenticated_object(user) + end + + defp maybe_restrict_unauthenticated(query, _), do: query + defp restrict_quote_url(query, %{quote_url: quote_url}) do from([_activity, object] in query, where: fragment("(?)->'quoteUrl' = ?", object.data, ^quote_url) @@ -1514,6 +1551,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do |> maybe_preload_report_notes(opts) |> maybe_set_thread_muted_field(opts) |> maybe_order(opts) + |> maybe_restrict_unauthenticated(opts) |> restrict_recipients_or_hashtags(recipients, opts[:user], opts[:followed_hashtags]) |> restrict_replies(opts) |> restrict_since(opts) diff --git a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex index 6d5851029..631102843 100644 --- a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex @@ -348,6 +348,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do params |> Map.delete(:tagged) |> Map.put(:tag, params[:tagged]) + |> Map.put(:restrict_unauthenticated, true) activities = ActivityPub.fetch_user_activities(user, reading_user, params) diff --git a/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs index 01e195842..a2ac6382a 100644 --- a/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs @@ -458,6 +458,112 @@ defmodule Pleroma.Web.MastodonAPI.AccountControllerTest do assert [%{"id" => ^image_post_id}] = json_response_and_validate_schema(conn, 200) end + test "media respects activity restrictions when unauthenticated profiles are enabled", %{ + conn: conn + } do + clear_config([:instance, :public], false) + clear_config([:restrict_unauthenticated, :profiles, :local], false) + + user = insert(:user) + + file = %Plug.Upload{ + content_type: "image/jpeg", + path: Path.absname("test/fixtures/image.jpg"), + filename: "an_image.jpg" + } + + {:ok, %{id: media_id}} = ActivityPub.upload(file, actor: user.ap_id) + {:ok, activity} = CommonAPI.post(user, %{status: "cofe", media_ids: [media_id]}) + user_id = user.id + + assert %{"id" => ^user_id} = + build_conn() + |> get("/api/v1/accounts/#{user.id}") + |> json_response_and_validate_schema(200) + + assert [] = + build_conn() + |> get("/api/v1/accounts/#{user.id}/statuses?only_media=true") + |> json_response_and_validate_schema(200) + + assert [%{"id" => activity_id}] = + conn + |> get("/api/v1/accounts/#{user.id}/statuses?only_media=true") + |> json_response_and_validate_schema(200) + + assert activity_id == activity.id + end + + test "activity restrictions apply to boosted objects", %{conn: conn} do + clear_config([:restrict_unauthenticated, :activities, :local], false) + clear_config([:restrict_unauthenticated, :activities, :remote], true) + + local_user = insert(:user) + remote_user = insert(:user, local: false, domain: "example.com") + local_activity = insert(:note_activity, user: local_user) + + remote_activity = + insert(:note_activity, local: false, object_local: false, user: remote_user) + + {:ok, announce} = CommonAPI.repeat(remote_activity.id, local_user) + + status_ids = + build_conn() + |> get("/api/v1/accounts/#{local_user.id}/statuses") + |> json_response_and_validate_schema(200) + |> Enum.map(& &1["id"]) + + assert local_activity.id in status_ids + refute announce.id in status_ids + + authenticated_status_ids = + conn + |> get("/api/v1/accounts/#{local_user.id}/statuses") + |> json_response_and_validate_schema(200) + |> Enum.map(& &1["id"]) + + assert local_activity.id in authenticated_status_ids + assert announce.id in authenticated_status_ids + end + + test "local object restrictions apply to remote boosts", %{conn: conn} do + clear_config([:restrict_unauthenticated, :activities, :local], true) + clear_config([:restrict_unauthenticated, :activities, :remote], false) + + local_user = insert(:user) + remote_user = insert(:user, local: false, domain: "example.com") + local_activity = insert(:note_activity, user: local_user) + + remote_activity = + insert(:note_activity, local: false, object_local: false, user: remote_user) + + remote_announce = + insert(:announce_activity, note_activity: local_activity, user: remote_user) + |> Ecto.Changeset.change(%{ + local: false, + recipients: ["https://www.w3.org/ns/activitystreams#Public"] + }) + |> Repo.update!() + + status_ids = + build_conn() + |> get("/api/v1/accounts/#{remote_user.id}/statuses") + |> json_response_and_validate_schema(200) + |> Enum.map(& &1["id"]) + + assert remote_activity.id in status_ids + refute remote_announce.id in status_ids + + authenticated_status_ids = + conn + |> get("/api/v1/accounts/#{remote_user.id}/statuses") + |> json_response_and_validate_schema(200) + |> Enum.map(& &1["id"]) + + assert remote_activity.id in authenticated_status_ids + assert remote_announce.id in authenticated_status_ids + end + test "gets a user's statuses without reblogs", %{user: user, conn: conn} do {:ok, %{id: post_id}} = CommonAPI.post(user, %{status: "HI!!!"}) {:ok, _} = CommonAPI.repeat(post_id, user) From 6132c9e88a10192f6f34ae56ef3ca831decd0e3a Mon Sep 17 00:00:00 2001 From: Phantasm Date: Mon, 20 Jul 2026 14:15:58 +0200 Subject: [PATCH 70/90] Fix missing readme logo --- README.md | 2 +- logo-with-name.png | Bin 0 -> 6886 bytes 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 logo-with-name.png diff --git a/README.md b/README.md index 982a7249a..bccb39bdf 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ - + ## About diff --git a/logo-with-name.png b/logo-with-name.png new file mode 100644 index 0000000000000000000000000000000000000000..940d32b80341be7a07d77e2bc58f55ec3db91472 GIT binary patch literal 6886 zcmYjW1z40pyIlp8?(PQZ?r>=dK}1+Y8bqWU76n|IrI8K=q@78sp1A|kF?pBwV#MIP!DGe-n!BP!DqeT8n~3MJI9U}#*%+fcp37Hn4ACyZqsLzvM6O<^|DI zWgSwDU5*940t)oFN0nwoln|Y6FB&x1{{<~enTv2iiRE$Hg|J@N{gV%~toFL$+B}2u zQJ&t3?-J6cjHG}P>-H`m+ER1kJ)E}IS;9+!@$I%(I|>9z_Sj%TVRu|me3>&(YQ+$p z)!UCDPkYqBMUtnC4kLrNA0ri{YD0%$E{j)<6UW7eBq-%J5L)0*g)(D1gjl8_z?X2e zQm!w_cN`HMD6!#y0ZF@1O)nuP^zU6khcD}HPq?APA5tlV}DQ42>ue>oN)AK+>iC z8)P)%rL3`)nr(@N@zdj(BXV$Yve|!?cuR5`mJGEoj?CwjsS~T{vWXa%b+B5r_bF?X zVIzFuQMQ|5gYaHr2_-d3Hte>4SiHY0nO4B9bp`~*`i90hD|Vg_cFMP(VP&esDtp9y z!m7bw0+eW0Yy|Uv8%!iKHsZexDIbCv`@f9_3iSNH=D%b9Ga3`QZc6knf)OLnhna=! zveAP2UVAu^`oo8ifTPTLW@cu{e82_&c)5vs7mEfeq6n#!`w#}B{^KqUzQc%^2>tzc z3Z@Sa4}0`5aVQV|Mn*-oHTfOkJ3rOeR|;c=W4&)W3mq9B58Ilk%r7g`-jg`pon=!d z`D00<`#~pD!v5N<<&E^x^0ME|=;&yCF7_b`dyAEqR}0nrEH1^Sbdli$(kJ32Z#uU@@+=;XxjJl7O?dHE(w z)`w8>`Yhjgg1J_{+pV~?G@ni--hxC?)O#D&(AfCAwlXP+EGsK(VR7+Ym322O4-YP1 z-_q}|9+S=fk`*t$S`E8RPfz?CiREDOQrz!4Glh;QPxv_45)lCV8&$ z%tS~b4Kp)F9$sF*ot2dpz0xrVr0#jlzkl%GO@3tQ>FKa~X9na?CnqN<*98bGE32NF*}z4hjSV|FE*1_Y=;6t)q%<@- zYBSt&x;NjQ$f5O7zod4$(@L09(9F@hL6evr76PRXtAe2*AM^6=ZftBco&5Tx7tv`2 z0EdZ0F{1B=;6byqvxTiXZZQY|b}lqT%)!CY5O||dIhzF+B<0JKgxV4`PqyC;`td^( zpIYc)X}Ni8fV=<<<33?XTpSUI?)1#e6Cc)f>@ z&;7n?%D5PEWGA(-73MvBYKXeJdTm|Z;$nL^HxG}JwY7EXtbY{@-`w1Ma%yUE#-(-a zBD*_|iHM7fi#u79oPvU;d~u|+zn|dFojU;a;^N{BUR#FsUe&(7zTYP%sQCKw8XIZB zd+YHev%}~KW>LsS22%W02vz|m*B4&Pn^C1c+Vp3C6 z3+oj(557l$Xl+%zyt*PGCJuU%uLfmJSU?K4Ea0RV@Th8Q|k(yTLAIXpfo2_wa}A1+M5C`Zkv9^0`#ln)*W2@DLJ zoS9kvnJy}a-khlyOyM&|6c=+#NlCrBs{Hs7VsCHX)8CKPy%8wT+}wOHZsM{SN@e!z zhg~%v5|2t?p=MA))Xy`7<=@|p;D`w9rju!h{o~`Yf9C%FvH~v}g03!3m$#>Cz}#64 z>BGAyz2baf^xejPcCa}nOFw_so|}LF8O~z%=0LT~q@K{(*;%dA?)-2iB09R`M>4P5 z&a`;*jPq1AR&H)CTzNVw3b%Z4xhtmqi_;9)Zk7feq5G$&-5KHzLSQrIpYjL^5d-|g zs%>ycNl9IgR(0~bb#-;aD=P)*85po`0u<6#oz)yvRRs&hm@Jg2-?;prj=CLu{chSv}qnW99 z3D3@EaFMm1H)khhQBYuXb90kJS5{Pv`^qQ@VtjoO8y$_$&d$DYcCZW|TwGWPI#}+e zYl{8mvv(H)Dp-74S_Hs!rY8|1I2hz8LcZs<*stqot*WSng@t{$%Ih)ZRaoxC#>A*< zY2oSX>ua~1o}8TcUL8+V`5su!dDul5*StiJk5lfoTnOM8`XpwXn^fsh7iOZqm0HCA&9mf6Ixf9j|?2$s9_KUsN+co5OGyUex zZ)I(b6{vEbh*1S3w_AwQ)Bl`VjZAC$w^^#8tvzbelDieT0jJ57Z~_quDJ(~!h)YUJ zq`WrYcj3kN+W20duT^<&=%1aP?JMQ(%+%*sRgtU>exQ;g>Z`s@zF+j#w-#&oi#0O2CsFSM|qOh=UWxQYGazS zg1IUi8d8E_DkH`ihouw?U{OAUO5~boOdq-)^@tx=o1qY#lD>XNzxTt+kBOe zL&ix>&TEaRd)ZIRUnTSAOo)_>%+FdVS(C%v!^7?J%;0`%g=Hhw|0^a(5N44~Y8XzJ30T`yudJI+j7UP@LJ6Gs&_uiXu%hP+^ar z*Qg>yn{x#JO{?bysrTMo43bJPulZ`N$720g?%9_4&5xtA4v^pZU_mi-lC;gIqILtL z5y|=`9dph83F+y*ndReCQ#20aWoVFV{JLQcu9ihu$RD(ruNo0FA}|j)=Wq#> zl$4yDo?hyTp@X3;?#%fv6J#j{nrW3mK9-f?S5;MU^YM}63+gOho*ykp5#EmB1SR|Z zdvbJ1-PYDtLVSEs5|{4bH5D~A_xeVHg%=_*oTuX@`SMJ4m$Q7_Zj7iT%f{yB=i~8b zB?gn#wp#Vr1*xf2bN+ks{cCXw|F8=R3OE^Vh6}WszJP*2Gpk#ScLSmsQd-LUI7{kX zs(Ap7dMrKk*)xi$sHpwp33I&F#++Z3=8ENepbUdl@X9^T+}*v}A{@NC-6`5PBFYaUdsG%Gk z9Z`NqFLeSm$_?3ASkM;SqclVgQ`NQ^zX}Qq?}5rEaek&G{-n{ej97_-@#^2PvbVRy z+}l&roL>rlGamF@Fwr`&=gW^?+S=j_q~#K&*w(Ff?@=4}r;B#83u>j-)`}&qu{k(8 zlH(TN^1oa_?B%I}BpqMB zgdC*uc+CP=clY5-{$x#I5Jh|jnfE^09=Il6@}zIu0v5jCxdeNd^zvpe;8eDJx^R5l zF!uA9-mM-blGbbbAcSpN>*ipj1%X+uK6E`xbMVEusd`o*iIn@g#d+oPq?fQ7aGE zM9Im^`!%DLj$gCGn%@Hqq#1ml7^6NMe(~akuTx6Jzz79Qgi_~K1i?h_oGlAlVq(V0 z!9i3uYlMPM3scMJFQ5y6z~;O?F~+=qpG`_CjcXn9(@Z@HvoA`yt*B1oV=hx=yaf&I z*2FAqIFrM(IlCN-l$6iI18Pb#;MFP2r-J|#21o@%Hobs=fF>$s2Zz#e6f0?&IdKZ- zyKhfXA!?UI3^KLq?ZX3bm{Lw$JaJ~Q-qPf6zgE6;0aFsvO~!V5bYibfn!;NdKpTYJ zmQ>~->nA7PFIwKv6zP8GDtTIjvg*VwPo(x(J7PswTM_lqVrdl{Jr+oHc$RGPEu7;o zL$pq$HU8{|IZ!~ZplCiu&W-G&v&t+&J=vPn)~zDC-swc4iitm8fpc692*;J`qeQz%P=T+6y4bz+KBK0jIKr*uGLY` z>kdl^2?6X65V~R9^MM?BRw~>KjH_lJfDA^W{p%HPYs0y=NgRt*9Wx!oq4|>Te6h`=BZH%6@hVFa&!^etyarvGu5U6nN>vKf{1C)GC-axS~J#=WbbgbrLg^mKB9mJT%tt zwiZz0!0BR?%N33SOpUW&?-z}yf0oP<3UB2%2ntrWG_A7Hx`?PK|7XGKj`q!1`V`@B zGE!3eIXX}MjlJZ{Dq#B|m|G8WGw%!AzMdIi^(3^WIK8<(ZS^Cn)8=e1`VcT(&RUxJ zTfF{d30taw>5@+7Pa*tU#W#Mb?d|Oy5gR*>TcmLQhYVTKo498k>0`4^-6hwo9>;hv zAR2J#8%)M6f*)KzA3%2-6tqW@#W5)$R+SE>^KBTWk$StmFon-x{{iOKbOH(%3mY3o zPEHOd-Y*0e-hOK9W#^|IuOb#VZ*NL`dmU)GAs%jDM(9`YBOAM{XJ}YBCgC_K)O5a@30Gdtmi4g$QEIxux_9qz zx~aJt2px;YaE-&EBNa8Z!#TZtwV1o@LfI8<5+T3xuoV;(5J`PrHWL-5*m!ust*xzw z7d;7V37MHlAe5N7xb6eFNb91_n()279b>sWP93N#!1^uh?Cygc{`TU8#&k;kmCSrl z5CoZ8Bm@7;{c<2H5`_{fKS|~_%4=?B04nO96B7eN42M?AD+MM{W6S)|o<|Rr!|SfZt3C@=YP73^LVUu(4nuVhxmo^ zL8eO!dLUcYZRH0=xvSIP^&wcJ=Yzt>*`MwE1_rvocGkh&0!fq@tR~>&W@qUM3GYD7 z2Xb6WR(5%J_8UT4HBtp}yftyeiECp_Nhj&70_+azuy-2l!zMBd8yk^;Dn>KPKQFzy zzPfN$GBNO%1<7eNhn?us5OsiAv%eF!|z0^Z0{8`Dyp*K{pd#<4G4c zKfkJ#RUTS;ZN**sR$>u~@>j)yKOqH}8Nw!v9z=v)ZDF zrBFK^(bJ=mWOq*fy!fNF0xu`$y+@B8aT;l)CMR10YMmzSQ8-+c&h z{(!*;>2?DOk~mVu)zvi!#wQ`e8Eb3&si0tS0Udl>@W3+M1)v3xa`@sWQn(Egf|U1m zTCcnnPcp8wv@|WS{?rBy|tBG zj8A2Kb{AT2t_0oP-T5+d|AJW-mzAklTj!h6Ny*5x4GobfUhW4kcwk%_u$Vo0@(%1e z10y2`oe(?FC;j+YkEpk+>-Pe!R6P@u(yx0w$(p5WMm))FU0wGcKYrZcwj72|H3l9A zr%px)jp$1YU?jPEd36FIwtox0mTf{txd`Cz0WjM)O;=ZoI8LI_+Y~w6?3WY^o10N# zVPVc62-?QRC}Lw{fsOU-*|S**U0&v@803f>}F0ny3EIe^S1nd zcV?&r1Ox!E>l(1rRPK25q9DIDDB^UL+}oosQjQ!ZC1xGU2QSyR||-1&MiMm93E( z__xm+*VQ3IW)ymh+&<1&aq7HI&x4$l6cZ;U`RL(P*k2@3&+SufXT`o0C&U}azQ(2df~7c7AUj1N+Z=Qxn3h=ExBq}^D>&n1 zLw8Z@ZcVU Date: Mon, 27 Jul 2026 11:57:16 +0400 Subject: [PATCH 71/90] fix(oauth): preserve dependent app cleanup --- .../optimize-orphaned-apps-cleanup.change | 2 +- lib/pleroma/web/o_auth/app.ex | 31 ++++++++++++------- test/pleroma/web/o_auth/app_test.exs | 12 ++++++- 3 files changed, 31 insertions(+), 14 deletions(-) diff --git a/changelog.d/optimize-orphaned-apps-cleanup.change b/changelog.d/optimize-orphaned-apps-cleanup.change index e86b75524..3f037f125 100644 --- a/changelog.d/optimize-orphaned-apps-cleanup.change +++ b/changelog.d/optimize-orphaned-apps-cleanup.change @@ -1 +1 @@ -- Optimize orphaned OAuth apps cleanup by using a single delete query instead of deleting records one by one. +- Optimize orphaned OAuth apps cleanup by deleting records in batches instead of one by one. diff --git a/lib/pleroma/web/o_auth/app.ex b/lib/pleroma/web/o_auth/app.ex index 4ea02ee01..9a3dcfcea 100644 --- a/lib/pleroma/web/o_auth/app.ex +++ b/lib/pleroma/web/o_auth/app.ex @@ -8,7 +8,7 @@ defmodule Pleroma.Web.OAuth.App do import Ecto.Query alias Pleroma.Repo alias Pleroma.User - alias Pleroma.Web.OAuth.Token + alias Pleroma.Web.OAuth.{Authorization, Token} @type t :: %__MODULE__{} @@ -193,18 +193,25 @@ defmodule Pleroma.Web.OAuth.App do def remove_orphans(limit \\ 100) do fifteen_mins_ago = NaiveDateTime.add(NaiveDateTime.utc_now(), -900) - # First get the IDs of apps to delete - app_ids = - from(a in __MODULE__, - where: is_nil(a.user_id) and a.inserted_at < ^fifteen_mins_ago, - limit: ^limit, - select: a.id - ) - |> Repo.all() + Repo.transaction(fn -> + app_ids = + from(a in __MODULE__, + where: is_nil(a.user_id) and a.inserted_at < ^fifteen_mins_ago, + limit: ^limit, + lock: "FOR UPDATE SKIP LOCKED", + select: a.id + ) + |> Repo.all() - # Then delete those specific apps - from(a in __MODULE__, where: a.id in ^app_ids) - |> Repo.delete_all() + # Bulk deletes bypass the schema's association cleanup. + from(a in Authorization, where: a.app_id in ^app_ids) |> Repo.delete_all() + from(t in Token, where: t.app_id in ^app_ids) |> Repo.delete_all() + + from(a in __MODULE__, + where: a.id in ^app_ids and is_nil(a.user_id) and a.inserted_at < ^fifteen_mins_ago + ) + |> Repo.delete_all() + end) :ok end diff --git a/test/pleroma/web/o_auth/app_test.exs b/test/pleroma/web/o_auth/app_test.exs index 4d100cd13..26879d6ee 100644 --- a/test/pleroma/web/o_auth/app_test.exs +++ b/test/pleroma/web/o_auth/app_test.exs @@ -5,7 +5,9 @@ defmodule Pleroma.Web.OAuth.AppTest do use Pleroma.DataCase, async: true - alias Pleroma.Web.OAuth.App + alias Pleroma.MFA.Token, as: MFAToken + alias Pleroma.Web.OAuth.{App, Authorization, Token} + alias Pleroma.Web.Push.Subscription import Pleroma.Factory describe "get_or_make/2" do @@ -57,6 +59,10 @@ defmodule Pleroma.Web.OAuth.AppTest do test "removes orphaned apps" do attrs = %{client_name: "Mastodon-Local", redirect_uris: "."} {:ok, %App{} = old_app} = App.get_or_make(attrs, ["write"]) + authorization = insert(:oauth_authorization, app: old_app) + token = insert(:oauth_token, app: old_app) + mfa_token = insert(:mfa_token, authorization: authorization) + push_subscription = insert(:push_subscription, token: token) user = insert(:user) attrs = %{client_name: "OldButValid", redirect_uris: ".", user_id: user.id} @@ -74,6 +80,10 @@ defmodule Pleroma.Web.OAuth.AppTest do App.remove_orphans() refute Repo.get(App, old_app.id) + refute Repo.get(Authorization, authorization.id) + refute Repo.get(Token, token.id) + refute Repo.get(MFAToken, mfa_token.id) + refute Repo.get(Subscription, push_subscription.id) assert Repo.get(App, recent_app.id) assert Repo.get(App, old_owned_app.id) end From 030c7e929c46210f408bc8a2c7f2fa603a9a80a5 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Mon, 27 Jul 2026 12:14:58 +0400 Subject: [PATCH 72/90] style(oauth): use single alias directives --- lib/pleroma/web/o_auth/app.ex | 3 ++- test/pleroma/web/o_auth/app_test.exs | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/o_auth/app.ex b/lib/pleroma/web/o_auth/app.ex index 9a3dcfcea..9c16c23b6 100644 --- a/lib/pleroma/web/o_auth/app.ex +++ b/lib/pleroma/web/o_auth/app.ex @@ -8,7 +8,8 @@ defmodule Pleroma.Web.OAuth.App do import Ecto.Query alias Pleroma.Repo alias Pleroma.User - alias Pleroma.Web.OAuth.{Authorization, Token} + alias Pleroma.Web.OAuth.Authorization + alias Pleroma.Web.OAuth.Token @type t :: %__MODULE__{} diff --git a/test/pleroma/web/o_auth/app_test.exs b/test/pleroma/web/o_auth/app_test.exs index 26879d6ee..9a5045155 100644 --- a/test/pleroma/web/o_auth/app_test.exs +++ b/test/pleroma/web/o_auth/app_test.exs @@ -6,7 +6,9 @@ defmodule Pleroma.Web.OAuth.AppTest do use Pleroma.DataCase, async: true alias Pleroma.MFA.Token, as: MFAToken - alias Pleroma.Web.OAuth.{App, Authorization, Token} + alias Pleroma.Web.OAuth.App + alias Pleroma.Web.OAuth.Authorization + alias Pleroma.Web.OAuth.Token alias Pleroma.Web.Push.Subscription import Pleroma.Factory From 3b71770cfa26ee6512324201a3426b4e7a18faff Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Mon, 27 Jul 2026 15:31:59 +0400 Subject: [PATCH 73/90] docs: update Debian and Ubuntu requirements --- changelog.d/improve-doc-debian-13.change | 2 +- docs/installation/debian_based_en.md | 2 +- docs/installation/otp_en.md | 32 +++++++++++++----------- 3 files changed, 20 insertions(+), 16 deletions(-) diff --git a/changelog.d/improve-doc-debian-13.change b/changelog.d/improve-doc-debian-13.change index c8f62ef71..0fd7c6e45 100644 --- a/changelog.d/improve-doc-debian-13.change +++ b/changelog.d/improve-doc-debian-13.change @@ -1 +1 @@ -Improve Debian 13 OTP dependencies and remove Debian 12 in source installation +Update Debian and Ubuntu installation requirements and OTP dependencies diff --git a/docs/installation/debian_based_en.md b/docs/installation/debian_based_en.md index b6730ce19..f9086d2f9 100644 --- a/docs/installation/debian_based_en.md +++ b/docs/installation/debian_based_en.md @@ -4,7 +4,7 @@ ## Installation -This guide will assume you are on Debian 13 (“Trixie”) or later. This guide should also work with Ubuntu 24.04 (“noble”) and later. It also assumes that you have administrative rights, either as root or a user with [sudo permissions](https://www.digitalocean.com/community/tutorials/how-to-add-delete-and-grant-sudo-privileges-to-users-on-a-debian-vps). If you want to run this guide with root, ignore the `sudo` at the beginning of the lines, unless it calls a user like `sudo -Hu pleroma`; in this case, use `su -s $SHELL -c 'command'` instead. +This guide will assume you are on Debian 13 (“Trixie”) or later. This guide should also work with Ubuntu 26.04 (“resolute”) and later. It also assumes that you have administrative rights, either as root or a user with [sudo permissions](https://www.digitalocean.com/community/tutorials/how-to-add-delete-and-grant-sudo-privileges-to-users-on-a-debian-vps). If you want to run this guide with root, ignore the `sudo` at the beginning of the lines, unless it calls a user like `sudo -Hu pleroma`; in this case, use `su -s $SHELL -c 'command'` instead. {! backend/installation/generic_dependencies.include !} diff --git a/docs/installation/otp_en.md b/docs/installation/otp_en.md index a1238b160..196685de6 100644 --- a/docs/installation/otp_en.md +++ b/docs/installation/otp_en.md @@ -45,14 +45,16 @@ Other than things bundled in the OTP release Pleroma depends on: ``` === "Debian/Ubuntu" - ``` - apt install curl unzip libncurses5 postgresql postgresql-contrib nginx certbot libmagic-dev - ``` - On Debian 13 and Ubuntu 24.04, replace `libncurses5` by `libncurses6` and add `libvips42t64`. So ``` apt install curl unzip libncurses6 libvips42t64 postgresql postgresql-contrib nginx certbot libmagic-dev ``` + On Debian 12 and Ubuntu 22.04, replace `libvips42t64` with `libvips42`: + + ``` + apt install curl unzip libncurses6 libvips42 postgresql postgresql-contrib nginx certbot libmagic-dev + ``` + ### Installing optional packages Per [`docs/installation/optional/media_graphics_packages.md`](optional/media_graphics_packages.md): @@ -92,20 +94,22 @@ RUM indexes are an alternative indexing scheme that is not included in PostgreSQ ``` === "Debian/Ubuntu" + Install the package matching your distribution's PostgreSQL version: + ``` - # On Trixie and Ubuntu 25.04 + # Debian 13 apt install postgresql-17-rum - ``` - # On Debian 12 - ``` + + # Debian 12 apt install postgresql-15-rum - ``` - # On Ubuntu 24.04 - ``` + + # Ubuntu 26.04 + apt install postgresql-18-rum + + # Ubuntu 24.04 apt install postgresql-16-rum - ``` - # On Ubuntu 22.04 - ``` + + # Ubuntu 22.04 apt install postgresql-14-rum ``` From be5a3957b2ddeaf51c4e7a5eab2a8b35041750f3 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Mon, 27 Jul 2026 16:12:21 +0400 Subject: [PATCH 74/90] docs: explain libvips dependency --- docs/installation/otp_en.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/installation/otp_en.md b/docs/installation/otp_en.md index 196685de6..5fc1db21e 100644 --- a/docs/installation/otp_en.md +++ b/docs/installation/otp_en.md @@ -32,6 +32,7 @@ Other than things bundled in the OTP release Pleroma depends on: * curl (to download the release build) * unzip (needed to unpack release builds) * ncurses (ERTS won't run without it) +* libvips (for image processing) * PostgreSQL (also utilizes extensions in postgresql-contrib) * nginx (could be swapped with another reverse proxy but this guide covers only it) * certbot (for Let's Encrypt certificates, could be swapped with another ACME client, but this guide covers only it) From bc999b330982bc8d0c95818ebfddd5a306524fa2 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Mon, 27 Jul 2026 16:32:38 +0400 Subject: [PATCH 75/90] fix(media-proxy): abort truncated streams --- changelog.d/media-proxy-truncated-stream.fix | 1 + lib/pleroma/reverse_proxy.ex | 211 ++++++--- test/pleroma/reverse_proxy_test.exs | 446 ++++++++++++++++++- 3 files changed, 591 insertions(+), 67 deletions(-) create mode 100644 changelog.d/media-proxy-truncated-stream.fix diff --git a/changelog.d/media-proxy-truncated-stream.fix b/changelog.d/media-proxy-truncated-stream.fix new file mode 100644 index 000000000..12413e53e --- /dev/null +++ b/changelog.d/media-proxy-truncated-stream.fix @@ -0,0 +1 @@ +Abort incomplete media proxy responses instead of caching partial bodies. diff --git a/lib/pleroma/reverse_proxy.ex b/lib/pleroma/reverse_proxy.ex index e87c3f625..12e48eadd 100644 --- a/lib/pleroma/reverse_proxy.ex +++ b/lib/pleroma/reverse_proxy.ex @@ -12,7 +12,7 @@ defmodule Pleroma.ReverseProxy do @keep_resp_headers @resp_cache_headers ++ ~w(content-length content-type content-disposition content-encoding) ++ ~w(content-range accept-ranges vary) - @default_cache_control_header "public, max-age=1209600, immutable" + @default_cache_control_header "public, max-age=1209600, immutable, no-transform" @valid_resp_codes [200, 206, 304] @max_read_duration :timer.seconds(30) @max_body_length :infinity @@ -24,6 +24,16 @@ defmodule Pleroma.ReverseProxy do @cachex Pleroma.Config.get([:cachex, :provider], Cachex) + defmodule StreamError do + @moduledoc false + defexception [:url, :reason] + + @impl true + def message(%{url: url, reason: reason}) do + "reverse proxy stream from #{url} failed: #{inspect(reason)}" + end + end + def max_read_duration_default, do: @max_read_duration def default_cache_control_header, do: @default_cache_control_header @@ -190,34 +200,56 @@ defmodule Pleroma.ReverseProxy do end defp request_with_constraints(method, url, headers, client_opts, opts) do - case request(method, url, headers, client_opts) do - {:ok, _code, headers, client} = response -> - case header_length_constraint( - headers, - Keyword.get(opts, :max_body_length, @max_body_length) - ) do - :ok -> - response + method + |> request(url, headers, client_opts) + |> constrain_response(Keyword.get(opts, :max_body_length, @max_body_length)) + end - error -> - client().close(client) - error - end + defp constrain_response({:ok, code, headers, client}, limit) do + case header_length_constraint(headers, limit) do + {:ok, headers} -> + {:ok, code, headers, client} - response -> - response + error -> + client().close(client) + error end end + defp constrain_response({:ok, code, headers}, limit) do + case header_length_constraint(headers, limit) do + {:ok, headers} -> {:ok, code, headers} + error -> error + end + end + + defp constrain_response(response, _limit), do: response + defp response(conn, client, url, status, headers, opts) do Logger.debug("#{__MODULE__} #{status} #{url} #{inspect(headers)}") - {headers, client, prefetched} = maybe_prefetch_for_content_type(headers, client, opts) + case maybe_prefetch_for_content_type(headers, client, opts) do + {_headers, client, {:error, {:upstream, error}}} -> + Logger.warning( + "#{__MODULE__} request to #{url} failed before streaming: #{inspect(error)}" + ) + client().close(client) + track_failed_url(url, error, opts) + + conn + |> error_or_redirect(url, 500, "Request failed", opts) + |> halt() + + {headers, client, prefetched} -> + stream_response(conn, client, url, status, headers, opts, prefetched) + end + end + + defp stream_response(conn, client, url, status, headers, opts, prefetched) do result = conn |> put_resp_headers(build_resp_headers(headers, opts)) - |> streaming_compat |> send_chunked(status) |> chunk_reply(client, opts, prefetched) @@ -225,17 +257,18 @@ defmodule Pleroma.ReverseProxy do {:ok, conn} -> halt(conn) - {:error, :closed, conn} -> + {:error, {:downstream, _error}, conn} -> client().close(client) halt(conn) - {:error, error, conn} -> + {:error, {:upstream, error}, conn} -> Logger.warning( "#{__MODULE__} request to #{url} failed while reading/chunking: #{inspect(error)}" ) client().close(client) - halt(conn) + track_failed_url(url, error, opts) + raise_stream_error(conn, url, error) end end @@ -252,10 +285,11 @@ defmodule Pleroma.ReverseProxy do byte_size(data), Keyword.get(opts, :max_body_length, @max_body_length) ), - {:ok, conn} <- chunk(conn, data) do + {:ok, conn} <- write_chunk(conn, data) do chunk_reply(conn, client, opts, byte_size(data), duration) else - {:error, error} -> {:error, error, conn} + {:error, {:downstream, error}} -> {:error, {:downstream, error}, conn} + {:error, error} -> {:error, {:upstream, error}, conn} end end @@ -267,23 +301,37 @@ defmodule Pleroma.ReverseProxy do sent_so_far, Keyword.get(opts, :max_body_length, @max_body_length) ), - {:ok, conn} <- chunk(conn, data) do + {:ok, conn} <- write_chunk(conn, data) do chunk_reply(conn, client, opts, sent_so_far, duration) else :done -> {:ok, conn} - {:error, error} -> {:error, error, conn} + {:error, {source, error}} -> {:error, {source, error}, conn} + {:error, error} -> {:error, {:upstream, error}, conn} end end defp read_chunk(client, duration, opts) do - with {:ok, timer} <- - check_read_duration( - duration, - Keyword.get(opts, :max_read_duration, @max_read_duration) - ), - {:ok, data, client} <- client().stream_body(client), - {:ok, duration} <- increase_read_duration(timer) do - {:ok, data, client, duration} + result = + with {:ok, timer} <- + check_read_duration( + duration, + Keyword.get(opts, :max_read_duration, @max_read_duration) + ), + {:ok, data, client} <- client().stream_body(client), + {:ok, duration} <- increase_read_duration(timer) do + {:ok, data, client, duration} + end + + case result do + {:error, error} -> {:error, {:upstream, error}} + result -> result + end + end + + defp write_chunk(conn, data) do + case chunk(conn, data) do + {:error, error} -> {:error, {:downstream, error}} + result -> result end end @@ -427,7 +475,24 @@ defmodule Pleroma.ReverseProxy do |> build_resp_cache_headers(opts) |> sanitise_content_type() |> build_resp_content_disposition_header(opts) - |> Keyword.merge(Keyword.get(opts, :resp_headers, [])) + |> merge_headers(Keyword.get(opts, :resp_headers, [])) + |> ensure_no_transform() + end + + defp ensure_no_transform(headers) do + {_, cache_control} = + List.keyfind(headers, "cache-control", 0, {"cache-control", @default_cache_control_header}) + + directives = + cache_control + |> String.split(",") + |> Enum.map(&(&1 |> String.trim() |> String.downcase())) + + if "no-transform" in directives do + headers + else + replace_header(headers, "cache-control", cache_control <> ", no-transform") + end end defp sanitise_content_type(headers) do @@ -531,21 +596,34 @@ defmodule Pleroma.ReverseProxy do end end - defp header_length_constraint(headers, limit) when is_integer(limit) and limit > 0 do - with {_, size} <- List.keyfind(headers, "content-length", 0), - {size, _} <- Integer.parse(size), - true <- size <= limit do - :ok - else - false -> - {:error, :body_too_large} + defp header_length_constraint(headers, limit) do + lengths = + for {"content-length", value} <- headers, + do: parse_content_length(value) + + case Enum.uniq(lengths) do + [] -> + {:ok, headers} + + [size] when is_integer(size) -> + case body_size_constraint(size, limit) do + :ok -> {:ok, replace_header(headers, "content-length", to_string(size))} + error -> error + end _ -> - :ok + {:error, :invalid_content_length} end end - defp header_length_constraint(_, _), do: :ok + defp parse_content_length(value) when is_binary(value) do + case Integer.parse(value) do + {size, ""} when size >= 0 -> size + _ -> :invalid + end + end + + defp parse_content_length(_value), do: :invalid defp body_size_constraint(size, limit) when is_integer(limit) and limit > 0 and size > limit do {:error, :body_too_large} @@ -574,6 +652,40 @@ defmodule Pleroma.ReverseProxy do defp client, do: Pleroma.ReverseProxy.Client.Wrapper + # Neither Plug adapter exposes a public way to abort a committed response. + # Cowboy finalizes chunked HTTP/1 responses when the request process exits, + # so terminate the connection first to leave the response visibly incomplete. + defp raise_stream_error( + %Plug.Conn{adapter: {Plug.Cowboy.Conn, %{pid: connection_pid}}} = conn, + url, + error + ) do + if Plug.Conn.get_http_protocol(conn) in [:"HTTP/1.0", :"HTTP/1.1"] do + Process.exit(connection_pid, :kill) + end + + raise StreamError, url: url, reason: error + end + + # Bandit turns this exception into RST_STREAM; a regular exception sends an + # empty DATA frame with END_STREAM and makes the partial body look complete. + defp raise_stream_error( + %Plug.Conn{ + adapter: {Bandit.Adapter, %{transport: %{stream_id: stream_id}}} + }, + url, + error + ) do + raise Bandit.HTTP2.Errors.StreamError, + message: StreamError.message(%StreamError{url: url, reason: error}), + error_code: Bandit.HTTP2.Errors.internal_error(), + stream_id: stream_id + end + + defp raise_stream_error(_conn, url, error) do + raise StreamError, url: url, reason: error + end + defp track_failed_url(url, error, opts) do ttl = unless error in [:body_too_large, 400, 204] do @@ -585,19 +697,6 @@ defmodule Pleroma.ReverseProxy do @cachex.put(:failed_proxy_url_cache, url, true, ttl: ttl) end - # When Cowboy handles a chunked response with a content-length header it streams - # over HTTP 1.1 instead of chunking. Bandit cannot stream over HTTP 1.1 so the header - # must be stripped or it breaks RFC compliance for Transfer Encoding: Chunked. RFC9112§6.2 - # - # HTTP2 is always streamed for all adapters. - defp streaming_compat(conn) do - with Phoenix.Endpoint.Cowboy2Adapter <- Pleroma.Web.Endpoint.config(:adapter) do - conn - else - _ -> delete_resp_header(conn, "content-length") - end - end - # Only when Tesla adapter is Hackney or Finch does the URL # need encoding before Reverse Proxying as both end up # using the raw Hackney client and cannot leverage our diff --git a/test/pleroma/reverse_proxy_test.exs b/test/pleroma/reverse_proxy_test.exs index 63ab4202c..bf0a809c4 100644 --- a/test/pleroma/reverse_proxy_test.exs +++ b/test/pleroma/reverse_proxy_test.exs @@ -11,6 +11,15 @@ defmodule Pleroma.ReverseProxyTest do alias Pleroma.ReverseProxy.ClientMock alias Plug.Conn + defmodule ProxyPlug do + @moduledoc false + + def init(options), do: options + + def call(conn, {url, opts}), do: Pleroma.ReverseProxy.call(conn, url, opts) + def call(conn, url), do: Pleroma.ReverseProxy.call(conn, url) + end + setup_all do {:ok, _} = Registry.start_link(keys: :unique, name: ClientMock) :ok @@ -18,6 +27,88 @@ defmodule Pleroma.ReverseProxyTest do setup :verify_on_exit! + defp start_bandit(url, opts \\ []) do + start_supervised!( + {Bandit, + plug: {ProxyPlug, {url, opts}}, + ip: {127, 0, 0, 1}, + port: 0, + http_options: [log_exceptions_with_status_codes: []]} + ) + end + + defp start_cowboy(url, opts \\ []) do + ref = {__MODULE__, make_ref()} + + {:ok, _pid} = + Plug.Cowboy.http(ProxyPlug, {url, opts}, + ip: {127, 0, 0, 1}, + port: 0, + ref: ref, + protocol_options: [stream_handlers: [:cowboy_stream_h]] + ) + + on_exit(fn -> Plug.Cowboy.shutdown(ref) end) + :ranch.get_port(ref) + end + + defp bandit_request(url, headers \\ [], opts \\ []) do + pid = start_bandit(url, opts) + {:ok, {{127, 0, 0, 1}, port}} = ThousandIsland.listener_info(pid) + http1_request(port, headers) + end + + defp cowboy_request(url, headers \\ [], opts \\ []) do + url + |> start_cowboy(opts) + |> http1_request(headers) + end + + defp http1_request(port, headers) do + {:ok, socket} = :gen_tcp.connect({127, 0, 0, 1}, port, [:binary, active: false]) + + request_headers = + [["host: localhost", "connection: close"] | headers] + |> List.flatten() + |> Enum.join("\r\n") + + :ok = :gen_tcp.send(socket, "GET / HTTP/1.1\r\n#{request_headers}\r\n\r\n") + recv_all(socket, []) + end + + defp recv_all(socket, chunks) do + case :gen_tcp.recv(socket, 0, 5_000) do + {:ok, data} -> recv_all(socket, [data | chunks]) + {:error, reason} -> {reason, chunks |> Enum.reverse() |> IO.iodata_to_binary()} + end + end + + defp receive_mint_responses(conn, responses \\ []) do + receive do + message -> + case Mint.HTTP2.stream(conn, message) do + {:ok, conn, new_responses} -> + responses = responses ++ new_responses + + if Enum.any?(new_responses, fn response -> + elem(response, 0) in [:done, :error] + end) do + responses + else + receive_mint_responses(conn, responses) + end + + {:error, _conn, error, new_responses} -> + responses ++ new_responses ++ [{:connection_error, error}] + + :unknown -> + receive_mint_responses(conn, responses) + end + after + 2_000 -> responses ++ [:timeout] + end + end + defp request_mock(invokes) do ClientMock |> expect(:request, fn :get, url, headers, _body, _opts -> @@ -43,6 +134,45 @@ defmodule Pleroma.ReverseProxyTest do end) end + defp stream_then_error_mock(url, headers, chunk \\ "partial", error \\ :closed) do + ClientMock + |> expect(:request, fn :get, ^url, _, _, _ -> + Registry.register(ClientMock, url, 0) + {:ok, 200, headers, %{url: url}} + end) + |> expect(:stream_body, 2, fn %{url: ^url} = client -> + case Registry.lookup(ClientMock, url) do + [{_, 0}] -> + Registry.update_value(ClientMock, url, &(&1 + 1)) + {:ok, chunk, client} + + [{_, 1}] -> + Registry.unregister(ClientMock, url) + {:error, error} + end + end) + |> expect(:close, fn _ -> :ok end) + end + + defp complete_stream_mock(url, headers, body) do + ClientMock + |> expect(:request, fn :get, ^url, _, _, _ -> + Registry.register(ClientMock, url, 0) + {:ok, 200, headers, %{url: url}} + end) + |> expect(:stream_body, 2, fn %{url: ^url} = client -> + case Registry.lookup(ClientMock, url) do + [{_, 0}] -> + Registry.update_value(ClientMock, url, &(&1 + 1)) + {:ok, body, client} + + [{_, 1}] -> + Registry.unregister(ClientMock, url) + :done + end + end) + end + describe "reverse proxy" do test "do not track successful request", %{conn: conn} do request_mock(2) @@ -70,14 +200,202 @@ defmodule Pleroma.ReverseProxyTest do assert response == %{"user-agent" => Pleroma.Application.user_agent()} end - test "closed connection", %{conn: conn} do - ClientMock - |> expect(:request, fn :get, "/closed", _, _, _ -> {:ok, 200, [], %{}} end) - |> expect(:stream_body, fn _ -> {:error, :closed} end) - |> expect(:close, fn _ -> :ok end) + test "aborts a partially streamed response when the upstream closes", %{conn: conn} do + stream_then_error_mock("/closed", [{"content-length", "20"}]) - conn = ReverseProxy.call(conn, "/closed") - assert conn.halted + assert_raise ReverseProxy.StreamError, fn -> + ReverseProxy.call(conn, "/closed") + end + + assert Cachex.get(:failed_proxy_url_cache, "/closed") == {:ok, true} + end + + describe "Bandit HTTP/1 streaming" do + test "closes an incomplete chunked response without a terminating chunk" do + Mox.set_mox_global(ClientMock) + stream_then_error_mock("/bandit-closed", [{"content-type", "image/png"}]) + + assert {:closed, response} = bandit_request("/bandit-closed") + assert response =~ "transfer-encoding: chunked" + assert response =~ "partial" + refute response =~ "0\r\n\r\n" + end + + test "closes an incomplete fixed-length response before the declared length" do + Mox.set_mox_global(ClientMock) + + stream_then_error_mock("/bandit-fixed-closed", [ + {"content-length", "20"}, + {"content-type", "image/png"} + ]) + + assert {:closed, response} = bandit_request("/bandit-fixed-closed") + assert response =~ "content-length: 20" + refute response =~ "transfer-encoding" + assert String.ends_with?(response, "partial") + end + + test "streams a complete fixed-length response without transformation" do + Mox.set_mox_global(ClientMock) + body = "complete body" + + complete_stream_mock( + "/bandit-complete", + [ + {"content-length", to_string(byte_size(body))}, + {"content-type", "image/png"} + ], + body + ) + + assert {:closed, response} = + bandit_request("/bandit-complete", ["accept-encoding: deflate"]) + + assert response =~ "content-length: #{byte_size(body)}" + assert response =~ "cache-control: public, max-age=1209600, immutable, no-transform" + refute response =~ "content-encoding: gzip" + refute response =~ "content-encoding: deflate" + assert String.ends_with?(response, body) + end + + test "keeps no-transform when custom response headers replace cache-control" do + Mox.set_mox_global(ClientMock) + body = "complete body" + + complete_stream_mock( + "/bandit-custom-cache-control", + [ + {"content-length", to_string(byte_size(body))}, + {"content-type", "image/png"} + ], + body + ) + + assert {:closed, response} = + bandit_request( + "/bandit-custom-cache-control", + ["accept-encoding: deflate"], + resp_headers: [{"cache-control", "private, max-age=60"}] + ) + + assert response =~ "content-length: #{byte_size(body)}" + assert response =~ "cache-control: private, max-age=60, no-transform" + refute response =~ "content-encoding: gzip" + refute response =~ "content-encoding: deflate" + assert String.ends_with?(response, body) + end + + test "does not blacklist the upstream when the downstream disconnects" do + Mox.set_mox_global(ClientMock) + test_pid = self() + + ClientMock + |> expect(:request, fn :get, "/bandit-client-closed", _, _, _ -> + {:ok, 200, [{"content-type", "image/png"}], %{url: "/bandit-client-closed"}} + end) + |> stub(:stream_body, fn client -> + send(test_pid, {:stream_requested, self()}) + + receive do + :continue -> {:ok, :binary.copy("x", 1_000_000), client} + after + 1_000 -> raise "timed out waiting for downstream disconnect" + end + end) + |> expect(:close, fn _ -> + send(test_pid, :upstream_closed) + :ok + end) + + pid = start_bandit("/bandit-client-closed") + {:ok, {{127, 0, 0, 1}, port}} = ThousandIsland.listener_info(pid) + {:ok, socket} = :gen_tcp.connect({127, 0, 0, 1}, port, [:binary, active: false]) + :ok = :gen_tcp.send(socket, "GET / HTTP/1.1\r\nhost: localhost\r\n\r\n") + + assert_receive {:stream_requested, request_pid} + :ok = :inet.setopts(socket, linger: {true, 0}) + :ok = :gen_tcp.close(socket) + send(request_pid, :continue) + + assert_receive :upstream_closed + assert Cachex.get(:failed_proxy_url_cache, "/bandit-client-closed") == {:ok, nil} + end + end + + describe "Cowboy streaming" do + test "closes an incomplete HTTP/1 chunked response without a terminating chunk" do + Mox.set_mox_global(ClientMock) + stream_then_error_mock("/cowboy-closed", [{"content-type", "image/png"}]) + + assert {:closed, response} = cowboy_request("/cowboy-closed") + assert response =~ "transfer-encoding: chunked" + assert response =~ "partial" + refute response =~ "0\r\n\r\n" + end + + test "closes an incomplete HTTP/1 fixed-length response before the declared length" do + Mox.set_mox_global(ClientMock) + + stream_then_error_mock("/cowboy-fixed-closed", [ + {"content-length", "20"}, + {"content-type", "image/png"} + ]) + + assert {:closed, response} = cowboy_request("/cowboy-fixed-closed") + assert response =~ "content-length: 20" + refute response =~ "transfer-encoding" + assert String.ends_with?(response, "partial") + end + + test "resets an incomplete HTTP/2 stream" do + Mox.set_mox_global(ClientMock) + stream_then_error_mock("/cowboy-http2-closed", [{"content-type", "image/png"}]) + + port = start_cowboy("/cowboy-http2-closed") + {:ok, mint} = Mint.HTTP2.connect(:http, "localhost", port) + {:ok, mint, request_ref} = Mint.HTTP2.request(mint, "GET", "/", [], nil) + responses = receive_mint_responses(mint) + + assert {:status, request_ref, 200} in responses + assert {:data, request_ref, "partial"} in responses + + assert Enum.any?(responses, fn + {:error, ^request_ref, %Mint.HTTPError{reason: {:server_closed_request, _}}} -> + true + + _ -> + false + end) + + refute {:done, request_ref} in responses + refute :timeout in responses + end + end + + test "resets an incomplete Bandit HTTP/2 stream" do + Mox.set_mox_global(ClientMock) + stream_then_error_mock("/bandit-http2-closed", [{"content-type", "image/png"}]) + + pid = start_bandit("/bandit-http2-closed") + {:ok, {{127, 0, 0, 1}, port}} = ThousandIsland.listener_info(pid) + {:ok, mint} = Mint.HTTP2.connect(:http, "localhost", port) + {:ok, mint, request_ref} = Mint.HTTP2.request(mint, "GET", "/", [], nil) + responses = receive_mint_responses(mint) + + assert {:status, request_ref, 200} in responses + assert {:data, request_ref, "partial"} in responses + + assert Enum.any?(responses, fn + {:error, ^request_ref, + %Mint.HTTPError{reason: {:server_closed_request, :internal_error}}} -> + true + + _ -> + false + end) + + refute {:done, request_ref} in responses + refute :timeout in responses end defp stream_mock(invokes, with_close? \\ false) do @@ -142,9 +460,14 @@ defmodule Pleroma.ReverseProxyTest do test "max_body_length returns error if streaming body more than that option", %{conn: conn} do stream_mock(3, true) - assert capture_log(fn -> - ReverseProxy.call(conn, "/stream-bytes/50", max_body_length: 29) - end) =~ + log = + capture_log(fn -> + assert_raise ReverseProxy.StreamError, fn -> + ReverseProxy.call(conn, "/stream-bytes/50", max_body_length: 29) + end + end) + + assert log =~ "Elixir.Pleroma.ReverseProxy request to /stream-bytes/50 failed while reading/chunking: :body_too_large" end @@ -169,6 +492,19 @@ defmodule Pleroma.ReverseProxyTest do assert Conn.get_resp_header(conn, "content-type") == ["image/png"] assert conn.resp_body == "" end + + test "rejects a malformed content-length", %{conn: conn} do + url = "/head-invalid-content-length" + + expect(ClientMock, :request, fn :head, ^url, _, _, _ -> + {:ok, 200, [{"content-length", "20junk"}]} + end) + + conn = ReverseProxy.call(Map.put(conn, :method, "HEAD"), url) + + assert conn.status == 500 + assert Cachex.get(:failed_proxy_url_cache, url) == {:ok, true} + end end defp error_mock(status) when is_integer(status) do @@ -271,6 +607,28 @@ defmodule Pleroma.ReverseProxyTest do ] end + test "redirects a content sniffing failure before committing the response", %{conn: conn} do + url = "https://example.com/extensionless" + + ClientMock + |> expect(:request, fn :get, ^url, _, _, _ -> + {:ok, 200, [{"content-type", "application/octet-stream"}], %{url: url}} + end) + |> expect(:stream_body, fn _ -> {:error, :closed} end) + |> expect(:close, fn _ -> :ok end) + + conn = + ReverseProxy.call(conn, url, + sniff_content_type: true, + redirect_on_failure: true + ) + + assert conn.status == 302 + assert Conn.get_resp_header(conn, "location") == [url] + assert conn.state == :sent + assert Cachex.get(:failed_proxy_url_cache, url) == {:ok, true} + end + test "preserves a generic content type for a non-image body", %{conn: conn} do body = "not an image" @@ -365,7 +723,73 @@ defmodule Pleroma.ReverseProxyTest do |> expect(:stream_body, fn _ -> :done end) conn = ReverseProxy.call(conn, "/cache") - assert {"cache-control", "public, max-age=1209600, immutable"} in conn.resp_headers + + assert {"cache-control", "public, max-age=1209600, immutable, no-transform"} in conn.resp_headers + end + + test "preserves a valid upstream content-length", %{conn: conn} do + body = "complete body" + + ClientMock + |> expect(:request, fn :get, "/content-length", _, _, _ -> + {:ok, 200, [{"content-length", to_string(byte_size(body))}], %{body: body}} + end) + |> expect(:stream_body, fn %{body: ^body} = client -> + {:ok, body, Map.delete(client, :body)} + end) + |> expect(:stream_body, fn %{} -> :done end) + + conn = ReverseProxy.call(conn, "/content-length") + + assert Conn.get_resp_header(conn, "content-length") == [to_string(byte_size(body))] + end + + test "rejects malformed upstream content-length values" do + Enum.each(["20junk", "-1"], fn content_length -> + url = "/invalid-content-length/#{content_length}" + + ClientMock + |> expect(:request, fn :get, ^url, _, _, _ -> + {:ok, 200, [{"content-length", content_length}], %{url: url}} + end) + |> expect(:close, fn %{url: ^url} -> :ok end) + + conn = ReverseProxy.call(build_conn(), url) + + assert conn.status == 500 + assert Cachex.get(:failed_proxy_url_cache, url) == {:ok, true} + end) + end + + test "rejects conflicting upstream content-length values" do + url = "/conflicting-content-length" + + ClientMock + |> expect(:request, fn :get, ^url, _, _, _ -> + {:ok, 200, [{"content-length", "7"}, {"content-length", "20"}], %{url: url}} + end) + |> stub(:stream_body, fn _ -> :done end) + |> stub(:close, fn _ -> :ok end) + + conn = ReverseProxy.call(build_conn(), url) + + assert conn.status == 500 + assert Cachex.get(:failed_proxy_url_cache, url) == {:ok, true} + end + + test "normalizes equivalent upstream content-length values", %{conn: conn} do + body = "partial" + + complete_stream_mock( + "/duplicate-content-length", + [{"content-length", "7"}, {"content-length", "07"}], + body + ) + + conn = ReverseProxy.call(conn, "/duplicate-content-length") + + assert Conn.get_resp_header(conn, "content-length") == ["7"] + assert conn.resp_body == body end end From 5c85dadb292a5399213f3af50e45f376fa43a79d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?nicole=20miko=C5=82ajczyk?= Date: Tue, 28 Jul 2026 01:06:00 +0200 Subject: [PATCH 76/90] Fix typo in spec MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: nicole mikołajczyk --- changelog.d/typo.skip | 1 + lib/pleroma/user.ex | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/typo.skip diff --git a/changelog.d/typo.skip b/changelog.d/typo.skip new file mode 100644 index 000000000..d249c9bd4 --- /dev/null +++ b/changelog.d/typo.skip @@ -0,0 +1 @@ +Fix typo in spec diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 56c4565d9..3adc90c67 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -327,7 +327,7 @@ defmodule Pleroma.User do @spec visible_for(User.t(), User.t() | nil) :: :visible | :invisible - | :restricted_unauthenticated + | :restrict_unauthenticated | :deactivated | :confirmation_pending def visible_for(user, for_user \\ nil) From 4a47a756bee471791e39b38fc0b24eaf043772f7 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 28 Jul 2026 16:01:48 +0400 Subject: [PATCH 77/90] fix(mastodon-api): optimize account status restrictions --- lib/pleroma/web/activity_pub/activity_pub.ex | 58 +++++++++-------- .../controllers/account_controller_test.exs | 62 ++++++++++++++++++- 2 files changed, 93 insertions(+), 27 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 722bf8b60..363b5f3c3 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -710,6 +710,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do |> Map.put(:type, ["Create", "Announce"]) |> Map.put(:user, reading_user) |> Map.put(:actor_id, user.ap_id) + |> Map.put(:actor_local, user.local) |> Map.put(:pinned_object_ids, Map.keys(user.pinned_objects)) params = @@ -1357,41 +1358,48 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp restrict_unauthenticated(query, _), do: query - defp restrict_unauthenticated_object(query, nil) do - local = Config.restrict_unauthenticated_access?(:activities, :local) - remote = Config.restrict_unauthenticated_access?(:activities, :remote) - local_object_pattern = Pleroma.Web.Endpoint.url() <> "/%" + defp restrict_activity_and_object_origin(query, local?) do + local_object_prefix = Pleroma.Web.Endpoint.url() <> "/" + + # Keeping the correlated origin checks inside CASE prevents PostgreSQL from choosing a + # global bitmap-and-sort plan before applying the account statuses limit. + from([activity, object: object] in query, + where: + fragment( + "CASE WHEN ? = ? AND starts_with((?->>'id'), ?) = ? THEN true ELSE false END", + activity.local, + ^local?, + object.data, + ^local_object_prefix, + ^local? + ) + ) + end + + defp maybe_restrict_unauthenticated( + query, + %{restrict_unauthenticated: true, user: nil, actor_local: actor_local} + ) + when is_boolean(actor_local) do + local_restricted = Config.restrict_unauthenticated_access?(:activities, :local) + remote_restricted = Config.restrict_unauthenticated_access?(:activities, :remote) + actor_restricted = if actor_local, do: local_restricted, else: remote_restricted cond do - local and remote -> - from([object: object] in query, where: false) + actor_restricted -> + from(activity in query, where: false) - local -> - from([object: object] in query, - where: not fragment("(?->>'id') LIKE ?", object.data, ^local_object_pattern) - ) + local_restricted -> + restrict_activity_and_object_origin(query, false) - remote -> - from([object: object] in query, - where: fragment("(?->>'id') LIKE ?", object.data, ^local_object_pattern) - ) + remote_restricted -> + restrict_activity_and_object_origin(query, true) true -> query end end - defp restrict_unauthenticated_object(query, _), do: query - - defp maybe_restrict_unauthenticated( - query, - %{restrict_unauthenticated: true, user: user} - ) do - query - |> restrict_unauthenticated(user) - |> restrict_unauthenticated_object(user) - end - defp maybe_restrict_unauthenticated(query, _), do: query defp restrict_quote_url(query, %{quote_url: quote_url}) do diff --git a/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs index a2ac6382a..e82d795e1 100644 --- a/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs @@ -494,7 +494,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountControllerTest do assert activity_id == activity.id end - test "activity restrictions apply to boosted objects", %{conn: conn} do + test "remote activity restrictions apply to created and boosted objects", %{conn: conn} do clear_config([:restrict_unauthenticated, :activities, :local], false) clear_config([:restrict_unauthenticated, :activities, :remote], true) @@ -502,6 +502,9 @@ defmodule Pleroma.Web.MastodonAPI.AccountControllerTest do remote_user = insert(:user, local: false, domain: "example.com") local_activity = insert(:note_activity, user: local_user) + mixed_create = + insert(:note_activity, local: true, object_local: false, user: local_user) + remote_activity = insert(:note_activity, local: false, object_local: false, user: remote_user) @@ -514,6 +517,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountControllerTest do |> Enum.map(& &1["id"]) assert local_activity.id in status_ids + refute mixed_create.id in status_ids refute announce.id in status_ids authenticated_status_ids = @@ -523,10 +527,11 @@ defmodule Pleroma.Web.MastodonAPI.AccountControllerTest do |> Enum.map(& &1["id"]) assert local_activity.id in authenticated_status_ids + assert mixed_create.id in authenticated_status_ids assert announce.id in authenticated_status_ids end - test "local object restrictions apply to remote boosts", %{conn: conn} do + test "local activity restrictions apply to created and boosted objects", %{conn: conn} do clear_config([:restrict_unauthenticated, :activities, :local], true) clear_config([:restrict_unauthenticated, :activities, :remote], false) @@ -537,6 +542,9 @@ defmodule Pleroma.Web.MastodonAPI.AccountControllerTest do remote_activity = insert(:note_activity, local: false, object_local: false, user: remote_user) + mixed_create = + insert(:note_activity, local: false, object_local: true, user: remote_user) + remote_announce = insert(:announce_activity, note_activity: local_activity, user: remote_user) |> Ecto.Changeset.change(%{ @@ -552,6 +560,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountControllerTest do |> Enum.map(& &1["id"]) assert remote_activity.id in status_ids + refute mixed_create.id in status_ids refute remote_announce.id in status_ids authenticated_status_ids = @@ -561,9 +570,58 @@ defmodule Pleroma.Web.MastodonAPI.AccountControllerTest do |> Enum.map(& &1["id"]) assert remote_activity.id in authenticated_status_ids + assert mixed_create.id in authenticated_status_ids assert remote_announce.id in authenticated_status_ids end + test "restricted local accounts reject activities with mismatched remote origins", %{ + conn: conn + } do + clear_config([:restrict_unauthenticated, :activities, :local], true) + clear_config([:restrict_unauthenticated, :activities, :remote], false) + + local_user = insert(:user) + + mismatched_activity = + insert(:note_activity, local: false, object_local: false, user: local_user) + + assert [] = + build_conn() + |> get("/api/v1/accounts/#{local_user.id}/statuses") + |> json_response_and_validate_schema(200) + + assert [%{"id" => activity_id}] = + conn + |> get("/api/v1/accounts/#{local_user.id}/statuses") + |> json_response_and_validate_schema(200) + + assert activity_id == mismatched_activity.id + end + + test "restricted remote accounts reject activities with mismatched local origins", %{ + conn: conn + } do + clear_config([:restrict_unauthenticated, :activities, :local], false) + clear_config([:restrict_unauthenticated, :activities, :remote], true) + + remote_user = insert(:user, local: false, domain: "example.com") + + mismatched_activity = + insert(:note_activity, local: true, object_local: true, user: remote_user) + + assert [] = + build_conn() + |> get("/api/v1/accounts/#{remote_user.id}/statuses") + |> json_response_and_validate_schema(200) + + assert [%{"id" => activity_id}] = + conn + |> get("/api/v1/accounts/#{remote_user.id}/statuses") + |> json_response_and_validate_schema(200) + + assert activity_id == mismatched_activity.id + end + test "gets a user's statuses without reblogs", %{user: user, conn: conn} do {:ok, %{id: post_id}} = CommonAPI.post(user, %{status: "HI!!!"}) {:ok, _} = CommonAPI.repeat(post_id, user) From 8943ab8140c4bb160b9e3d5df5ee8bb2c3410a41 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Wed, 29 Jul 2026 15:02:16 +0400 Subject: [PATCH 78/90] fix(openrc): preserve migration execution context --- installation/init.d/pleroma | 3 ++- rel/files/installation/init.d/pleroma | 7 ++----- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/installation/init.d/pleroma b/installation/init.d/pleroma index 2cd366140..3b584bd45 100755 --- a/installation/init.d/pleroma +++ b/installation/init.d/pleroma @@ -39,7 +39,8 @@ depend() { start_pre() { ebegin "running migrations if any" - su -s /bin/sh -c '/usr/bin/mix ecto.migrate' -l "${command_user}" + su -s /bin/sh -c "cd \"${directory}\" && exec /usr/bin/mix ecto.migrate" \ + "${command_user%%:*}" eend "$?" } diff --git a/rel/files/installation/init.d/pleroma b/rel/files/installation/init.d/pleroma index 0e117e9b8..2d3977e41 100755 --- a/rel/files/installation/init.d/pleroma +++ b/rel/files/installation/init.d/pleroma @@ -16,10 +16,6 @@ retry="SIGTERM/30/SIGKILL/5" pidfile="/var/run/pleroma.pid" -# Needs OpenRC >= 0.42 -respawn_max=200 -respawn_period=86400 # 1*day - depend() { want nginx need postgresql @@ -27,6 +23,7 @@ depend() { start_pre() { ebegin "running migrations if any" - su -s /bin/sh -c '/opt/pleroma/bin/pleroma_ctl migrate' -l "${command_user}" + su -s /bin/sh -c "cd \"${directory}\" && exec /opt/pleroma/bin/pleroma_ctl migrate" \ + "${command_user%%:*}" eend "$?" } From 253bbb80f5a772aeff06520e677359497bea76dd Mon Sep 17 00:00:00 2001 From: mkljczk Date: Mon, 30 Dec 2024 23:12:12 +0100 Subject: [PATCH 79/90] Add rollback migrations for Rebased Signed-off-by: mkljczk --- changelog.d/rebased-migration.change | 1 + ...12185407_add_email_list_field_to_users.exs | 14 ++++++ ...00_add_activity_assigned_account_index.exs | 13 ++++++ .../20220314220000_add_location_to_users.exs | 11 +++++ .../20220624104914_create_webhooks.exs | 10 +++++ ...ipation_accepted_to_notifications_enum.exs | 45 +++++++++++++++++++ ...220927220033_add_last_move_at_to_users.exs | 11 +++++ ...0221029171353_add_internal_to_webhooks.exs | 11 +++++ .../20221207235104_add_events_index.exs | 17 +++++++ 9 files changed, 133 insertions(+) create mode 100644 changelog.d/rebased-migration.change create mode 100644 priv/repo/optional_migrations/rebased_rollbacks/20210612185407_add_email_list_field_to_users.exs create mode 100644 priv/repo/optional_migrations/rebased_rollbacks/20220225164000_add_activity_assigned_account_index.exs create mode 100644 priv/repo/optional_migrations/rebased_rollbacks/20220314220000_add_location_to_users.exs create mode 100644 priv/repo/optional_migrations/rebased_rollbacks/20220624104914_create_webhooks.exs create mode 100644 priv/repo/optional_migrations/rebased_rollbacks/20220819171321_add_pleroma_participation_accepted_to_notifications_enum.exs create mode 100644 priv/repo/optional_migrations/rebased_rollbacks/20220927220033_add_last_move_at_to_users.exs create mode 100644 priv/repo/optional_migrations/rebased_rollbacks/20221029171353_add_internal_to_webhooks.exs create mode 100644 priv/repo/optional_migrations/rebased_rollbacks/20221207235104_add_events_index.exs diff --git a/changelog.d/rebased-migration.change b/changelog.d/rebased-migration.change new file mode 100644 index 000000000..53ae0951a --- /dev/null +++ b/changelog.d/rebased-migration.change @@ -0,0 +1 @@ +Add optional migrations for rollback from Rebased diff --git a/priv/repo/optional_migrations/rebased_rollbacks/20210612185407_add_email_list_field_to_users.exs b/priv/repo/optional_migrations/rebased_rollbacks/20210612185407_add_email_list_field_to_users.exs new file mode 100644 index 000000000..7dd37a6db --- /dev/null +++ b/priv/repo/optional_migrations/rebased_rollbacks/20210612185407_add_email_list_field_to_users.exs @@ -0,0 +1,14 @@ +# Adapted from Rebased +# https://gitlab.com/soapbox-pub/rebased/-/blob/main/priv/repo/migrations/20210612185407_add_email_list_field_to_users.exs + +defmodule Pleroma.Repo.Migrations.AddEmailListFieldToUsers do + use Ecto.Migration + + def up, do: :noop + + def down do + alter table(:users) do + remove_if_exists(:accepts_email_list, :boolean) + end + end +end diff --git a/priv/repo/optional_migrations/rebased_rollbacks/20220225164000_add_activity_assigned_account_index.exs b/priv/repo/optional_migrations/rebased_rollbacks/20220225164000_add_activity_assigned_account_index.exs new file mode 100644 index 000000000..fd2193a12 --- /dev/null +++ b/priv/repo/optional_migrations/rebased_rollbacks/20220225164000_add_activity_assigned_account_index.exs @@ -0,0 +1,13 @@ +defmodule Pleroma.Repo.Migrations.AddActivityAssignedAccountIndex do + use Ecto.Migration + + def up, do: :noop + + def down do + drop_if_exists( + index(:activities, ["(data->>'assigned_account')"], + name: :activities_assigned_account_index + ) + ) + end +end diff --git a/priv/repo/optional_migrations/rebased_rollbacks/20220314220000_add_location_to_users.exs b/priv/repo/optional_migrations/rebased_rollbacks/20220314220000_add_location_to_users.exs new file mode 100644 index 000000000..d2d317d63 --- /dev/null +++ b/priv/repo/optional_migrations/rebased_rollbacks/20220314220000_add_location_to_users.exs @@ -0,0 +1,11 @@ +defmodule Pleroma.Repo.Migrations.AddLocationToUsers do + use Ecto.Migration + + def up, do: :noop + + def down do + alter table(:users) do + remove_if_exists(:location, :string) + end + end +end diff --git a/priv/repo/optional_migrations/rebased_rollbacks/20220624104914_create_webhooks.exs b/priv/repo/optional_migrations/rebased_rollbacks/20220624104914_create_webhooks.exs new file mode 100644 index 000000000..421dc0107 --- /dev/null +++ b/priv/repo/optional_migrations/rebased_rollbacks/20220624104914_create_webhooks.exs @@ -0,0 +1,10 @@ +defmodule Pleroma.Repo.Migrations.CreateWebhooks do + use Ecto.Migration + + def up, do: :noop + + def down do + drop_if_exists(unique_index(:webhooks, [:url])) + drop_if_exists(table(:webhooks)) + end +end diff --git a/priv/repo/optional_migrations/rebased_rollbacks/20220819171321_add_pleroma_participation_accepted_to_notifications_enum.exs b/priv/repo/optional_migrations/rebased_rollbacks/20220819171321_add_pleroma_participation_accepted_to_notifications_enum.exs new file mode 100644 index 000000000..68d242309 --- /dev/null +++ b/priv/repo/optional_migrations/rebased_rollbacks/20220819171321_add_pleroma_participation_accepted_to_notifications_enum.exs @@ -0,0 +1,45 @@ +defmodule Pleroma.Repo.Migrations.AddPleromaParticipationAcceptedToNotificationsEnum do + use Ecto.Migration + + def up, do: :noop + + def down do + alter table(:notifications) do + modify(:type, :string) + end + + """ + delete from notifications where type in ('pleroma:participation_accepted', 'pleroma:participation_request', 'pleroma:event_reminder', 'pleroma:event_update') + """ + |> execute() + + """ + drop type if exists notification_type + """ + |> execute() + + """ + create type notification_type as enum ( + 'follow', + 'follow_request', + 'mention', + 'move', + 'pleroma:emoji_reaction', + 'pleroma:chat_mention', + 'reblog', + 'favourite', + 'pleroma:report', + 'poll', + 'status', + 'update' + ) + """ + |> execute() + + """ + alter table notifications + alter column type type notification_type using (type::notification_type) + """ + |> execute() + end +end diff --git a/priv/repo/optional_migrations/rebased_rollbacks/20220927220033_add_last_move_at_to_users.exs b/priv/repo/optional_migrations/rebased_rollbacks/20220927220033_add_last_move_at_to_users.exs new file mode 100644 index 000000000..3414d5fb5 --- /dev/null +++ b/priv/repo/optional_migrations/rebased_rollbacks/20220927220033_add_last_move_at_to_users.exs @@ -0,0 +1,11 @@ +defmodule Pleroma.Repo.Migrations.AddLastMoveAtToUsers do + use Ecto.Migration + + def up, do: :noop + + def down do + alter table(:users) do + remove_if_exists(:last_move_at, :naive_datetime) + end + end +end diff --git a/priv/repo/optional_migrations/rebased_rollbacks/20221029171353_add_internal_to_webhooks.exs b/priv/repo/optional_migrations/rebased_rollbacks/20221029171353_add_internal_to_webhooks.exs new file mode 100644 index 000000000..fc85eebb9 --- /dev/null +++ b/priv/repo/optional_migrations/rebased_rollbacks/20221029171353_add_internal_to_webhooks.exs @@ -0,0 +1,11 @@ +defmodule Pleroma.Repo.Migrations.AddInternalToWebhooks do + use Ecto.Migration + + def up, do: :noop + + def down do + alter table(:webhooks) do + remove_if_exists(:internal, :boolean) + end + end +end diff --git a/priv/repo/optional_migrations/rebased_rollbacks/20221207235104_add_events_index.exs b/priv/repo/optional_migrations/rebased_rollbacks/20221207235104_add_events_index.exs new file mode 100644 index 000000000..4e2709277 --- /dev/null +++ b/priv/repo/optional_migrations/rebased_rollbacks/20221207235104_add_events_index.exs @@ -0,0 +1,17 @@ +# Adapted from Rebased +# https://gitlab.com/soapbox-pub/rebased/-/blob/main/priv/repo/migrations/20221207235104_add_events_index.exs + +defmodule Pleroma.Repo.Migrations.AddEventsIndex do + use Ecto.Migration + + def up, do: :noop + + def down do + drop_if_exists( + index(:objects, ["(data->>'type')"], + where: "data->>'type' = 'Event'", + name: :objects_events + ) + ) + end +end From e516c43411b469438283625ae44f0f8cf3bc33f6 Mon Sep 17 00:00:00 2001 From: mkljczk Date: Mon, 17 Feb 2025 18:47:54 +0100 Subject: [PATCH 80/90] Add docs for Rebased migration Signed-off-by: mkljczk --- docs/installation/migrating_from_rebased.md | 43 +++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 docs/installation/migrating_from_rebased.md diff --git a/docs/installation/migrating_from_rebased.md b/docs/installation/migrating_from_rebased.md new file mode 100644 index 000000000..da5b57ffc --- /dev/null +++ b/docs/installation/migrating_from_rebased.md @@ -0,0 +1,43 @@ +# Migrating from Rebased + +## Code migration + +As Rebased only officially supported running from source code, we assume you're running it from source as well. To migrate source from upstream, you need to set the origin repository URL to upstream and pull the changes. + +```bash +git remote set-url origin https://git.pleroma.social/pleroma/pleroma +git pull -r +``` + +Then, install the dependencies and compile as usual: + +```bash +MIX_ENV=prod mix deps.get +MIX_ENV=prod mix compile +``` + +As Rebased recommends using [`asdf` version manager](https://asdf-vm.com/), you might want to either keep using it or switch to system Elixir installation. If you choose the latter, follow Pleroma installation instructions for your distribution and replace systemd service with one provided by Pleroma upstream. + +```bash +sudo cp /opt/pleroma/installation/pleroma.service /etc/systemd/system/pleroma.service +sudo systemctl daemon-reload``` + +Because Rebased doesn't come with a bundled frontend, you most likely have one installed in thhe `instance/static` directory. You can remove it, as Pleroma comes with a bundled frontend. Be sure not to remove other files you might have there, like custom emojis. + +## Database migration + +> Note: While it is not necessary to rollback Rebased-specific migrations because they don't include breaking changes, it is recommended to do so to keep the database clean. You will lose data related to Rebased-specific features, like the configured webhooks or user-defined location. Consider taking a backup. + +To rollback Rebased-specific migrations: + +```bash +MIX_ENV=prod mix ecto.rollback --migrations-path priv/repo/optional_migrations/rebased_rollbacks --all +``` + +Then, just + +```bash +MIX_ENV=prod mix ecto.migrate +``` + +to apply Pleroma database migrations. From 2e59dcb9f6c13ad3de47bc067fcd76ca369b2c31 Mon Sep 17 00:00:00 2001 From: mkljczk Date: Mon, 17 Feb 2025 18:50:51 +0100 Subject: [PATCH 81/90] style Signed-off-by: mkljczk --- docs/installation/migrating_from_rebased.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/installation/migrating_from_rebased.md b/docs/installation/migrating_from_rebased.md index da5b57ffc..94fc8ffe5 100644 --- a/docs/installation/migrating_from_rebased.md +++ b/docs/installation/migrating_from_rebased.md @@ -16,11 +16,12 @@ MIX_ENV=prod mix deps.get MIX_ENV=prod mix compile ``` -As Rebased recommends using [`asdf` version manager](https://asdf-vm.com/), you might want to either keep using it or switch to system Elixir installation. If you choose the latter, follow Pleroma installation instructions for your distribution and replace systemd service with one provided by Pleroma upstream. +As Rebased recommends using [`asdf` version manager](https://asdf-vm.com/), you might want to either keep using it or switch to system Elixir installation. If you choose the latter, follow Pleroma installation instructions for your distribution and replace the Pleroma systemd service with one provided by Pleroma upstream. ```bash sudo cp /opt/pleroma/installation/pleroma.service /etc/systemd/system/pleroma.service -sudo systemctl daemon-reload``` +sudo systemctl daemon-reload +``` Because Rebased doesn't come with a bundled frontend, you most likely have one installed in thhe `instance/static` directory. You can remove it, as Pleroma comes with a bundled frontend. Be sure not to remove other files you might have there, like custom emojis. From e1e853e8f75c2c5a62bf7b6f6b1e8cb40a9b0e9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?nicole=20miko=C5=82ajczyk?= Date: Wed, 28 Jan 2026 14:24:25 +0100 Subject: [PATCH 82/90] Remove migration merged upstream MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: nicole mikołajczyk --- ...25164000_add_activity_assigned_account_index.exs | 13 ------------- 1 file changed, 13 deletions(-) delete mode 100644 priv/repo/optional_migrations/rebased_rollbacks/20220225164000_add_activity_assigned_account_index.exs diff --git a/priv/repo/optional_migrations/rebased_rollbacks/20220225164000_add_activity_assigned_account_index.exs b/priv/repo/optional_migrations/rebased_rollbacks/20220225164000_add_activity_assigned_account_index.exs deleted file mode 100644 index fd2193a12..000000000 --- a/priv/repo/optional_migrations/rebased_rollbacks/20220225164000_add_activity_assigned_account_index.exs +++ /dev/null @@ -1,13 +0,0 @@ -defmodule Pleroma.Repo.Migrations.AddActivityAssignedAccountIndex do - use Ecto.Migration - - def up, do: :noop - - def down do - drop_if_exists( - index(:activities, ["(data->>'assigned_account')"], - name: :activities_assigned_account_index - ) - ) - end -end From 478b021fbe974ec9f0b2e9517063d76a84440535 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Wed, 29 Jul 2026 15:22:01 +0400 Subject: [PATCH 83/90] fix(migrations): make Rebased rollback safe --- docs/installation/migrating_from_rebased.md | 118 ++++++++++-- .../20220624104914_create_webhooks.exs | 10 - ...0221029171353_add_internal_to_webhooks.exs | 11 -- .../rebased_rollbacks_test.exs | 174 ++++++++++++++++++ 4 files changed, 276 insertions(+), 37 deletions(-) delete mode 100644 priv/repo/optional_migrations/rebased_rollbacks/20220624104914_create_webhooks.exs delete mode 100644 priv/repo/optional_migrations/rebased_rollbacks/20221029171353_add_internal_to_webhooks.exs create mode 100644 test/pleroma/repo/optional_migrations/rebased_rollbacks_test.exs diff --git a/docs/installation/migrating_from_rebased.md b/docs/installation/migrating_from_rebased.md index 94fc8ffe5..ce7f43b0d 100644 --- a/docs/installation/migrating_from_rebased.md +++ b/docs/installation/migrating_from_rebased.md @@ -1,44 +1,130 @@ # Migrating from Rebased -## Code migration +Rebased and Pleroma currently have compatible database schemas, so removing +Rebased-specific database objects is optional. The cleanup described below is +destructive and can take a long time on large databases. -As Rebased only officially supported running from source code, we assume you're running it from source as well. To migrate source from upstream, you need to set the origin repository URL to upstream and pull the changes. +This guide requires a Pleroma release that contains this document and native +webhook support. If you are reading the development version of this guide +before those changes reach `stable`, wait for the corresponding stable release. + +## Prepare and back up + +Plan a maintenance window and read the [backup guide](../administration/backup.md) +before starting. + +Stop Rebased and confirm it is stopped before making the backup. Omit the +backup guide's final restart step and leave Rebased stopped: ```bash -git remote set-url origin https://git.pleroma.social/pleroma/pleroma -git pull -r +sudo systemctl stop pleroma +sudo systemctl is-active pleroma ``` -Then, install the dependencies and compile as usual: +Make a verified database backup and a copy of the Rebased installation, +including its configuration, uploads, and `instance` directory. Also save the +current systemd unit if you plan to replace it. + +## Install Pleroma + +Do not use `git pull --rebase` to change an existing Rebased checkout into a +Pleroma checkout. The forks use different branches, and rebasing can replay +Rebased-specific commits over Pleroma. Move the old checkout aside and clone a +clean copy of Pleroma instead. The paths below assume the default source +installation in `/opt/pleroma`: ```bash -MIX_ENV=prod mix deps.get -MIX_ENV=prod mix compile +sudo mv /opt/pleroma /opt/pleroma.rebased +sudo mkdir /opt/pleroma +sudo chown pleroma:pleroma /opt/pleroma +sudo -Hu pleroma git clone -b stable https://git.pleroma.social/pleroma/pleroma /opt/pleroma ``` -As Rebased recommends using [`asdf` version manager](https://asdf-vm.com/), you might want to either keep using it or switch to system Elixir installation. If you choose the latter, follow Pleroma installation instructions for your distribution and replace the Pleroma systemd service with one provided by Pleroma upstream. +Restore the instance-specific files from the old checkout. This usually +includes `config/prod.secret.exs`, `config/prod.exported_from_db.secret.exs` if +present, `uploads`, and `instance`. Diff and restore any other non-stock +configuration files, preserve their ownership, and review them before starting +Pleroma. + +Pleroma requires Elixir 1.15 or newer. Install supported Erlang and Elixir +versions as described in the installation guide for your distribution before +running any Mix command. Do not reuse Rebased's `.tool-versions`. If you keep +using [`asdf`](https://asdf-vm.com/), select supported versions in the +`pleroma` user's home and use the asdf Mix shim for every command below. + +Set `MIX` to the appropriate binary, verify the selected runtime, then install +the dependencies and compile Pleroma: + +```bash +cd /opt/pleroma +MIX=/usr/bin/mix +# If using asdf instead: +# MIX=/var/lib/pleroma/.asdf/shims/mix +sudo -Hu pleroma "$MIX" --version +sudo -Hu pleroma MIX_ENV=prod "$MIX" deps.get +sudo -Hu pleroma MIX_ENV=prod "$MIX" compile +``` + +Compare your existing systemd unit with `installation/pleroma.service`. If you +replace it, review its user, paths, and environment first. Reload systemd after +making changes: ```bash -sudo cp /opt/pleroma/installation/pleroma.service /etc/systemd/system/pleroma.service sudo systemctl daemon-reload ``` -Because Rebased doesn't come with a bundled frontend, you most likely have one installed in thhe `instance/static` directory. You can remove it, as Pleroma comes with a bundled frontend. Be sure not to remove other files you might have there, like custom emojis. +Rebased does not bundle a frontend, so `instance/static` may contain an +installed frontend. Pleroma includes a bundled frontend. You can remove the +separately installed frontend after the migration, but do not remove other +instance files such as custom emojis. ## Database migration -> Note: While it is not necessary to rollback Rebased-specific migrations because they don't include breaking changes, it is recommended to do so to keep the database clean. You will lose data related to Rebased-specific features, like the configured webhooks or user-defined location. Consider taking a backup. +Rebased's extra columns, index, and enum labels do not prevent Pleroma from +starting. Existing notifications with Rebased-only event types are not +supported by Pleroma, however, and can cause notification API errors. Run the +cleanup below, or archive and remove those notification rows separately before +starting Pleroma. Skipping the cleanup preserves them if you need to restore +Rebased. -To rollback Rebased-specific migrations: +Pleroma supports Rebased's webhook schema. The cleanup does not roll back the +shared webhook migrations, so webhook configuration is preserved. + +To perform the optional cleanup, run the rollback while the service is stopped: ```bash -MIX_ENV=prod mix ecto.rollback --migrations-path priv/repo/optional_migrations/rebased_rollbacks --all +MIX=/usr/bin/mix +# If using asdf instead: +# MIX=/var/lib/pleroma/.asdf/shims/mix +sudo -Hu pleroma MIX_ENV=prod "$MIX" ecto.rollback --migrations-path priv/repo/optional_migrations/rebased_rollbacks --all ``` -Then, just +This deletes the Rebased-only `accepts_email_list`, `location`, and +`last_move_at` user fields and the `pleroma:participation_accepted`, +`pleroma:participation_request`, `pleroma:event_reminder`, and +`pleroma:event_update` notifications. It also rewrites the notifications table +and can require substantial time, free disk space, and write-ahead log capacity +on a large instance. + +Apply all pending Pleroma migrations: ```bash -MIX_ENV=prod mix ecto.migrate +sudo -Hu pleroma MIX_ENV=prod "$MIX" ecto.migrate ``` -to apply Pleroma database migrations. +## Start and verify Pleroma + +Start Pleroma and inspect its status and logs: + +```bash +sudo systemctl start pleroma +sudo systemctl status pleroma +sudo journalctl -u pleroma -n 100 --no-pager +``` + +Verify that the instance, uploads, frontend, and webhook configuration work as +expected before deleting `/opt/pleroma.rebased` or the database backup. If the +migration fails, keep the service stopped and restore both the database backup +and the previous checkout before restarting Rebased. If you changed the +systemd unit or runtime setup, restore those too and run +`systemctl daemon-reload` before restarting the service. diff --git a/priv/repo/optional_migrations/rebased_rollbacks/20220624104914_create_webhooks.exs b/priv/repo/optional_migrations/rebased_rollbacks/20220624104914_create_webhooks.exs deleted file mode 100644 index 421dc0107..000000000 --- a/priv/repo/optional_migrations/rebased_rollbacks/20220624104914_create_webhooks.exs +++ /dev/null @@ -1,10 +0,0 @@ -defmodule Pleroma.Repo.Migrations.CreateWebhooks do - use Ecto.Migration - - def up, do: :noop - - def down do - drop_if_exists(unique_index(:webhooks, [:url])) - drop_if_exists(table(:webhooks)) - end -end diff --git a/priv/repo/optional_migrations/rebased_rollbacks/20221029171353_add_internal_to_webhooks.exs b/priv/repo/optional_migrations/rebased_rollbacks/20221029171353_add_internal_to_webhooks.exs deleted file mode 100644 index fc85eebb9..000000000 --- a/priv/repo/optional_migrations/rebased_rollbacks/20221029171353_add_internal_to_webhooks.exs +++ /dev/null @@ -1,11 +0,0 @@ -defmodule Pleroma.Repo.Migrations.AddInternalToWebhooks do - use Ecto.Migration - - def up, do: :noop - - def down do - alter table(:webhooks) do - remove_if_exists(:internal, :boolean) - end - end -end diff --git a/test/pleroma/repo/optional_migrations/rebased_rollbacks_test.exs b/test/pleroma/repo/optional_migrations/rebased_rollbacks_test.exs new file mode 100644 index 000000000..3a26bbfc7 --- /dev/null +++ b/test/pleroma/repo/optional_migrations/rebased_rollbacks_test.exs @@ -0,0 +1,174 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Repo.OptionalMigrations.RebasedRollbacksTest do + use ExUnit.Case, async: false + + alias Ecto.Adapters.SQL + + defmodule MigrationRepo do + use Ecto.Repo, + otp_app: :pleroma, + adapter: Ecto.Adapters.Postgres + end + + @migrations_path "priv/repo/migrations" + @rollbacks_path "priv/repo/optional_migrations/rebased_rollbacks" + + @expected_versions ~w{ + 20210612185407 + 20220314220000 + 20220819171321 + 20220927220033 + 20221207235104 + } + + test "only includes Rebased-specific migration versions" do + rollback_versions = migration_versions(@rollbacks_path) + pleroma_versions = migration_versions(@migrations_path) + + assert rollback_versions == @expected_versions + assert MapSet.disjoint?(MapSet.new(rollback_versions), MapSet.new(pleroma_versions)) + end + + test "rolls back Rebased schema without removing Pleroma data" do + prefix = "rebased_rollbacks_#{Ecto.UUID.generate() |> String.replace("-", "")}" + + repo_config = [ + username: "postgres", + password: "postgres", + database: System.get_env("DB_NAME", "pleroma_test"), + hostname: System.get_env("DB_HOST", "localhost"), + port: String.to_integer(System.get_env("DB_PORT", "5432")), + pool_size: 2, + parameters: [search_path: prefix] + ] + + {:ok, repo} = MigrationRepo.start_link(repo_config) + Process.unlink(repo) + + on_exit(fn -> + query!("DROP SCHEMA IF EXISTS #{prefix} CASCADE") + MigrationRepo.stop() + end) + + query!("CREATE SCHEMA #{prefix}") + + create_rebased_schema() + + expected_rollbacks = + @expected_versions + |> Enum.reverse() + |> Enum.map(&String.to_integer/1) + + assert Ecto.Migrator.run(MigrationRepo, @rollbacks_path, :down, all: true) == + expected_rollbacks + + assert query!("SELECT version FROM schema_migrations ORDER BY version").rows == [ + [20_220_624_104_914], + [20_221_029_171_353] + ] + + assert query!(""" + SELECT column_name + FROM information_schema.columns + WHERE table_schema = current_schema() AND table_name = 'users' + ORDER BY column_name + """).rows == [["id"]] + + assert query!("SELECT type::text FROM notifications ORDER BY type::text").rows == [["status"]] + + assert query!(""" + SELECT enumlabel + FROM pg_enum + JOIN pg_type ON pg_type.oid = enumtypid + JOIN pg_namespace ON pg_namespace.oid = pg_type.typnamespace + WHERE typname = 'notification_type' AND nspname = current_schema() + ORDER BY enumsortorder + """).rows == + Enum.map( + ~w{follow follow_request mention move pleroma:emoji_reaction pleroma:chat_mention reblog favourite pleroma:report poll status update}, + &[&1] + ) + + assert query!("SELECT to_regclass('objects_events')").rows == [[nil]] + + assert query!("SELECT url, secret, internal FROM webhooks").rows == [ + ["https://example.com/webhook", "preserve-me", false] + ] + end + + defp migration_versions(path) do + path + |> Path.join("*.exs") + |> Path.wildcard() + |> Enum.map(fn migration -> + migration + |> Path.basename() + |> String.split("_", parts: 2) + |> hd() + end) + |> Enum.sort() + end + + defp create_rebased_schema do + statements = [ + """ + CREATE TYPE notification_type AS ENUM ( + 'follow', 'follow_request', 'mention', 'move', 'pleroma:emoji_reaction', + 'pleroma:chat_mention', 'reblog', 'favourite', 'pleroma:report', 'poll', + 'status', 'update', 'pleroma:participation_accepted', + 'pleroma:participation_request', 'pleroma:event_reminder', + 'pleroma:event_update' + ) + """, + "CREATE TABLE notifications (id bigserial PRIMARY KEY, type notification_type)", + """ + INSERT INTO notifications (type) VALUES + ('status'), + ('pleroma:participation_accepted'), + ('pleroma:participation_request'), + ('pleroma:event_reminder'), + ('pleroma:event_update') + """, + """ + CREATE TABLE users ( + id bigserial PRIMARY KEY, + accepts_email_list boolean, + location varchar(255), + last_move_at timestamp + ) + """, + "CREATE TABLE objects (id bigserial PRIMARY KEY, data jsonb)", + "CREATE INDEX objects_events ON objects ((data->>'type')) WHERE data->>'type' = 'Event'", + """ + CREATE TABLE webhooks ( + id bigserial PRIMARY KEY, + url varchar(255) NOT NULL, + events varchar[] NOT NULL DEFAULT '{}', + secret varchar(255) NOT NULL DEFAULT '', + enabled boolean NOT NULL DEFAULT true, + internal boolean NOT NULL DEFAULT false + ) + """, + "CREATE UNIQUE INDEX webhooks_url_index ON webhooks (url)", + "INSERT INTO webhooks (url, secret) VALUES ('https://example.com/webhook', 'preserve-me')", + "CREATE TABLE schema_migrations (version bigint PRIMARY KEY, inserted_at timestamp(0))", + """ + INSERT INTO schema_migrations (version, inserted_at) VALUES + (20210612185407, NOW()), + (20220314220000, NOW()), + (20220624104914, NOW()), + (20220819171321, NOW()), + (20220927220033, NOW()), + (20221029171353, NOW()), + (20221207235104, NOW()) + """ + ] + + Enum.each(statements, &query!/1) + end + + defp query!(statement), do: SQL.query!(MigrationRepo, statement, []) +end From 2876fed876045ded1d86812a550bd0feccd72226 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?nicole=20miko=C5=82ajczyk?= Date: Sun, 2 Aug 2026 22:13:31 +0200 Subject: [PATCH 84/90] Embed notificaiton details for grouped move/reaction/chat/report notification MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: nicole mikołajczyk --- changelog.d/grouped-notifs-embed.skip | 0 .../operations/notification_operation.ex | 21 +++++- .../mastodon_api/views/notification_view.ex | 65 ++++++++++++++++--- .../views/notification_view_test.exs | 35 ++++++++++ 4 files changed, 112 insertions(+), 9 deletions(-) create mode 100644 changelog.d/grouped-notifs-embed.skip diff --git a/changelog.d/grouped-notifs-embed.skip b/changelog.d/grouped-notifs-embed.skip new file mode 100644 index 000000000..e69de29bb diff --git a/lib/pleroma/web/api_spec/operations/notification_operation.ex b/lib/pleroma/web/api_spec/operations/notification_operation.ex index 6c8f7d5e9..740f7645d 100644 --- a/lib/pleroma/web/api_spec/operations/notification_operation.ex +++ b/lib/pleroma/web/api_spec/operations/notification_operation.ex @@ -8,6 +8,7 @@ defmodule Pleroma.Web.ApiSpec.NotificationOperation do alias Pleroma.Web.ApiSpec.Schemas.Account alias Pleroma.Web.ApiSpec.Schemas.ApiError alias Pleroma.Web.ApiSpec.Schemas.BooleanLike + alias Pleroma.Web.ApiSpec.Schemas.ChatMessage alias Pleroma.Web.ApiSpec.Schemas.Status alias Pleroma.Web.ApiSpec.Schemas.VisibilityScope @@ -304,7 +305,25 @@ defmodule Pleroma.Web.ApiSpec.NotificationOperation do page_max_id: %Schema{type: :string, nullable: true}, latest_page_notification_at: %Schema{type: :string, format: :"date-time", nullable: true}, sample_account_ids: %Schema{type: :array, items: %Schema{type: :string}}, - status_id: %Schema{type: :string, nullable: true} + status_id: %Schema{type: :string, nullable: true}, + target_id: %Schema{ + type: :string, + description: "ID of the target account for a move notification" + }, + emoji: %Schema{ + type: :string, + description: "Emoji used for an emoji reaction notification" + }, + emoji_url: %Schema{ + type: :string, + nullable: true, + description: "URL of the custom emoji used for an emoji reaction notification" + }, + chat_message: ChatMessage, + report: %Schema{ + type: :object, + description: "Report embedded in a report notification" + } }, required: [ :group_key, diff --git a/lib/pleroma/web/mastodon_api/views/notification_view.ex b/lib/pleroma/web/mastodon_api/views/notification_view.ex index 33d4b2924..eadfac4a0 100644 --- a/lib/pleroma/web/mastodon_api/views/notification_view.ex +++ b/lib/pleroma/web/mastodon_api/views/notification_view.ex @@ -101,13 +101,13 @@ defmodule Pleroma.Web.MastodonAPI.NotificationView do |> Enum.filter(& &1) |> Enum.uniq_by(& &1[:id]) - actors = + accounts = notification_groups |> List.flatten() - |> notification_actors() + |> notification_accounts() %{ - accounts: AccountView.render("index.json", %{users: actors, for: reading_user}), + accounts: AccountView.render("index.json", %{users: accounts, for: reading_user}), statuses: statuses, notification_groups: Enum.map( @@ -191,7 +191,7 @@ defmodule Pleroma.Web.MastodonAPI.NotificationView do defp render_group( [%Notification{} = notification | _] = notifications, - _reading_user, + reading_user, grouped_types, notification_group_counts, notification_group_bounds, @@ -237,13 +237,49 @@ defmodule Pleroma.Web.MastodonAPI.NotificationView do response end - if status_activity do - Map.put(response, :status_id, to_string(status_activity.id)) - else - response + response = + if status_activity do + Map.put(response, :status_id, to_string(status_activity.id)) + else + response + end + + put_group_details(response, notification, reading_user) + end + + defp put_group_details(response, %Notification{type: "move", activity: activity}, _reading_user) do + case User.get_cached_by_ap_id(activity.data["target"]) do + %User{id: id} -> Map.put(response, :target_id, to_string(id)) + _ -> response end end + defp put_group_details( + response, + %Notification{type: "pleroma:emoji_reaction", activity: activity}, + _reading_user + ) do + put_emoji(response, activity) + end + + defp put_group_details( + response, + %Notification{type: "pleroma:chat_mention", activity: activity}, + reading_user + ) do + put_chat_message(response, activity, reading_user, %{}) + end + + defp put_group_details( + response, + %Notification{type: "pleroma:report", activity: activity}, + _reading_user + ) do + put_report(response, activity) + end + + defp put_group_details(response, _notification, _reading_user), do: response + defp status_activity_for_group([%Notification{} = notification | _], grouped_types) do status_activity_for(notification, grouped_types) end @@ -287,6 +323,19 @@ defmodule Pleroma.Web.MastodonAPI.NotificationView do |> Enum.uniq_by(& &1.id) end + defp notification_accounts(notifications) do + move_targets = + notifications + |> Enum.filter(&(&1.type == "move")) + |> Enum.map(&User.get_cached_by_ap_id(&1.activity.data["target"])) + |> Enum.filter(& &1) + + notifications + |> notification_actors() + |> Kernel.++(move_targets) + |> Enum.uniq_by(& &1.id) + end + defp put_emoji(response, activity) do response |> Map.put(:emoji, activity.data["content"]) diff --git a/test/pleroma/web/mastodon_api/views/notification_view_test.exs b/test/pleroma/web/mastodon_api/views/notification_view_test.exs index 2d0801589..e6fb31e28 100644 --- a/test/pleroma/web/mastodon_api/views/notification_view_test.exs +++ b/test/pleroma/web/mastodon_api/views/notification_view_test.exs @@ -65,6 +65,15 @@ defmodule Pleroma.Web.MastodonAPI.NotificationViewTest do } test_notifications_rendering([notification], recipient, [expected]) + + grouped = + NotificationView.render("grouped_index.json", %{ + notifications: [notification], + for: recipient + }) + + assert [group] = grouped.notification_groups + assert group.chat_message == expected.chat_message end test "Mention notification" do @@ -182,6 +191,16 @@ defmodule Pleroma.Web.MastodonAPI.NotificationViewTest do } test_notifications_rendering([notification], follower, [expected]) + + grouped = + NotificationView.render("grouped_index.json", %{ + notifications: [notification], + for: follower + }) + + assert [group] = grouped.notification_groups + assert group.target_id == new_user.id + assert Enum.any?(grouped.accounts, &(&1.id == new_user.id)) end test "EmojiReact notification" do @@ -210,6 +229,13 @@ defmodule Pleroma.Web.MastodonAPI.NotificationViewTest do } test_notifications_rendering([notification], user, [expected]) + + grouped = + NotificationView.render("grouped_index.json", %{notifications: [notification], for: user}) + + assert [group] = grouped.notification_groups + assert group.emoji == expected.emoji + assert group.emoji_url == expected.emoji_url end test "EmojiReact custom emoji notification" do @@ -295,6 +321,15 @@ defmodule Pleroma.Web.MastodonAPI.NotificationViewTest do } test_notifications_rendering([notification], moderator_user, [expected]) + + grouped = + NotificationView.render("grouped_index.json", %{ + notifications: [notification], + for: moderator_user + }) + + assert [group] = grouped.notification_groups + assert group.report == expected.report end test "Edit notification" do From f672ab337c53fcb7ef50baaf5d03c40c24e2aed9 Mon Sep 17 00:00:00 2001 From: Phantasm Date: Thu, 13 Aug 2026 18:11:18 +0200 Subject: [PATCH 85/90] Revert "Merge pull request 'MediaProxy: detect images when there's no file extension' (#7933) from mediaproxy-content-type into develop" A partial revert, keeping the updated @inline_content_types list and cosmetic changes. This reverts commit 4fe567064a9ea8ff4cc6c56e0f89caad71275a99, reversing changes made to 9b8aa2e2ee225dc69a528e7063be3620a79f9e33. --- changelog.d/mediaproxy-content-type.fix | 1 - lib/pleroma/reverse_proxy.ex | 180 +++--------------- .../web/mastodon_api/views/status_view.ex | 1 - .../web/media_proxy/media_proxy_controller.ex | 1 - test/pleroma/reverse_proxy_test.exs | 112 ----------- .../mastodon_api/views/status_view_test.exs | 17 -- .../media_proxy_controller_test.exs | 5 +- 7 files changed, 23 insertions(+), 294 deletions(-) delete mode 100644 changelog.d/mediaproxy-content-type.fix diff --git a/changelog.d/mediaproxy-content-type.fix b/changelog.d/mediaproxy-content-type.fix deleted file mode 100644 index f36dd6283..000000000 --- a/changelog.d/mediaproxy-content-type.fix +++ /dev/null @@ -1 +0,0 @@ -Detect image content types for extensionless MediaProxy responses. diff --git a/lib/pleroma/reverse_proxy.ex b/lib/pleroma/reverse_proxy.ex index 12e48eadd..ee20303a3 100644 --- a/lib/pleroma/reverse_proxy.ex +++ b/lib/pleroma/reverse_proxy.ex @@ -17,7 +17,6 @@ defmodule Pleroma.ReverseProxy do @max_read_duration :timer.seconds(30) @max_body_length :infinity @failed_request_ttl :timer.seconds(60) - @sniff_bytes 8 * 1024 @methods ~w(GET HEAD) @allowed_mime_types Pleroma.Config.get([Pleroma.Upload, :allowed_mime_types], []) @@ -70,12 +69,7 @@ defmodule Pleroma.ReverseProxy do * `inline_content_types`: * `true` will not alter `content-disposition` (up to the upstream), * `false` will add `content-disposition: attachment` to any request, - * a list of whitelisted content types for which `content-disposition: inline` - is always set (overriding any upstream header) so the media can be embedded - in pages; the filename is derived from the content type - - * `sniff_content_type` (default `false`): detects an image MIME type from the first - response chunk when the upstream type is missing or `application/octet-stream`. + * a list of whitelisted content types * `req_headers`, `resp_headers` additional headers. @@ -112,7 +106,6 @@ defmodule Pleroma.ReverseProxy do | {:req_headers, [{String.t(), String.t()}]} | {:resp_headers, [{String.t(), String.t()}]} | {:inline_content_types, boolean() | list(String.t())} - | {:sniff_content_type, boolean()} | {:redirect_on_failure, boolean()} @spec call(Plug.Conn.t(), String.t(), list(option())) :: Plug.Conn.t() @@ -228,30 +221,11 @@ defmodule Pleroma.ReverseProxy do defp response(conn, client, url, status, headers, opts) do Logger.debug("#{__MODULE__} #{status} #{url} #{inspect(headers)}") - case maybe_prefetch_for_content_type(headers, client, opts) do - {_headers, client, {:error, {:upstream, error}}} -> - Logger.warning( - "#{__MODULE__} request to #{url} failed before streaming: #{inspect(error)}" - ) - - client().close(client) - track_failed_url(url, error, opts) - - conn - |> error_or_redirect(url, 500, "Request failed", opts) - |> halt() - - {headers, client, prefetched} -> - stream_response(conn, client, url, status, headers, opts, prefetched) - end - end - - defp stream_response(conn, client, url, status, headers, opts, prefetched) do result = conn |> put_resp_headers(build_resp_headers(headers, opts)) |> send_chunked(status) - |> chunk_reply(client, opts, prefetched) + |> chunk_reply(client, opts) case result do {:ok, conn} -> @@ -272,29 +246,18 @@ defmodule Pleroma.ReverseProxy do end end - defp chunk_reply(conn, client, opts, :none) do + defp chunk_reply(conn, client, opts) do chunk_reply(conn, client, opts, 0, 0) end - defp chunk_reply(conn, _client, _opts, :done), do: {:ok, conn} - defp chunk_reply(conn, _client, _opts, {:error, error}), do: {:error, error, conn} - - defp chunk_reply(conn, client, opts, {:ok, data, duration}) do - with :ok <- - body_size_constraint( - byte_size(data), - Keyword.get(opts, :max_body_length, @max_body_length) - ), - {:ok, conn} <- write_chunk(conn, data) do - chunk_reply(conn, client, opts, byte_size(data), duration) - else - {:error, {:downstream, error}} -> {:error, {:downstream, error}, conn} - {:error, error} -> {:error, {:upstream, error}, conn} - end - end - defp chunk_reply(conn, client, opts, sent_so_far, duration) do - with {:ok, data, client, duration} <- read_chunk(client, duration, opts), + with {:ok, duration} <- + check_read_duration( + duration, + Keyword.get(opts, :max_read_duration, @max_read_duration) + ), + {:ok, data, client} <- client().stream_body(client), + {:ok, duration} <- increase_read_duration(duration), sent_so_far = sent_so_far + byte_size(data), :ok <- body_size_constraint( @@ -310,24 +273,6 @@ defmodule Pleroma.ReverseProxy do end end - defp read_chunk(client, duration, opts) do - result = - with {:ok, timer} <- - check_read_duration( - duration, - Keyword.get(opts, :max_read_duration, @max_read_duration) - ), - {:ok, data, client} <- client().stream_body(client), - {:ok, duration} <- increase_read_duration(timer) do - {:ok, data, client, duration} - end - - case result do - {:error, error} -> {:error, {:upstream, error}} - result -> result - end - end - defp write_chunk(conn, data) do case chunk(conn, data) do {:error, error} -> {:error, {:downstream, error}} @@ -335,54 +280,6 @@ defmodule Pleroma.ReverseProxy do end end - defp maybe_prefetch_for_content_type(headers, client, opts) do - if Keyword.get(opts, :sniff_content_type, false) and generic_content_type?(headers) do - case read_chunk(client, 0, opts) do - {:ok, data, client, duration} -> - {maybe_put_image_content_type(headers, data), client, {:ok, data, duration}} - - :done -> - {headers, client, :done} - - {:error, error} -> - {headers, client, {:error, error}} - end - else - {headers, client, :none} - end - end - - defp generic_content_type?(headers) do - headers - |> get_content_type() - |> String.trim() - |> String.downcase() - |> then(&(&1 in ["", "application/octet-stream"])) - end - - defp maybe_put_image_content_type(headers, data) do - with false <- data == "", - {:ok, %{mime_type: "image/" <> _ = content_type}} <- - Majic.perform({:bytes, binary_part(data, 0, min(byte_size(data), @sniff_bytes))}, - pool: Pleroma.MajicPool - ) do - [ - {"content-type", content_type} - | Enum.reject(headers, fn {key, _} -> key == "content-type" end) - ] - else - _ -> headers - end - rescue - error -> - Logger.debug("#{__MODULE__}: content-type sniffing failed: #{Exception.message(error)}") - headers - catch - kind, reason -> - Logger.debug("#{__MODULE__}: content-type sniffing failed: #{kind}: #{inspect(reason)}") - headers - end - defp head_response(conn, url, code, headers, opts) do Logger.debug("#{__MODULE__} #{code} #{url} #{inspect(headers)}") @@ -430,33 +327,16 @@ defmodule Pleroma.ReverseProxy do |> build_req_range_or_encoding_header(opts) |> build_req_user_agent_header(opts) |> merge_headers(Keyword.get(opts, :req_headers, [])) - |> maybe_force_identity_encoding(opts) - end - - defp merge_headers(headers, extra_headers) do - Enum.reduce(extra_headers, headers, fn {key, value}, headers -> - List.keystore(headers, String.downcase(key), 0, {String.downcase(key), value}) - end) - end - - defp maybe_force_identity_encoding(headers, opts) do - if Keyword.get(opts, :sniff_content_type, false) do - List.keystore(headers, "accept-encoding", 0, {"accept-encoding", "identity"}) - else - headers - end end # Disable content-encoding if any @range_headers are requested (see #1823). defp build_req_range_or_encoding_header(headers, _opts) do range? = Enum.any?(headers, fn {header, _} -> Enum.member?(@range_headers, header) end) - cond do - range? && List.keymember?(headers, "accept-encoding", 0) -> - List.keydelete(headers, "accept-encoding", 0) - - true -> - headers + if range? && List.keymember?(headers, "accept-encoding", 0) do + List.keydelete(headers, "accept-encoding", 0) + else + headers end end @@ -479,6 +359,12 @@ defmodule Pleroma.ReverseProxy do |> ensure_no_transform() end + defp merge_headers(headers, extra_headers) do + Enum.reduce(extra_headers, headers, fn {key, value}, headers -> + List.keystore(headers, String.downcase(key), 0, {String.downcase(key), value}) + end) + end + defp ensure_no_transform(headers) do {_, cache_control} = List.keyfind(headers, "cache-control", 0, {"cache-control", @default_cache_control_header}) @@ -568,20 +454,7 @@ defmodule Pleroma.ReverseProxy do replace_header(headers, "content-disposition", disposition) else - if opt == true do - headers - else - name = inline_filename(content_type) - - disposition = - if name do - "inline; filename=\"#{name}\"" - else - "inline" - end - - replace_header(headers, "content-disposition", disposition) - end + headers end end @@ -589,13 +462,6 @@ defmodule Pleroma.ReverseProxy do [{key, value} | Enum.reject(headers, fn {header, _} -> header == key end)] end - defp inline_filename(content_type) do - case MIME.extensions(content_type) do - [ext | _] when ext != "" -> "inline.#{ext}" - _ -> nil - end - end - defp header_length_constraint(headers, limit) do lengths = for {"content-length", value} <- headers, @@ -640,9 +506,7 @@ defmodule Pleroma.ReverseProxy do end end - defp check_read_duration(_, _), do: {:ok, :no_duration_limit} - - defp increase_read_duration(:no_duration_limit), do: {:ok, :no_duration_limit} + defp check_read_duration(_, _), do: {:ok, :no_duration_limit, :no_duration_limit} defp increase_read_duration({previous_duration, started}) when is_integer(previous_duration) and is_integer(started) do diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 356e8fce5..4625be47c 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -631,7 +631,6 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do String.contains?(media_type, "image") -> "image" String.contains?(media_type, "video") -> "video" String.contains?(media_type, "audio") -> "audio" - attachment["type"] == "Image" -> "image" true -> "unknown" end diff --git a/lib/pleroma/web/media_proxy/media_proxy_controller.ex b/lib/pleroma/web/media_proxy/media_proxy_controller.ex index 93f692008..a0aafc32e 100644 --- a/lib/pleroma/web/media_proxy/media_proxy_controller.ex +++ b/lib/pleroma/web/media_proxy/media_proxy_controller.ex @@ -208,7 +208,6 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do defp media_proxy_opts do Config.get([:media_proxy, :proxy_opts], []) - |> Keyword.put_new(:sniff_content_type, true) end defp sandbox(conn, _params) do diff --git a/test/pleroma/reverse_proxy_test.exs b/test/pleroma/reverse_proxy_test.exs index bf0a809c4..156b7d7ed 100644 --- a/test/pleroma/reverse_proxy_test.exs +++ b/test/pleroma/reverse_proxy_test.exs @@ -579,56 +579,6 @@ defmodule Pleroma.ReverseProxyTest do assert Conn.get_resp_header(conn, "content-type") == ["application/octet-stream"] end - test "sniffs and preserves an image body with a generic content type", %{conn: conn} do - body = File.read!("test/fixtures/image.jpg") - - ClientMock - |> expect(:request, fn :get, "/extensionless", headers, _, _ -> - assert {"accept-encoding", "identity"} in headers - {:ok, 200, [{"content-type", "application/octet-stream"}], %{body: body}} - end) - |> expect(:stream_body, fn %{body: ^body} = client -> - {:ok, body, Map.delete(client, :body)} - end) - |> expect(:stream_body, fn %{} -> :done end) - - conn = - ReverseProxy.call(conn, "/extensionless", - sniff_content_type: true, - max_read_duration: :infinity, - req_headers: [{"accept-encoding", "gzip"}] - ) - - assert conn.resp_body == body - assert Conn.get_resp_header(conn, "content-type") == ["image/jpeg"] - - assert Conn.get_resp_header(conn, "content-disposition") == [ - "inline; filename=\"inline.jpg\"" - ] - end - - test "redirects a content sniffing failure before committing the response", %{conn: conn} do - url = "https://example.com/extensionless" - - ClientMock - |> expect(:request, fn :get, ^url, _, _, _ -> - {:ok, 200, [{"content-type", "application/octet-stream"}], %{url: url}} - end) - |> expect(:stream_body, fn _ -> {:error, :closed} end) - |> expect(:close, fn _ -> :ok end) - - conn = - ReverseProxy.call(conn, url, - sniff_content_type: true, - redirect_on_failure: true - ) - - assert conn.status == 302 - assert Conn.get_resp_header(conn, "location") == [url] - assert conn.state == :sent - assert Cachex.get(:failed_proxy_url_cache, url) == {:ok, true} - end - test "preserves a generic content type for a non-image body", %{conn: conn} do body = "not an image" @@ -823,24 +773,6 @@ defmodule Pleroma.ReverseProxyTest do conn = ReverseProxy.call(conn, "/disposition") assert {"content-type", "image/gif"} in conn.resp_headers - assert {"content-disposition", "inline; filename=\"inline.gif\""} in conn.resp_headers - end - - test "forces inline for inline content types overriding upstream attachment", %{ - conn: conn - } do - disposition_headers_mock([ - {"content-type", "image/png"}, - {"content-disposition", "attachment; filename=\"filename.png\""}, - {"content-disposition", "attachment; filename=\"duplicate.png\""}, - {"content-length", "0"} - ]) - - conn = ReverseProxy.call(conn, "/disposition") - - [disposition] = Conn.get_resp_header(conn, "content-disposition") - assert String.starts_with?(disposition, "inline") - refute String.starts_with?(disposition, "attachment") end test "with content-disposition header", %{conn: conn} do @@ -853,50 +785,6 @@ defmodule Pleroma.ReverseProxyTest do assert {"content-disposition", "attachment; filename=\"filename.jpg\""} in conn.resp_headers end - - test "with inline_content_types: true leaves upstream headers untouched", %{ - conn: conn - } do - # opt == true: the proxy must not synthesise or rewrite content-disposition. - disposition_headers_mock([ - {"content-type", "image/png"}, - {"content-disposition", "attachment; filename=\"upstream.png\""}, - {"content-length", "0"} - ]) - - conn = ReverseProxy.call(conn, "/disposition", inline_content_types: true) - - assert {"content-disposition", "attachment; filename=\"upstream.png\""} in conn.resp_headers - end - - test "forces bare inline for a whitelisted type with no MIME extension", %{ - conn: conn - } do - # image/x-foo-bar has no entry in MIME's database, so inline_filename/1 - # returns nil and the disposition should be the bare token "inline". - disposition_headers_mock([ - {"content-type", "image/x-foo-bar"}, - {"content-length", "0"} - ]) - - conn = ReverseProxy.call(conn, "/disposition", inline_content_types: ["image/x-foo-bar"]) - - [disposition] = Conn.get_resp_header(conn, "content-disposition") - assert disposition == "inline" - end - - test "serves modern browser image types inline", %{conn: conn} do - disposition_headers_mock([ - {"content-type", "image/webp"}, - {"content-length", "0"} - ]) - - conn = ReverseProxy.call(conn, "/disposition") - - assert Conn.get_resp_header(conn, "content-disposition") == [ - "inline; filename=\"inline.webp\"" - ] - end end describe "content-type sanitisation" do diff --git a/test/pleroma/web/mastodon_api/views/status_view_test.exs b/test/pleroma/web/mastodon_api/views/status_view_test.exs index 73adb196a..f7b815ed7 100644 --- a/test/pleroma/web/mastodon_api/views/status_view_test.exs +++ b/test/pleroma/web/mastodon_api/views/status_view_test.exs @@ -668,23 +668,6 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do assert expected == StatusView.render("attachment.json", %{attachment: object}) assert_schema(expected, "Attachment", api_spec) end - - test "uses the ActivityStreams image type when the MIME type is generic" do - object = %{ - "type" => "Image", - "url" => [ - %{ - "mediaType" => "application/octet-stream", - "href" => "https://example.com/extensionless" - } - ] - } - - attachment = StatusView.render("attachment.json", %{attachment: object}) - - assert attachment.type == "image" - assert attachment.pleroma.mime_type == "application/octet-stream" - end end test "put the url advertised in the Activity in to the url attribute" do diff --git a/test/pleroma/web/media_proxy/media_proxy_controller_test.exs b/test/pleroma/web/media_proxy/media_proxy_controller_test.exs index 800159c21..f7e52483c 100644 --- a/test/pleroma/web/media_proxy/media_proxy_controller_test.exs +++ b/test/pleroma/web/media_proxy/media_proxy_controller_test.exs @@ -71,10 +71,7 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyControllerTest do test "it performs ReverseProxy.call with valid signature", %{conn: conn, url: url} do with_mock Pleroma.ReverseProxy, - call: fn _conn, _url, opts -> - assert opts[:sniff_content_type] - %Conn{status: :success} - end do + call: fn _conn, _url, _opts -> %Conn{status: :success} end do assert %Conn{status: :success} = get(conn, url) end end From 2dda6cb52ad4814fa13dd565bd29dcf59187643a Mon Sep 17 00:00:00 2001 From: Phantasm Date: Thu, 13 Aug 2026 23:26:44 +0200 Subject: [PATCH 86/90] Docs: Update OTP bundle links left behind in the Forgejo migration --- docs/installation/migrating_from_source_otp_en.md | 2 +- docs/installation/otp_en.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/installation/migrating_from_source_otp_en.md b/docs/installation/migrating_from_source_otp_en.md index 798862566..548f10f47 100644 --- a/docs/installation/migrating_from_source_otp_en.md +++ b/docs/installation/migrating_from_source_otp_en.md @@ -87,7 +87,7 @@ export FLAVOUR="amd64-musl" # Clone the release build into a temporary directory and unpack it # Replace `stable` with `unstable` if you want to run the unstable branch sudo -Hu pleroma " -curl 'https://git.pleroma.social/api/v4/projects/2/jobs/artifacts/stable/download?job=$FLAVOUR' -o /tmp/pleroma.zip +curl 'https://git.pleroma.social/api/packages/pleroma/generic/pleroma-otp-stable-$FLAVOUR/latest/pleroma.zip' -o /tmp/pleroma.zip unzip /tmp/pleroma.zip -d /tmp/ " diff --git a/docs/installation/otp_en.md b/docs/installation/otp_en.md index 5fc1db21e..7410afe79 100644 --- a/docs/installation/otp_en.md +++ b/docs/installation/otp_en.md @@ -140,7 +140,7 @@ export FLAVOUR="amd64-musl" # Clone the release build into a temporary directory and unpack it sudo -Hu pleroma " -curl 'https://git.pleroma.social/api/v4/projects/2/jobs/artifacts/stable/download?job=$FLAVOUR' -o /tmp/pleroma.zip +curl 'https://git.pleroma.social/api/packages/pleroma/generic/pleroma-otp-stable-$FLAVOUR/latest/pleroma.zip' -o /tmp/pleroma.zip unzip /tmp/pleroma.zip -d /tmp/ " From 5ecc93644423266645410fa756239afc41486b90 Mon Sep 17 00:00:00 2001 From: Phantasm Date: Thu, 13 Aug 2026 23:37:34 +0200 Subject: [PATCH 87/90] Docs: split wall of install commands, fix downloading OTP bundle Bundle could not be downloaded, because sudo does not cope with the previous syntax that double quoted the whole command, and the download link was single quoted. --- changelog.d/docs-otp-install-links.fix | 1 + .../migrating_from_source_otp_en.md | 52 +++++++---- docs/installation/otp_en.md | 86 +++++++++++++------ 3 files changed, 93 insertions(+), 46 deletions(-) create mode 100644 changelog.d/docs-otp-install-links.fix diff --git a/changelog.d/docs-otp-install-links.fix b/changelog.d/docs-otp-install-links.fix new file mode 100644 index 000000000..28d504bc0 --- /dev/null +++ b/changelog.d/docs-otp-install-links.fix @@ -0,0 +1 @@ +Docs: Fixed OTP install links in documentation, broke up wall of install commands into smaller chunks, fixed sudo commands execution diff --git a/docs/installation/migrating_from_source_otp_en.md b/docs/installation/migrating_from_source_otp_en.md index 548f10f47..450d33804 100644 --- a/docs/installation/migrating_from_source_otp_en.md +++ b/docs/installation/migrating_from_source_otp_en.md @@ -76,35 +76,51 @@ $EDITOR /etc/pleroma/config.exs ``` ## Installing the release Before proceeding, get the flavour from [Detecting flavour](otp_en.md#detecting-flavour) section in OTP installation guide. + + +Delete all files in pleroma user's directory: ```sh -# Delete all files in pleroma user's directory rm -r ~pleroma/* +``` -# Set the flavour environment variable to the string you got in Detecting flavour section. -# For example if the flavour is `amd64-musl` the command will be +Set the flavour environment variable to the string you got in Detecting flavour section. +For example if the flavour is `amd64-musl` the command will be: + +```sh export FLAVOUR="amd64-musl" +``` -# Clone the release build into a temporary directory and unpack it -# Replace `stable` with `unstable` if you want to run the unstable branch -sudo -Hu pleroma " -curl 'https://git.pleroma.social/api/packages/pleroma/generic/pleroma-otp-stable-$FLAVOUR/latest/pleroma.zip' -o /tmp/pleroma.zip -unzip /tmp/pleroma.zip -d /tmp/ -" +Clone the release build into a temporary directory and unpack it. +Replace `stable` with `develop`, if you want to run the develop branch. -# Move the release to the home directory and delete temporary files -sudo -Hu pleroma " -mv /tmp/release/* ~pleroma/ -rmdir /tmp/release -rm /tmp/pleroma.zip -" +```sh +sudo -Hu pleroma curl "https://git.pleroma.social/api/packages/pleroma/generic/pleroma-otp-stable-$FLAVOUR/latest/pleroma.zip" -o /tmp/pleroma.zip +sudo -Hu pleroma unzip /tmp/pleroma.zip -d /tmp/ +``` -# Start the instance to verify that everything is working as expected +Move the release to the home directory and delete temporary files: + +```sh +sudo -Hu pleroma mv /tmp/release/* ~pleroma/ +sudo -Hu pleroma rmdir /tmp/release +sudo -Hu pleroma rm /tmp/pleroma.zip +``` + +Start the instance to verify that everything is working as expected: + +```sh sudo -Hu pleroma "./bin/pleroma daemon" +``` -# Wait for about 20 seconds and query the instance endpoint, if it shows your uri, name and email correctly, you are configured correctly +Wait for about 20 seconds and query the instance endpoint, if it shows your uri, name and email correctly, you are configured correctly: + +```sh sleep 20 && curl http://localhost:4000/api/v1/instance +``` -# Stop the instance +Stop the instance: + +```sh sudo -Hu pleroma "./bin/pleroma stop" ``` diff --git a/docs/installation/otp_en.md b/docs/installation/otp_en.md index 7410afe79..0f296a9f6 100644 --- a/docs/installation/otp_en.md +++ b/docs/installation/otp_en.md @@ -130,60 +130,90 @@ Restart PostgreSQL to apply configuration changes: ``` ### Installing Pleroma +Create a Pleroma user: + ```sh -# Create a Pleroma user adduser --system --shell /bin/false --home /opt/pleroma pleroma +``` -# Set the flavour environment variable to the string you got in Detecting flavour section. -# For example if the flavour is `amd64-musl` the command will be +Set the flavour environment variable to the string you got in Detecting flavour section. For example if the flavour is `amd64-musl` the command will be: + +```sh export FLAVOUR="amd64-musl" +``` -# Clone the release build into a temporary directory and unpack it -sudo -Hu pleroma " -curl 'https://git.pleroma.social/api/packages/pleroma/generic/pleroma-otp-stable-$FLAVOUR/latest/pleroma.zip' -o /tmp/pleroma.zip -unzip /tmp/pleroma.zip -d /tmp/ -" +Clone the release build into a temporary directory and unpack it: -# Move the release to the home directory and delete temporary files -sudo -Hu pleroma " -mv /tmp/release/* /opt/pleroma -rmdir /tmp/release -rm /tmp/pleroma.zip -" -# Create uploads directory and set proper permissions (skip if planning to use a remote uploader) -# Note: It does not have to be `/var/lib/pleroma/uploads`, the config generator will ask about the upload directory later +```sh +sudo -Hu pleroma curl "https://git.pleroma.social/api/packages/pleroma/generic/pleroma-otp-stable-$FLAVOUR/latest/pleroma.zip" -o /tmp/pleroma.zip +sudo -Hu pleroma unzip /tmp/pleroma.zip -d /tmp/ +``` +Move the release to the home directory and delete temporary files: + +```sh +sudo -Hu pleroma mv /tmp/release/* /opt/pleroma +sudo -Hu pleroma rmdir /tmp/release +sudo -Hu pleroma rm /tmp/pleroma.zip +``` + +Create uploads directory and set proper permissions (skip if planning to use a remote uploader). +Note: It does not have to be `/var/lib/pleroma/uploads`, the config generator will ask about the upload directory later. + +```sh mkdir -p /var/lib/pleroma/uploads chown -R pleroma /var/lib/pleroma +``` -# Create custom public files directory (custom emojis, frontend bundle overrides, robots.txt, etc.) -# Note: It does not have to be `/var/lib/pleroma/static`, the config generator will ask about the custom public files directory later +Create custom public files directory (custom emojis, frontend bundle overrides, robots.txt, etc.) +Note: It does not have to be `/var/lib/pleroma/static`, the config generator will ask about the custom public files directory later. + +```sh mkdir -p /var/lib/pleroma/static chown -R pleroma /var/lib/pleroma +``` -# Create a config directory +Create a config directory: + +```sh mkdir -p /etc/pleroma chown -R pleroma /etc/pleroma +``` -# Run the config generator +Run the config generator: + +```sh sudo -Hu pleroma "./bin/pleroma_ctl instance gen --output /etc/pleroma/config.exs --output-psql /tmp/setup_db.psql" +``` -# Create the postgres database +Create the postgres database and schema: + +```sh sudo -u postgres -s $SHELL -lc "psql -f /tmp/setup_db.psql" - -# Create the database schema sudo -Hu pleroma "./bin/pleroma_ctl migrate" +``` -# If you have installed RUM indexes uncommend and run -# sudo -Hu pleroma "./bin/pleroma_ctl migrate --migrations-path priv/repo/optional_migrations/rum_indexing/" +(Optional) If you have installed RUM indexes: -# Start the instance to verify that everything is working as expected +```sh +sudo -Hu pleroma "./bin/pleroma_ctl migrate --migrations-path priv/repo/optional_migrations/rum_indexing/" +``` + +Start the instance to verify that everything is working as expected: + +```sh sudo -Hu pleroma "./bin/pleroma daemon" +``` -# Wait for about 20 seconds and query the instance endpoint, if it shows your uri, name and email correctly, you are configured correctly +Wait for about 20 seconds and query the instance endpoint, if it shows your uri, name and email correctly, you are configured correctly: + +```sh sleep 20 && curl http://localhost:4000/api/v1/instance +``` -# Stop the instance +Stop the instance: + +```sh sudo -Hu pleroma "./bin/pleroma stop" ``` From 6a5ca6213a7965094769b7e51658c958984972ce Mon Sep 17 00:00:00 2001 From: Phantasm Date: Tue, 18 Aug 2026 01:01:26 +0200 Subject: [PATCH 88/90] Oban: Enable Reindexer plugin to reduce oban_jobs index DB bloat --- changelog.d/oban-reindexer.add | 1 + config/config.exs | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 changelog.d/oban-reindexer.add diff --git a/changelog.d/oban-reindexer.add b/changelog.d/oban-reindexer.add new file mode 100644 index 000000000..5b56d9482 --- /dev/null +++ b/changelog.d/oban-reindexer.add @@ -0,0 +1 @@ +Oban: Enabled the Reindexer plugin to reduce oban_jobs index DB bloat diff --git a/config/config.exs b/config/config.exs index d4a70450f..e2c110689 100644 --- a/config/config.exs +++ b/config/config.exs @@ -607,7 +607,11 @@ config :pleroma, Oban, search_indexing: [limit: 10, paused: true], slow: 5 ], - plugins: [Oban.Plugins.Lazarus, {Oban.Plugins.Pruner, max_age: 900}], + plugins: [ + Oban.Plugins.Lazarus, + {Oban.Plugins.Pruner, max_age: 900}, + {Oban.Plugins.Reindexer, schedule: "@weekly"} + ], crontab: [ {"0 0 * * 0", Pleroma.Workers.Cron.DigestEmailsWorker}, {"0 0 * * *", Pleroma.Workers.Cron.NewUsersDigestWorker}, From 16890c051ac1a64e6a7ddb376d4f11d33a1f26de Mon Sep 17 00:00:00 2001 From: Phantasm Date: Tue, 18 Aug 2026 19:25:26 +0200 Subject: [PATCH 89/90] Syslog: Fix intermittent failure to compile due to old pc version Upstream hasn't made a release with the fix, pin fixed commit for the time being. --- changelog.d/syslog-pc.fix | 1 + mix.exs | 7 +++++++ mix.lock | 2 +- 3 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 changelog.d/syslog-pc.fix diff --git a/changelog.d/syslog-pc.fix b/changelog.d/syslog-pc.fix new file mode 100644 index 000000000..3f08cec47 --- /dev/null +++ b/changelog.d/syslog-pc.fix @@ -0,0 +1 @@ +Fix intermittent failure to compile syslog package diff --git a/mix.exs b/mix.exs index 4f82b1f53..23af9d7df 100644 --- a/mix.exs +++ b/mix.exs @@ -172,6 +172,13 @@ defmodule Pleroma.Mixfile do {:gen_smtp, "~> 0.15"}, {:mua, "~> 0.2.4"}, {:mail, "~> 0.3.1"}, + # Fixes intermittent failure to compile due to an old pc version, + # upstream hasn't made a release with the fix yet. + # https://github.com/Vagabond/erlang-syslog/pull/44 + {:syslog, + git: "https://github.com/Vagabond/erlang-syslog", + ref: "614a1fde7a8654bee2815b599bbdaae1bb0323b5", + override: true}, {:ex_syslogger, "~> 1.5"}, {:floki, "~> 0.38"}, {:timex, "~> 3.7"}, diff --git a/mix.lock b/mix.lock index a4db3f68d..bca6f777b 100644 --- a/mix.lock +++ b/mix.lock @@ -139,7 +139,7 @@ "statistex": {:hex, :statistex, "1.1.0", "7fec1eb2f580a0d2c1a05ed27396a084ab064a40cfc84246dbfb0c72a5c761e5", [:mix], [], "hexpm", "f5950ea26ad43246ba2cce54324ac394a4e7408fdcf98b8e230f503a0cba9cf5"}, "sweet_xml": {:hex, :sweet_xml, "0.7.5", "803a563113981aaac202a1dbd39771562d0ad31004ddbfc9b5090bdcd5605277", [:mix], [], "hexpm", "193b28a9b12891cae351d81a0cead165ffe67df1b73fe5866d10629f4faefb12"}, "swoosh": {:hex, :swoosh, "1.16.12", "cbb24ad512f2f7f24c7a469661c188a00a8c2cd64e0ab54acd1520f132092dfd", [:mix], [{:bandit, ">= 1.0.0", [hex: :bandit, repo: "hexpm", optional: true]}, {:cowboy, "~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:ex_aws, "~> 2.1", [hex: :ex_aws, repo: "hexpm", optional: true]}, {:finch, "~> 0.6", [hex: :finch, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13 or ~> 1.0", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mua, "~> 0.2.3", [hex: :mua, repo: "hexpm", optional: true]}, {:multipart, "~> 0.4", [hex: :multipart, repo: "hexpm", optional: true]}, {:plug, "~> 1.9", [hex: :plug, repo: "hexpm", optional: true]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:req, "~> 0.5 or ~> 1.0", [hex: :req, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.2 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "0e262df1ae510d59eeaaa3db42189a2aa1b3746f73771eb2616fc3f7ee63cc20"}, - "syslog": {:hex, :syslog, "1.1.0", "6419a232bea84f07b56dc575225007ffe34d9fdc91abe6f1b2f254fd71d8efc2", [:rebar3], [], "hexpm", "4c6a41373c7e20587be33ef841d3de6f3beba08519809329ecc4d27b15b659e1"}, + "syslog": {:git, "https://github.com/Vagabond/erlang-syslog", "614a1fde7a8654bee2815b599bbdaae1bb0323b5", [ref: "614a1fde7a8654bee2815b599bbdaae1bb0323b5"]}, "table_rex": {:hex, :table_rex, "4.1.0", "fbaa8b1ce154c9772012bf445bfb86b587430fb96f3b12022d3f35ee4a68c918", [:mix], [], "hexpm", "95932701df195d43bc2d1c6531178fc8338aa8f38c80f098504d529c43bc2601"}, "telemetry": {:hex, :telemetry, "1.0.0", "0f453a102cdf13d506b7c0ab158324c337c41f1cc7548f0bc0e130bbf0ae9452", [:rebar3], [], "hexpm", "73bc09fa59b4a0284efb4624335583c528e07ec9ae76aca96ea0673850aec57a"}, "telemetry_metrics": {:hex, :telemetry_metrics, "0.6.2", "2caabe9344ec17eafe5403304771c3539f3b6e2f7fb6a6f602558c825d0d0bfb", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "9b43db0dc33863930b9ef9d27137e78974756f5f198cae18409970ed6fa5b561"}, From 59042e2793f67b4ca8fc89a3ad67c8409244d45e Mon Sep 17 00:00:00 2001 From: Phantasm Date: Tue, 18 Aug 2026 21:41:48 +0200 Subject: [PATCH 90/90] Syslog: set runtime: false to fix OTP releases The syslog package is not directly part of the runtime. --- changelog.d/syslog-fix-otp.skip | 0 mix.exs | 3 ++- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/syslog-fix-otp.skip diff --git a/changelog.d/syslog-fix-otp.skip b/changelog.d/syslog-fix-otp.skip new file mode 100644 index 000000000..e69de29bb diff --git a/mix.exs b/mix.exs index 23af9d7df..140aafb98 100644 --- a/mix.exs +++ b/mix.exs @@ -178,7 +178,8 @@ defmodule Pleroma.Mixfile do {:syslog, git: "https://github.com/Vagabond/erlang-syslog", ref: "614a1fde7a8654bee2815b599bbdaae1bb0323b5", - override: true}, + override: true, + runtime: false}, {:ex_syslogger, "~> 1.5"}, {:floki, "~> 0.38"}, {:timex, "~> 3.7"},