From 71ef9f9519e2617a0c05d7447bbc406ae4a8d849 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Sat, 17 Aug 2024 16:36:27 +0200 Subject: [PATCH 001/117] Allow providing avatar/header descriptions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/user.ex | 28 ++++++++++++++++--- .../api_spec/operations/account_operation.ex | 10 +++++++ .../controllers/account_controller.ex | 2 ++ .../web/mastodon_api/views/account_view.ex | 10 ++++++- 4 files changed, 45 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index c6c536943..f443b64ae 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -589,13 +589,21 @@ defmodule Pleroma.User do |> put_fields() |> put_emoji() |> put_change_if_present(:bio, &{:ok, parse_bio(&1, struct)}) - |> put_change_if_present(:avatar, &put_upload(&1, :avatar)) - |> put_change_if_present(:banner, &put_upload(&1, :banner)) + |> put_change_if_present( + :avatar, + &put_upload(&1, :avatar, Map.get(params, :avatar_description, nil)) + ) + |> put_change_if_present( + :banner, + &put_upload(&1, :banner, Map.get(params, :header_description, nil)) + ) |> put_change_if_present(:background, &put_upload(&1, :background)) |> put_change_if_present( :pleroma_settings_store, &{:ok, Map.merge(struct.pleroma_settings_store, &1)} ) + |> maybe_update_image_description(:avatar, Map.get(params, :avatar_description)) + |> maybe_update_image_description(:banner, Map.get(params, :header_description)) |> validate_fields(false) end @@ -674,13 +682,25 @@ defmodule Pleroma.User do end end - defp put_upload(value, type) do + defp put_upload(value, type, description \\ nil) do with %Plug.Upload{} <- value, - {:ok, object} <- ActivityPub.upload(value, type: type) do + {:ok, object} <- ActivityPub.upload(value, type: type, description: description) do {:ok, object.data} end end + defp maybe_update_image_description(changeset, image_field, description) do + with {:image_missing, true} <- {:image_missing, not changed?(changeset, image_field)}, + {:existing_image, %{"id" => id}} <- + {:existing_image, Map.get(changeset.data, image_field)}, + {:object, %Object{} = object} <- {:object, Object.get_by_ap_id(id)}, + {:ok, object} <- Object.update_data(object, %{"name" => description}) do + put_change(changeset, image_field, object.data) + else + e -> changeset + end + end + def update_as_admin_changeset(struct, params) do struct |> update_changeset(params) diff --git a/lib/pleroma/web/api_spec/operations/account_operation.ex b/lib/pleroma/web/api_spec/operations/account_operation.ex index d9614bc48..21a779dcb 100644 --- a/lib/pleroma/web/api_spec/operations/account_operation.ex +++ b/lib/pleroma/web/api_spec/operations/account_operation.ex @@ -813,6 +813,16 @@ defmodule Pleroma.Web.ApiSpec.AccountOperation do allOf: [BooleanLike], nullable: true, description: "User's birthday will be visible" + }, + avatar_description: %Schema{ + type: :string, + nullable: true, + description: "Avatar image description." + }, + header_description: %Schema{ + type: :string, + nullable: true, + description: "Header image description." } }, example: %{ diff --git a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex index 54d46c86b..2302d6ed8 100644 --- a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex @@ -232,6 +232,8 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do |> Maps.put_if_present(:is_discoverable, params[:discoverable]) |> Maps.put_if_present(:birthday, params[:birthday]) |> Maps.put_if_present(:language, Pleroma.Web.Gettext.normalize_locale(params[:language])) + |> Maps.put_if_present(:avatar_description, params[:avatar_description]) + |> Maps.put_if_present(:header_description, params[:header_description]) # What happens here: # diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index 6976ca6e5..bd8af265a 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -220,8 +220,10 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do avatar = User.avatar_url(user) |> MediaProxy.url() avatar_static = User.avatar_url(user) |> MediaProxy.preview_url(static: true) + avatar_description = image_description(user.avatar) header = User.banner_url(user) |> MediaProxy.url() header_static = User.banner_url(user) |> MediaProxy.preview_url(static: true) + header_description = image_description(user.banner) following_count = if !user.hide_follows_count or !user.hide_follows or self, @@ -323,7 +325,9 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do background_image: image_url(user.background) |> MediaProxy.url(), accepts_chat_messages: user.accepts_chat_messages, favicon: favicon - } + }, + avatar_description: avatar_description, + header_description: header_description } |> maybe_put_role(user, opts[:for]) |> maybe_put_settings(user, opts[:for], opts) @@ -346,6 +350,10 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do defp username_from_nickname(_), do: nil + defp image_description(%{"name" => name}), do: name + + defp image_description(_), do: "" + defp maybe_put_follow_requests_count( data, %User{id: user_id} = user, From 681765669c5b5bdc92079357d76c859e60bb49d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Sat, 17 Aug 2024 17:02:44 +0200 Subject: [PATCH 002/117] Add test for avatar description MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/user.ex | 2 +- lib/pleroma/web/api_spec/schemas/account.ex | 4 ++ .../mastodon_api/update_credentials_test.exs | 42 +++++++++++++++++++ 3 files changed, 47 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index f443b64ae..c3cb72fab 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -697,7 +697,7 @@ defmodule Pleroma.User do {:ok, object} <- Object.update_data(object, %{"name" => description}) do put_change(changeset, image_field, object.data) else - e -> changeset + _ -> changeset end end diff --git a/lib/pleroma/web/api_spec/schemas/account.ex b/lib/pleroma/web/api_spec/schemas/account.ex index 8aeb821a8..32a0dd6cb 100644 --- a/lib/pleroma/web/api_spec/schemas/account.ex +++ b/lib/pleroma/web/api_spec/schemas/account.ex @@ -148,10 +148,13 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Account do } } } + avatar_description: %Schema{type: :string}, + header_description: %Schema{type: :string} }, example: %{ "acct" => "foobar", "avatar" => "https://mypleroma.com/images/avi.png", + "avatar_description" => "", "avatar_static" => "https://mypleroma.com/images/avi.png", "bot" => false, "created_at" => "2020-03-24T13:05:58.000Z", @@ -162,6 +165,7 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Account do "followers_count" => 0, "following_count" => 1, "header" => "https://mypleroma.com/images/banner.png", + "header_description" => "", "header_static" => "https://mypleroma.com/images/banner.png", "id" => "9tKi3esbG7OQgZ2920", "locked" => false, diff --git a/test/pleroma/web/mastodon_api/update_credentials_test.exs b/test/pleroma/web/mastodon_api/update_credentials_test.exs index bea0cae69..28d3b00db 100644 --- a/test/pleroma/web/mastodon_api/update_credentials_test.exs +++ b/test/pleroma/web/mastodon_api/update_credentials_test.exs @@ -430,6 +430,48 @@ defmodule Pleroma.Web.MastodonAPI.UpdateCredentialsTest do assert :ok == File.rm(Path.absname("test/tmp/large_binary.data")) end + test "adds avatar description with a new avatar", %{user: user, conn: conn} do + new_avatar = %Plug.Upload{ + content_type: "image/jpeg", + path: Path.absname("test/fixtures/image.jpg"), + filename: "an_image.jpg" + } + + res = + patch(conn, "/api/v1/accounts/update_credentials", %{ + "avatar" => new_avatar, + "avatar_description" => "me and pleroma tan" + }) + + assert json_response_and_validate_schema(res, 200) + + user = User.get_by_id(user.id) + assert user.avatar["name"] == "me and pleroma tan" + end + + test "adds avatar description to existing avatar", %{user: user, conn: conn} do + new_avatar = %Plug.Upload{ + content_type: "image/jpeg", + path: Path.absname("test/fixtures/image.jpg"), + filename: "an_image.jpg" + } + + assert user.avatar == %{} + + conn + |> patch("/api/v1/accounts/update_credentials", %{"avatar" => new_avatar}) + + assert conn + |> assign(:user, User.get_by_id(user.id)) + |> patch("/api/v1/accounts/update_credentials", %{ + "avatar_description" => "me and pleroma tan" + }) + |> json_response_and_validate_schema(200) + + user = User.get_by_id(user.id) + assert user.avatar["name"] == "me and pleroma tan" + end + test "Strip / from upload files", %{user: user, conn: conn} do new_image = %Plug.Upload{ content_type: "image/jpeg", From 071452a5d5e4e8d38d9c31bad171085574327fee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Sat, 17 Aug 2024 17:03:12 +0200 Subject: [PATCH 003/117] Update changelog MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- changelog.d/profile-image-descriptions.add | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/profile-image-descriptions.add diff --git a/changelog.d/profile-image-descriptions.add b/changelog.d/profile-image-descriptions.add new file mode 100644 index 000000000..85cc48083 --- /dev/null +++ b/changelog.d/profile-image-descriptions.add @@ -0,0 +1 @@ +Allow providing avatar/header descriptions \ No newline at end of file From 855c5a234f4ca743303f1b88974665d7b9f58684 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Sat, 17 Aug 2024 17:05:47 +0200 Subject: [PATCH 004/117] Update docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- .../development/API/differences_in_mastoapi_responses.md | 9 ++++++++- lib/pleroma/web/api_spec/schemas/account.ex | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/docs/development/API/differences_in_mastoapi_responses.md b/docs/development/API/differences_in_mastoapi_responses.md index 41464e802..114d6e32d 100644 --- a/docs/development/API/differences_in_mastoapi_responses.md +++ b/docs/development/API/differences_in_mastoapi_responses.md @@ -97,13 +97,18 @@ Endpoints which accept `with_relationships` parameter: - `/api/v1/accounts/:id/following` - `/api/v1/mutes` +Has these additional fields: + +- `avatar_description`: string, image description for user avatar, defaults to empty string +- `header_description`: string, image description for user banner, defaults to empty string + Has these additional fields under the `pleroma` object: - `ap_id`: nullable URL string, ActivityPub id of the user - `background_image`: nullable URL string, background image of the user - `tags`: Lists an array of tags for the user - `relationship` (object): Includes fields as documented for Mastodon API https://docs.joinmastodon.org/entities/relationship/ -- `is_moderator`: boolean, nullable, true if user is a moderator +- `is_moderator`: boolean, nullable, true if user is a moderator - `is_admin`: boolean, nullable, true if user is an admin - `confirmation_pending`: boolean, true if a new user account is waiting on email confirmation to be activated - `hide_favorites`: boolean, true when the user has hiding favorites enabled @@ -255,6 +260,8 @@ Additional parameters can be added to the JSON body/Form data: - `actor_type` - the type of this account. - `accepts_chat_messages` - if false, this account will reject all chat messages. - `language` - user's preferred language for receiving emails (digest, confirmation, etc.) +- `avatar_description` - image description for user avatar +- `header_description` - image description for user banner All images (avatar, banner and background) can be reset to the default by sending an empty string ("") instead of a file. diff --git a/lib/pleroma/web/api_spec/schemas/account.ex b/lib/pleroma/web/api_spec/schemas/account.ex index 32a0dd6cb..3f2310df9 100644 --- a/lib/pleroma/web/api_spec/schemas/account.ex +++ b/lib/pleroma/web/api_spec/schemas/account.ex @@ -147,7 +147,7 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Account do } } } - } + }, avatar_description: %Schema{type: :string}, header_description: %Schema{type: :string} }, From c802f3b7f61e1c4bbe2f4eec757802e30f88b6a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Sat, 17 Aug 2024 19:58:32 +0200 Subject: [PATCH 005/117] Validate media description length MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/user.ex | 24 ++++++++++++++--- .../controllers/account_controller.ex | 6 +++++ .../mastodon_api/update_credentials_test.exs | 27 +++++++++++++++++++ 3 files changed, 54 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index c3cb72fab..517009253 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -586,16 +586,18 @@ defmodule Pleroma.User do |> validate_length(:bio, max: bio_limit) |> validate_length(:name, min: 1, max: name_limit) |> validate_inclusion(:actor_type, Pleroma.Constants.allowed_user_actor_types()) + |> validate_image_description(:avatar_description, params) + |> validate_image_description(:header_description, params) |> put_fields() |> put_emoji() |> put_change_if_present(:bio, &{:ok, parse_bio(&1, struct)}) |> put_change_if_present( :avatar, - &put_upload(&1, :avatar, Map.get(params, :avatar_description, nil)) + &put_upload(&1, :avatar, Map.get(params, :avatar_description)) ) |> put_change_if_present( :banner, - &put_upload(&1, :banner, Map.get(params, :header_description, nil)) + &put_upload(&1, :banner, Map.get(params, :header_description)) ) |> put_change_if_present(:background, &put_upload(&1, :background)) |> put_change_if_present( @@ -689,7 +691,20 @@ defmodule Pleroma.User do end end - defp maybe_update_image_description(changeset, image_field, description) do + defp validate_image_description(changeset, key, params) do + description_limit = Config.get([:instance, :description_limit], 5_000) + description = Map.get(params, key) + + if is_binary(description) and String.length(description) > description_limit do + changeset + |> add_error(key, "#{key} is too long") + else + changeset + end + end + + defp maybe_update_image_description(changeset, image_field, description) + when is_binary(description) do with {:image_missing, true} <- {:image_missing, not changed?(changeset, image_field)}, {:existing_image, %{"id" => id}} <- {:existing_image, Map.get(changeset.data, image_field)}, @@ -697,10 +712,13 @@ defmodule Pleroma.User do {:ok, object} <- Object.update_data(object, %{"name" => description}) do put_change(changeset, image_field, object.data) else + {:description_too_long, true} -> {:error} _ -> changeset end end + defp maybe_update_image_description(changeset, _, _), do: changeset + def update_as_admin_changeset(struct, params) do struct |> update_changeset(params) diff --git a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex index 2302d6ed8..68157b0c4 100644 --- a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex @@ -279,6 +279,12 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do {:error, %Ecto.Changeset{errors: [{:name, {_, _}} | _]}} -> render_error(conn, :request_entity_too_large, "Name is too long") + {:error, %Ecto.Changeset{errors: [{:avatar_description, {_, _}} | _]}} -> + render_error(conn, :request_entity_too_large, "Avatar description is too long") + + {:error, %Ecto.Changeset{errors: [{:header_description, {_, _}} | _]}} -> + render_error(conn, :request_entity_too_large, "Banner description is too long") + {:error, %Ecto.Changeset{errors: [{:fields, {"invalid", _}} | _]}} -> render_error(conn, :request_entity_too_large, "One or more field entries are too long") diff --git a/test/pleroma/web/mastodon_api/update_credentials_test.exs b/test/pleroma/web/mastodon_api/update_credentials_test.exs index 28d3b00db..97ad2e849 100644 --- a/test/pleroma/web/mastodon_api/update_credentials_test.exs +++ b/test/pleroma/web/mastodon_api/update_credentials_test.exs @@ -472,6 +472,33 @@ defmodule Pleroma.Web.MastodonAPI.UpdateCredentialsTest do assert user.avatar["name"] == "me and pleroma tan" end + test "limit", %{user: user, conn: conn} do + new_header = %Plug.Upload{ + content_type: "image/jpeg", + path: Path.absname("test/fixtures/image.jpg"), + filename: "an_image.jpg" + } + + assert user.banner == %{} + + conn + |> patch("/api/v1/accounts/update_credentials", %{"header" => new_header}) + + description_limit = Config.get([:instance, :description_limit], 100) + + description = String.duplicate(".", description_limit + 1) + + conn = + conn + |> assign(:user, User.get_by_id(user.id)) + |> patch("/api/v1/accounts/update_credentials", %{ + "header_description" => description + }) + + assert %{"error" => "Banner description is too long"} = + json_response_and_validate_schema(conn, 413) + end + test "Strip / from upload files", %{user: user, conn: conn} do new_image = %Plug.Upload{ content_type: "image/jpeg", From 3498662712c088cf578e7241c12aa1e5da42745a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Sat, 17 Aug 2024 19:59:39 +0200 Subject: [PATCH 006/117] Move new fields to pleroma object MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- docs/development/API/differences_in_mastoapi_responses.md | 7 ++----- lib/pleroma/web/api_spec/schemas/account.ex | 8 ++++---- lib/pleroma/web/mastodon_api/views/account_view.ex | 8 ++++---- 3 files changed, 10 insertions(+), 13 deletions(-) diff --git a/docs/development/API/differences_in_mastoapi_responses.md b/docs/development/API/differences_in_mastoapi_responses.md index 114d6e32d..22a26b77b 100644 --- a/docs/development/API/differences_in_mastoapi_responses.md +++ b/docs/development/API/differences_in_mastoapi_responses.md @@ -97,11 +97,6 @@ Endpoints which accept `with_relationships` parameter: - `/api/v1/accounts/:id/following` - `/api/v1/mutes` -Has these additional fields: - -- `avatar_description`: string, image description for user avatar, defaults to empty string -- `header_description`: string, image description for user banner, defaults to empty string - Has these additional fields under the `pleroma` object: - `ap_id`: nullable URL string, ActivityPub id of the user @@ -125,6 +120,8 @@ Has these additional fields under the `pleroma` object: - `notification_settings`: object, can be absent. See `/api/v1/pleroma/notification_settings` for the parameters/keys returned. - `accepts_chat_messages`: boolean, but can be null if we don't have that information about a user - `favicon`: nullable URL string, Favicon image of the user's instance +- `avatar_description`: string, image description for user avatar, defaults to empty string +- `header_description`: string, image description for user banner, defaults to empty string ### Source diff --git a/lib/pleroma/web/api_spec/schemas/account.ex b/lib/pleroma/web/api_spec/schemas/account.ex index 3f2310df9..1f73ef60c 100644 --- a/lib/pleroma/web/api_spec/schemas/account.ex +++ b/lib/pleroma/web/api_spec/schemas/account.ex @@ -111,7 +111,9 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Account do format: :uri, nullable: true, description: "Favicon image of the user's instance" - } + }, + avatar_description: %Schema{type: :string}, + header_description: %Schema{type: :string} } }, source: %Schema{ @@ -147,9 +149,7 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Account do } } } - }, - avatar_description: %Schema{type: :string}, - header_description: %Schema{type: :string} + } }, example: %{ "acct" => "foobar", diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index bd8af265a..0643b8f14 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -324,10 +324,10 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do skip_thread_containment: user.skip_thread_containment, background_image: image_url(user.background) |> MediaProxy.url(), accepts_chat_messages: user.accepts_chat_messages, - favicon: favicon - }, - avatar_description: avatar_description, - header_description: header_description + favicon: favicon, + avatar_description: avatar_description, + header_description: header_description + } } |> maybe_put_role(user, opts[:for]) |> maybe_put_settings(user, opts[:for], opts) From 917ac89b4f944a80b1d168fd07d94c762ee04ed9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Sat, 17 Aug 2024 20:01:25 +0200 Subject: [PATCH 007/117] Update tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- test/pleroma/web/mastodon_api/views/account_view_test.exs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/test/pleroma/web/mastodon_api/views/account_view_test.exs b/test/pleroma/web/mastodon_api/views/account_view_test.exs index dca64853d..0301a4cca 100644 --- a/test/pleroma/web/mastodon_api/views/account_view_test.exs +++ b/test/pleroma/web/mastodon_api/views/account_view_test.exs @@ -96,7 +96,9 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do hide_follows_count: false, relationship: %{}, skip_thread_containment: false, - accepts_chat_messages: nil + accepts_chat_messages: nil, + avatar_description: "", + header_description: "" } } @@ -340,7 +342,9 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do hide_follows_count: false, relationship: %{}, skip_thread_containment: false, - accepts_chat_messages: nil + accepts_chat_messages: nil, + avatar_description: "", + header_description: "" } } From 010edcbcb51dfddc83d5a3810c257c1678429c2d Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 21 Aug 2024 14:50:19 -0400 Subject: [PATCH 008/117] Use Map.filter now that minimum Elixir version is 1.13 --- lib/pleroma/maps.ex | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/pleroma/maps.ex b/lib/pleroma/maps.ex index 5020a8ff8..1afbde484 100644 --- a/lib/pleroma/maps.ex +++ b/lib/pleroma/maps.ex @@ -20,15 +20,13 @@ defmodule Pleroma.Maps do end def filter_empty_values(data) do - # TODO: Change to Map.filter in Elixir 1.13+ data - |> Enum.filter(fn + |> Map.filter(fn {_k, nil} -> false {_k, ""} -> false {_k, []} -> false {_k, %{} = v} -> Map.keys(v) != [] {_k, _v} -> true end) - |> Map.new() end end From e65555e8c5cacb36a404579f56fb501a7fba0781 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 21 Aug 2024 15:11:41 -0400 Subject: [PATCH 009/117] Remove workaround for URI.merge bug on nil fields before Elixir 1.13 https://github.com/elixir-lang/elixir/issues/10771 --- lib/pleroma/web/mastodon_api/views/status_view.ex | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 747638c53..1b78477d0 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -803,19 +803,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do defp build_application(_), do: nil - # Workaround for Elixir issue #10771 - # Avoid applying URI.merge unless necessary - # TODO: revert to always attempting URI.merge(image_url_data, page_url_data) - # when Elixir 1.12 is the minimum supported version - @spec build_image_url(struct() | nil, struct()) :: String.t() | nil - defp build_image_url( - %URI{scheme: image_scheme, host: image_host} = image_url_data, - %URI{} = _page_url_data - ) - when not is_nil(image_scheme) and not is_nil(image_host) do - image_url_data |> to_string - end - + @spec build_image_url(URI.t(), URI.t()) :: String.t() defp build_image_url(%URI{} = image_url_data, %URI{} = page_url_data) do URI.merge(page_url_data, image_url_data) |> to_string end From 5138a4984ba8cf04e0b6015a7a9253a9013e013e Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 21 Aug 2024 15:24:33 -0400 Subject: [PATCH 010/117] Skip changelog --- changelog.d/todo-cleanup.skip | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 changelog.d/todo-cleanup.skip diff --git a/changelog.d/todo-cleanup.skip b/changelog.d/todo-cleanup.skip new file mode 100644 index 000000000..e69de29bb From 5f6506d864239408e9fa3705c5dd7b241307241a Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 27 Aug 2024 20:39:32 -0400 Subject: [PATCH 011/117] Pleroma.HTTP: option stream: true will return a stream as the body for Gun adapter --- lib/pleroma/http/adapter_helper/gun.ex | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lib/pleroma/http/adapter_helper/gun.ex b/lib/pleroma/http/adapter_helper/gun.ex index 1fe8dd4b2..f9a8180f2 100644 --- a/lib/pleroma/http/adapter_helper/gun.ex +++ b/lib/pleroma/http/adapter_helper/gun.ex @@ -32,6 +32,7 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do |> AdapterHelper.maybe_add_proxy(proxy) |> Keyword.merge(incoming_opts) |> put_timeout() + |> maybe_stream() end defp add_scheme_opts(opts, %{scheme: "http"}), do: opts @@ -47,6 +48,14 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do Keyword.put(opts, :timeout, recv_timeout) end + # Tesla Gun adapter uses body_as: :stream + defp maybe_stream(opts) do + case Keyword.pop(opts, :stream, nil) do + {true, opts} -> Keyword.put(opts, :body_as, :stream) + {_, opts} -> opts + end + end + @spec pool_timeout(pool()) :: non_neg_integer() def pool_timeout(pool) do default = Config.get([:pools, :default, :recv_timeout], 5_000) From bb279c28025522764272468e3177a5f6701bc155 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 27 Aug 2024 21:08:25 -0400 Subject: [PATCH 012/117] Pleroma.HTTP add AdapterHelper.can_stream? to assist with discovering if the current adapter supports returning a Stream body --- lib/pleroma/http/adapter_helper.ex | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lib/pleroma/http/adapter_helper.ex b/lib/pleroma/http/adapter_helper.ex index dcb27a29d..f8bde2ac3 100644 --- a/lib/pleroma/http/adapter_helper.ex +++ b/lib/pleroma/http/adapter_helper.ex @@ -118,4 +118,13 @@ defmodule Pleroma.HTTP.AdapterHelper do host_charlist end end + + #TODO add Finch support once we have an AdapterHelper for it + @spec can_stream? :: bool() + def can_stream? do + case Application.get_env(:tesla, :adapter) do + Tesla.Adapter.Gun -> true + _ -> false + end + end end From ec8db9d4eedfade5a8b74425b21b07b3f4e44992 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 27 Aug 2024 21:09:15 -0400 Subject: [PATCH 013/117] RichMedia: skip the HTTP HEAD request for adapters that support streaming the response body --- lib/pleroma/web/rich_media/helpers.ex | 38 +++++++++++++++++++++++---- 1 file changed, 33 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/web/rich_media/helpers.ex b/lib/pleroma/web/rich_media/helpers.ex index e2889b351..88bfbae68 100644 --- a/lib/pleroma/web/rich_media/helpers.ex +++ b/lib/pleroma/web/rich_media/helpers.ex @@ -11,16 +11,39 @@ defmodule Pleroma.Web.RichMedia.Helpers do @spec rich_media_get(String.t()) :: {:ok, String.t()} | get_errors() def rich_media_get(url) do - headers = [{"user-agent", Pleroma.Application.user_agent() <> "; Bot"}] + case Pleroma.HTTP.AdapterHelper.can_stream?() do + true -> stream(url) + false -> head_first(url) + end + |> handle_result(url) + end + defp stream(url) do + with {_, {:ok, %Tesla.Env{status: 200, body: stream_body, headers: headers}}} <- + {:head, Pleroma.HTTP.get(url, req_headers(), http_options())}, + {_, :ok} <- {:content_type, check_content_type(headers)}, + {_, :ok} <- {:content_length, check_content_length(headers)} do + body = Enum.into(stream_body, <<>>) + {:ok, body} + end + end + + defp head_first(url) do with {_, {:ok, %Tesla.Env{status: 200, headers: headers}}} <- - {:head, Pleroma.HTTP.head(url, headers, http_options())}, + {:head, Pleroma.HTTP.head(url, req_headers(), http_options())}, {_, :ok} <- {:content_type, check_content_type(headers)}, {_, :ok} <- {:content_length, check_content_length(headers)}, {_, {:ok, %Tesla.Env{status: 200, body: body}}} <- - {:get, Pleroma.HTTP.get(url, headers, http_options())} do + {:get, Pleroma.HTTP.get(url, req_headers(), http_options())} do {:ok, body} - else + end + end + + defp handle_result(result, url) do + case result do + {:ok, body} -> + {:ok, body} + {:head, _} -> Logger.debug("Rich media error for #{url}: HTTP HEAD failed") {:error, :head} @@ -74,7 +97,12 @@ defmodule Pleroma.Web.RichMedia.Helpers do [ pool: :rich_media, max_body: Config.get([:rich_media, :max_body], 5_000_000), - tesla_middleware: [{Tesla.Middleware.Timeout, timeout: timeout}] + tesla_middleware: [{Tesla.Middleware.Timeout, timeout: timeout}], + stream: true ] end + + defp req_headers do + [{"user-agent", Pleroma.Application.user_agent() <> "; Bot"}] + end end From 0a86d2b3ac9c90a16aec1237019ecfcb1e680728 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 27 Aug 2024 21:22:59 -0400 Subject: [PATCH 014/117] Handle streaming response errors --- lib/pleroma/http/adapter_helper.ex | 2 +- lib/pleroma/web/rich_media/helpers.ex | 16 ++++++++++------ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/lib/pleroma/http/adapter_helper.ex b/lib/pleroma/http/adapter_helper.ex index f8bde2ac3..4dbcccdcc 100644 --- a/lib/pleroma/http/adapter_helper.ex +++ b/lib/pleroma/http/adapter_helper.ex @@ -119,7 +119,7 @@ defmodule Pleroma.HTTP.AdapterHelper do end end - #TODO add Finch support once we have an AdapterHelper for it + # TODO add Finch support once we have an AdapterHelper for it @spec can_stream? :: bool() def can_stream? do case Application.get_env(:tesla, :adapter) do diff --git a/lib/pleroma/web/rich_media/helpers.ex b/lib/pleroma/web/rich_media/helpers.ex index 88bfbae68..db1310b23 100644 --- a/lib/pleroma/web/rich_media/helpers.ex +++ b/lib/pleroma/web/rich_media/helpers.ex @@ -11,10 +11,10 @@ defmodule Pleroma.Web.RichMedia.Helpers do @spec rich_media_get(String.t()) :: {:ok, String.t()} | get_errors() def rich_media_get(url) do - case Pleroma.HTTP.AdapterHelper.can_stream?() do - true -> stream(url) - false -> head_first(url) - end + case Pleroma.HTTP.AdapterHelper.can_stream?() do + true -> stream(url) + false -> head_first(url) + end |> handle_result(url) end @@ -22,8 +22,8 @@ defmodule Pleroma.Web.RichMedia.Helpers do with {_, {:ok, %Tesla.Env{status: 200, body: stream_body, headers: headers}}} <- {:head, Pleroma.HTTP.get(url, req_headers(), http_options())}, {_, :ok} <- {:content_type, check_content_type(headers)}, - {_, :ok} <- {:content_length, check_content_length(headers)} do - body = Enum.into(stream_body, <<>>) + {_, :ok} <- {:content_length, check_content_length(headers)}, + body <- Enum.into(stream_body, <<>>) do {:ok, body} end end @@ -59,6 +59,10 @@ defmodule Pleroma.Web.RichMedia.Helpers do {:get, _} -> Logger.debug("Rich media error for #{url}: HTTP GET failed") {:error, :get} + + {:error, :recv_chunk_timeout} -> + Logger.debug("Rich media error for #{url}: HTTP streaming response failed") + {:error, :get} end end From 116fe77b77eedd2feb073d3be256fea08169c95b Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 27 Aug 2024 21:55:06 -0400 Subject: [PATCH 015/117] Tesla.Middleware.Timeout breaks streaming bodies These are executed by Oban now and Oban can enforce the timeout if the regular HTTP timeout is not sufficient. --- lib/pleroma/web/rich_media/helpers.ex | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/pleroma/web/rich_media/helpers.ex b/lib/pleroma/web/rich_media/helpers.ex index db1310b23..880d19218 100644 --- a/lib/pleroma/web/rich_media/helpers.ex +++ b/lib/pleroma/web/rich_media/helpers.ex @@ -96,12 +96,9 @@ defmodule Pleroma.Web.RichMedia.Helpers do end defp http_options do - timeout = Config.get!([:rich_media, :timeout]) - [ pool: :rich_media, max_body: Config.get([:rich_media, :max_body], 5_000_000), - tesla_middleware: [{Tesla.Middleware.Timeout, timeout: timeout}], stream: true ] end From 44901502ffd7713d498976e2d2b9a55c298f1876 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 27 Aug 2024 21:56:02 -0400 Subject: [PATCH 016/117] Fix incorrect identifier for the with statement --- lib/pleroma/web/rich_media/helpers.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/rich_media/helpers.ex b/lib/pleroma/web/rich_media/helpers.ex index 880d19218..b81984343 100644 --- a/lib/pleroma/web/rich_media/helpers.ex +++ b/lib/pleroma/web/rich_media/helpers.ex @@ -20,7 +20,7 @@ defmodule Pleroma.Web.RichMedia.Helpers do defp stream(url) do with {_, {:ok, %Tesla.Env{status: 200, body: stream_body, headers: headers}}} <- - {:head, Pleroma.HTTP.get(url, req_headers(), http_options())}, + {:get, Pleroma.HTTP.get(url, req_headers(), http_options())}, {_, :ok} <- {:content_type, check_content_type(headers)}, {_, :ok} <- {:content_length, check_content_length(headers)}, body <- Enum.into(stream_body, <<>>) do From 0804b73c0ae5846a133386c09970546375e3d918 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 27 Aug 2024 22:08:29 -0400 Subject: [PATCH 017/117] This error is not returned by Tesla Upstream has a bug filed for this as they aren't handling this error internally, so it was raising --- lib/pleroma/web/rich_media/helpers.ex | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lib/pleroma/web/rich_media/helpers.ex b/lib/pleroma/web/rich_media/helpers.ex index b81984343..a242ca640 100644 --- a/lib/pleroma/web/rich_media/helpers.ex +++ b/lib/pleroma/web/rich_media/helpers.ex @@ -59,10 +59,6 @@ defmodule Pleroma.Web.RichMedia.Helpers do {:get, _} -> Logger.debug("Rich media error for #{url}: HTTP GET failed") {:error, :get} - - {:error, :recv_chunk_timeout} -> - Logger.debug("Rich media error for #{url}: HTTP streaming response failed") - {:error, :get} end end From 3419e2cbdd3bf1c21e3bbf44ea5313ecfd03c989 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Wed, 28 Aug 2024 17:37:42 +0200 Subject: [PATCH 018/117] Correct response in AdminAPI docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- changelog.d/docs-fix.skip | 0 docs/development/API/admin_api.md | 31 +++++++++++++++++-------------- 2 files changed, 17 insertions(+), 14 deletions(-) create mode 100644 changelog.d/docs-fix.skip diff --git a/changelog.d/docs-fix.skip b/changelog.d/docs-fix.skip new file mode 100644 index 000000000..e69de29bb diff --git a/docs/development/API/admin_api.md b/docs/development/API/admin_api.md index 5b373b8e1..409e78a1e 100644 --- a/docs/development/API/admin_api.md +++ b/docs/development/API/admin_api.md @@ -433,7 +433,7 @@ Response: * On success: URL of the unfollowed relay ```json -{"https://example.com/relay"} +"https://example.com/relay" ``` ## `POST /api/v1/pleroma/admin/users/invite_token` @@ -1193,20 +1193,23 @@ Loads json generated from `config/descriptions.exs`. - Response: ```json -[ - { - "id": 1234, - "data": { - "actor": { - "id": 1, - "nickname": "lain" +{ + "items": [ + { + "id": 1234, + "data": { + "actor": { + "id": 1, + "nickname": "lain" + }, + "action": "relay_follow" }, - "action": "relay_follow" - }, - "time": 1502812026, // timestamp - "message": "[2017-08-15 15:47:06] @nick0 followed relay: https://example.org/relay" // log message - } -] + "time": 1502812026, // timestamp + "message": "[2017-08-15 15:47:06] @nick0 followed relay: https://example.org/relay" // log message + } + ], + "total": 1 +} ``` ## `POST /api/v1/pleroma/admin/reload_emoji` From fc450fdefc2df2bbec20a79fb2c60a95e7f41833 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 28 Aug 2024 15:45:13 -0400 Subject: [PATCH 019/117] ReceiverWorker: cancel job if user fetch is forbidden An instance block with authenticated fetch being required can cause this as we couldn't get the user to find their public key to verify the signature. Commonly observed if someone boosts/Announces a post from an instance that blocked you. --- lib/pleroma/workers/receiver_worker.ex | 5 +- test/pleroma/workers/receiver_worker_test.exs | 48 +++++++++++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/workers/receiver_worker.ex b/lib/pleroma/workers/receiver_worker.ex index d4db97b63..7dce02a5f 100644 --- a/lib/pleroma/workers/receiver_worker.ex +++ b/lib/pleroma/workers/receiver_worker.ex @@ -56,17 +56,20 @@ defmodule Pleroma.Workers.ReceiverWorker do def timeout(_job), do: :timer.seconds(5) + defp process_errors({:error, {:error, _} = error}), do: process_errors(error) + defp process_errors(errors) do case errors do {:error, :origin_containment_failed} -> {:cancel, :origin_containment_failed} {:error, :already_present} -> {:cancel, :already_present} {:error, {:validate_object, _} = reason} -> {:cancel, reason} - {:error, {:error, {:validate, {:error, _changeset} = reason}}} -> {:cancel, reason} + {:error, {:validate, {:error, _changeset} = reason}} -> {:cancel, reason} {:error, {:reject, _} = reason} -> {:cancel, reason} {:signature, false} -> {:cancel, :invalid_signature} {:error, "Object has been deleted"} = reason -> {:cancel, reason} {:error, {:side_effects, {:error, :no_object_actor}} = reason} -> {:cancel, reason} {:error, :not_found} = reason -> {:cancel, reason} + {:error, :forbidden} = reason -> {:cancel, reason} {:error, _} = e -> e e -> {:error, e} end diff --git a/test/pleroma/workers/receiver_worker_test.exs b/test/pleroma/workers/receiver_worker_test.exs index 33be91085..640cefb78 100644 --- a/test/pleroma/workers/receiver_worker_test.exs +++ b/test/pleroma/workers/receiver_worker_test.exs @@ -51,6 +51,54 @@ defmodule Pleroma.Workers.ReceiverWorkerTest do }) end + test "it does not retry if a user fetch fails with a 403" do + Tesla.Mock.mock(fn + %{url: "https://simpsons.com/users/bart"} -> + %Tesla.Env{ + status: 403, + body: "" + } + end) + + params = + %{ + "@context" => [ + "https://www.w3.org/ns/activitystreams", + "https://w3id.org/security/v1" + ], + "actor" => "https://simpsons.com/users/bart", + "cc" => [], + "id" => "https://simpsons.com/activity/eat-my-shorts", + "object" => %{}, + "to" => ["https://www.w3.org/ns/activitystreams#Public"], + "type" => "Create" + } + + req_headers = [ + ["accept-encoding", "gzip"], + ["content-length", "31337"], + ["content-type", "application/activity+json"], + ["date", "Wed, 28 Aug 2024 15:36:31 GMT"], + ["digest", "SHA-256=ouge/6HP2/QryG6F3JNtZ6vzs/hSwMk67xdxe87eH7A="], + ["host", "bikeshed.party"], + [ + "signature", + "does not matter as user needs to be fetched first" + ] + ] + + {:ok, oban_job} = + Federator.incoming_ap_doc(%{ + method: "POST", + req_headers: req_headers, + request_path: "/inbox", + params: params, + query_string: "" + }) + + assert {:cancel, {:error, :forbidden}} = ReceiverWorker.perform(oban_job) + end + test "it can validate the signature" do Tesla.Mock.mock(fn %{url: "https://mastodon.social/users/bastianallgeier"} -> From 60101e240dea53c3496eda548dbe269fc22b2f72 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 28 Aug 2024 15:54:49 -0400 Subject: [PATCH 020/117] Add test confirming cancellation for activity by a deleted user --- test/pleroma/workers/receiver_worker_test.exs | 88 ++++++++++--------- 1 file changed, 46 insertions(+), 42 deletions(-) diff --git a/test/pleroma/workers/receiver_worker_test.exs b/test/pleroma/workers/receiver_worker_test.exs index 640cefb78..2c0da8887 100644 --- a/test/pleroma/workers/receiver_worker_test.exs +++ b/test/pleroma/workers/receiver_worker_test.exs @@ -51,52 +51,56 @@ defmodule Pleroma.Workers.ReceiverWorkerTest do }) end - test "it does not retry if a user fetch fails with a 403" do - Tesla.Mock.mock(fn - %{url: "https://simpsons.com/users/bart"} -> - %Tesla.Env{ - status: 403, - body: "" - } - end) + describe "cancels on a failed user fetch" do + setup do + Tesla.Mock.mock(fn + %{url: "https://springfield.social/users/bart"} -> + %Tesla.Env{ + status: 403, + body: "" + } - params = - %{ - "@context" => [ - "https://www.w3.org/ns/activitystreams", - "https://w3id.org/security/v1" - ], - "actor" => "https://simpsons.com/users/bart", - "cc" => [], - "id" => "https://simpsons.com/activity/eat-my-shorts", - "object" => %{}, - "to" => ["https://www.w3.org/ns/activitystreams#Public"], - "type" => "Create" - } + %{url: "https://springfield.social/users/troymcclure"} -> + %Tesla.Env{ + status: 404, + body: "" + } + end) + end - req_headers = [ - ["accept-encoding", "gzip"], - ["content-length", "31337"], - ["content-type", "application/activity+json"], - ["date", "Wed, 28 Aug 2024 15:36:31 GMT"], - ["digest", "SHA-256=ouge/6HP2/QryG6F3JNtZ6vzs/hSwMk67xdxe87eH7A="], - ["host", "bikeshed.party"], - [ - "signature", - "does not matter as user needs to be fetched first" - ] - ] + test "when request returns a 403" do + params = + insert(:note_activity).data + |> Map.put("actor", "https://springfield.social/users/bart") - {:ok, oban_job} = - Federator.incoming_ap_doc(%{ - method: "POST", - req_headers: req_headers, - request_path: "/inbox", - params: params, - query_string: "" - }) + {:ok, oban_job} = + Federator.incoming_ap_doc(%{ + method: "POST", + req_headers: [], + request_path: "/inbox", + params: params, + query_string: "" + }) - assert {:cancel, {:error, :forbidden}} = ReceiverWorker.perform(oban_job) + assert {:cancel, {:error, :forbidden}} = ReceiverWorker.perform(oban_job) + end + + test "when request returns a 404" do + params = + insert(:note_activity).data + |> Map.put("actor", "https://springfield.social/users/troymcclure") + + {:ok, oban_job} = + Federator.incoming_ap_doc(%{ + method: "POST", + req_headers: [], + request_path: "/inbox", + params: params, + query_string: "" + }) + + assert {:cancel, {:error, :not_found}} = ReceiverWorker.perform(oban_job) + end end test "it can validate the signature" do From 66e1b4089528dcd5bcdb61343f111cea03f17ab8 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 28 Aug 2024 16:04:12 -0400 Subject: [PATCH 021/117] Cancel if the User fetch resulted in a 410 --- test/pleroma/workers/receiver_worker_test.exs | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/test/pleroma/workers/receiver_worker_test.exs b/test/pleroma/workers/receiver_worker_test.exs index 2c0da8887..085108e37 100644 --- a/test/pleroma/workers/receiver_worker_test.exs +++ b/test/pleroma/workers/receiver_worker_test.exs @@ -65,6 +65,12 @@ defmodule Pleroma.Workers.ReceiverWorkerTest do status: 404, body: "" } + + %{url: "https://springfield.social/users/hankscorpio"} -> + %Tesla.Env{ + status: 410, + body: "" + } end) end @@ -101,6 +107,23 @@ defmodule Pleroma.Workers.ReceiverWorkerTest do assert {:cancel, {:error, :not_found}} = ReceiverWorker.perform(oban_job) end + + test "when request returns a 410" do + params = + insert(:note_activity).data + |> Map.put("actor", "https://springfield.social/users/hankscorpio") + + {:ok, oban_job} = + Federator.incoming_ap_doc(%{ + method: "POST", + req_headers: [], + request_path: "/inbox", + params: params, + query_string: "" + }) + + assert {:cancel, {:error, :not_found}} = ReceiverWorker.perform(oban_job) + end end test "it can validate the signature" do From 48a46618858c9b0dee5ade61c0d9113c521be289 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 28 Aug 2024 16:22:38 -0400 Subject: [PATCH 022/117] Simplify test, move data into a json fixture By removing the inReplyTo, tags, and cc we can simplify the test and it still passes signature validation --- test/fixtures/bastianallgeier.json | 117 -------------- .../receiver_worker_signature_activity.json | 127 ++++++++++----- test/pleroma/workers/receiver_worker_test.exs | 147 +----------------- 3 files changed, 89 insertions(+), 302 deletions(-) delete mode 100644 test/fixtures/bastianallgeier.json diff --git a/test/fixtures/bastianallgeier.json b/test/fixtures/bastianallgeier.json deleted file mode 100644 index 6b47e7db9..000000000 --- a/test/fixtures/bastianallgeier.json +++ /dev/null @@ -1,117 +0,0 @@ -{ - "@context": [ - "https://www.w3.org/ns/activitystreams", - "https://w3id.org/security/v1", - { - "Curve25519Key": "toot:Curve25519Key", - "Device": "toot:Device", - "Ed25519Key": "toot:Ed25519Key", - "Ed25519Signature": "toot:Ed25519Signature", - "EncryptedMessage": "toot:EncryptedMessage", - "PropertyValue": "schema:PropertyValue", - "alsoKnownAs": { - "@id": "as:alsoKnownAs", - "@type": "@id" - }, - "cipherText": "toot:cipherText", - "claim": { - "@id": "toot:claim", - "@type": "@id" - }, - "deviceId": "toot:deviceId", - "devices": { - "@id": "toot:devices", - "@type": "@id" - }, - "discoverable": "toot:discoverable", - "featured": { - "@id": "toot:featured", - "@type": "@id" - }, - "featuredTags": { - "@id": "toot:featuredTags", - "@type": "@id" - }, - "fingerprintKey": { - "@id": "toot:fingerprintKey", - "@type": "@id" - }, - "focalPoint": { - "@container": "@list", - "@id": "toot:focalPoint" - }, - "identityKey": { - "@id": "toot:identityKey", - "@type": "@id" - }, - "indexable": "toot:indexable", - "manuallyApprovesFollowers": "as:manuallyApprovesFollowers", - "memorial": "toot:memorial", - "messageFranking": "toot:messageFranking", - "messageType": "toot:messageType", - "movedTo": { - "@id": "as:movedTo", - "@type": "@id" - }, - "publicKeyBase64": "toot:publicKeyBase64", - "schema": "http://schema.org#", - "suspended": "toot:suspended", - "toot": "http://joinmastodon.org/ns#", - "value": "schema:value" - } - ], - "attachment": [ - { - "name": "Website", - "type": "PropertyValue", - "value": "https://bastianallgeier.com" - }, - { - "name": "Project", - "type": "PropertyValue", - "value": "https://getkirby.com" - }, - { - "name": "Github", - "type": "PropertyValue", - "value": "https://github.com/bastianallgeier" - } - ], - "devices": "https://mastodon.social/users/bastianallgeier/collections/devices", - "discoverable": true, - "endpoints": { - "sharedInbox": "https://mastodon.social/inbox" - }, - "featured": "https://mastodon.social/users/bastianallgeier/collections/featured", - "featuredTags": "https://mastodon.social/users/bastianallgeier/collections/tags", - "followers": "https://mastodon.social/users/bastianallgeier/followers", - "following": "https://mastodon.social/users/bastianallgeier/following", - "icon": { - "mediaType": "image/jpeg", - "type": "Image", - "url": "https://files.mastodon.social/accounts/avatars/000/007/393/original/0180a20079617c71.jpg" - }, - "id": "https://mastodon.social/users/bastianallgeier", - "image": { - "mediaType": "image/jpeg", - "type": "Image", - "url": "https://files.mastodon.social/accounts/headers/000/007/393/original/13d644ab46d50478.jpeg" - }, - "inbox": "https://mastodon.social/users/bastianallgeier/inbox", - "indexable": false, - "manuallyApprovesFollowers": false, - "memorial": false, - "name": "Bastian Allgeier", - "outbox": "https://mastodon.social/users/bastianallgeier/outbox", - "preferredUsername": "bastianallgeier", - "publicKey": { - "id": "https://mastodon.social/users/bastianallgeier#main-key", - "owner": "https://mastodon.social/users/bastianallgeier", - "publicKeyPem": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA3fz+hpgVztO9z6HUhyzv\nwP++ERBBoIwSLKf1TyIM8bvzGFm2YXaO5uxu1HvumYFTYc3ACr3q4j8VUb7NMxkQ\nlzu4QwPjOFJ43O+fY+HSPORXEDW5fXDGC5DGpox4+i08LxRmx7L6YPRUSUuPN8nI\nWyq1Qsq1zOQrNY/rohMXkBdSXxqC3yIRqvtLt4otCgay/5tMogJWkkS6ZKyFhb9z\nwVVy1fsbV10c9C+SHy4NH26CKaTtpTYLRBMjhTCS8bX8iDSjGIf2aZgYs1ir7gEz\n9wf5CvLiENmVWGwm64t6KSEAkA4NJ1hzgHUZPCjPHZE2SmhO/oHaxokTzqtbbENJ\n1QIDAQAB\n-----END PUBLIC KEY-----\n" - }, - "published": "2016-11-01T00:00:00Z", - "summary": "

Designer & developer. Creator of Kirby CMS

", - "tag": [], - "type": "Person", - "url": "https://mastodon.social/@bastianallgeier" -} diff --git a/test/fixtures/receiver_worker_signature_activity.json b/test/fixtures/receiver_worker_signature_activity.json index 3c3fb3fd2..19dc0087f 100644 --- a/test/fixtures/receiver_worker_signature_activity.json +++ b/test/fixtures/receiver_worker_signature_activity.json @@ -1,62 +1,109 @@ { "@context": [ "https://www.w3.org/ns/activitystreams", + "https://w3id.org/security/v1", { + "claim": { + "@id": "toot:claim", + "@type": "@id" + }, + "memorial": "toot:memorial", "atomUri": "ostatus:atomUri", + "manuallyApprovesFollowers": "as:manuallyApprovesFollowers", "blurhash": "toot:blurhash", - "conversation": "ostatus:conversation", + "ostatus": "http://ostatus.org#", + "discoverable": "toot:discoverable", "focalPoint": { "@container": "@list", "@id": "toot:focalPoint" }, - "inReplyToAtomUri": "ostatus:inReplyToAtomUri", - "ostatus": "http://ostatus.org#", + "votersCount": "toot:votersCount", + "Hashtag": "as:Hashtag", + "Emoji": "toot:Emoji", + "alsoKnownAs": { + "@id": "as:alsoKnownAs", + "@type": "@id" + }, "sensitive": "as:sensitive", + "movedTo": { + "@id": "as:movedTo", + "@type": "@id" + }, + "inReplyToAtomUri": "ostatus:inReplyToAtomUri", + "conversation": "ostatus:conversation", + "Device": "toot:Device", + "schema": "http://schema.org#", "toot": "http://joinmastodon.org/ns#", - "votersCount": "toot:votersCount" + "cipherText": "toot:cipherText", + "suspended": "toot:suspended", + "messageType": "toot:messageType", + "featuredTags": { + "@id": "toot:featuredTags", + "@type": "@id" + }, + "Curve25519Key": "toot:Curve25519Key", + "deviceId": "toot:deviceId", + "Ed25519Signature": "toot:Ed25519Signature", + "featured": { + "@id": "toot:featured", + "@type": "@id" + }, + "devices": { + "@id": "toot:devices", + "@type": "@id" + }, + "value": "schema:value", + "PropertyValue": "schema:PropertyValue", + "messageFranking": "toot:messageFranking", + "publicKeyBase64": "toot:publicKeyBase64", + "identityKey": { + "@id": "toot:identityKey", + "@type": "@id" + }, + "Ed25519Key": "toot:Ed25519Key", + "indexable": "toot:indexable", + "EncryptedMessage": "toot:EncryptedMessage", + "fingerprintKey": { + "@id": "toot:fingerprintKey", + "@type": "@id" + } } ], - "atomUri": "https://chaos.social/users/distantnative/statuses/109336635639931467", - "attachment": [ - { - "blurhash": "UAK1zS00OXIUxuMxIUM{?b-:-;W:Di?b%2M{", - "height": 960, - "mediaType": "image/jpeg", - "name": null, - "type": "Document", - "url": "https://assets.chaos.social/media_attachments/files/109/336/634/286/114/657/original/2e6122063d8bfb26.jpeg", - "width": 346 - } - ], - "attributedTo": "https://chaos.social/users/distantnative", - "cc": [ - "https://chaos.social/users/distantnative/followers" - ], - "content": "

Favorite piece of anthropology meta discourse.

", - "contentMap": { - "en": "

Favorite piece of anthropology meta discourse.

" - }, - "conversation": "tag:chaos.social,2022-11-13:objectId=71843781:objectType=Conversation", - "id": "https://chaos.social/users/distantnative/statuses/109336635639931467", + "actor": "https://phpc.social/users/denniskoch", + "cc": [], + "id": "https://phpc.social/users/denniskoch/statuses/112847382711461301/activity", "inReplyTo": null, "inReplyToAtomUri": null, - "published": "2022-11-13T13:04:20Z", - "replies": { - "first": { - "items": [], - "next": "https://chaos.social/users/distantnative/statuses/109336635639931467/replies?only_other_accounts=true&page=true", - "partOf": "https://chaos.social/users/distantnative/statuses/109336635639931467/replies", - "type": "CollectionPage" + "object": { + "atomUri": "https://phpc.social/users/denniskoch/statuses/112847382711461301", + "attachment": [], + "attributedTo": "https://phpc.social/users/denniskoch", + "cc": [], + "content": "

@bastianallgeier @distantnative @kev Another main argument: Discord is popular. Many people have an account, so you can just join an server quickly. Also you know the app and how to get around.

", + "contentMap": { + "en": "

@bastianallgeier @distantnative @kev Another main argument: Discord is popular. Many people have an account, so you can just join an server quickly. Also you know the app and how to get around.

" }, - "id": "https://chaos.social/users/distantnative/statuses/109336635639931467/replies", - "type": "Collection" + "conversation": "tag:mastodon.social,2024-07-25:objectId=760068442:objectType=Conversation", + "id": "https://phpc.social/users/denniskoch/statuses/112847382711461301", + "published": "2024-07-25T13:33:29Z", + "replies": null, + "sensitive": false, + "tag": [], + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "type": "Note", + "url": "https://phpc.social/@denniskoch/112847382711461301" + }, + "published": "2024-07-25T13:33:29Z", + "signature": { + "created": "2024-07-25T13:33:29Z", + "creator": "https://phpc.social/users/denniskoch#main-key", + "signatureValue": "slz9BKJzd2n1S44wdXGOU+bV/wsskdgAaUpwxj8R16mYOL8+DTpE6VnfSKoZGsBBJT8uG5gnVfVEz1YsTUYtymeUgLMh7cvd8VnJnZPS+oixbmBRVky/Myf91TEgQQE7G4vDmTdB4ii54hZrHcOOYYf5FKPNRSkMXboKA6LMqNtekhbI+JTUJYIB02WBBK6PUyo15f6B1RJ6HGWVgud9NE0y1EZXfrkqUt682p8/9D49ORf7AwjXUJibKic2RbPvhEBj70qUGfBm4vvgdWhSUn1IG46xh+U0+NrTSUED82j1ZVOeua/2k/igkGs8cSBkY35quXTkPz6gbqCCH66CuA==", + "type": "RsaSignature2017" }, - "sensitive": false, - "summary": null, - "tag": [], "to": [ "https://www.w3.org/ns/activitystreams#Public" ], - "type": "Note", - "url": "https://chaos.social/@distantnative/109336635639931467" + "type": "Create" } diff --git a/test/pleroma/workers/receiver_worker_test.exs b/test/pleroma/workers/receiver_worker_test.exs index 085108e37..cb434f52e 100644 --- a/test/pleroma/workers/receiver_worker_test.exs +++ b/test/pleroma/workers/receiver_worker_test.exs @@ -128,23 +128,6 @@ defmodule Pleroma.Workers.ReceiverWorkerTest do test "it can validate the signature" do Tesla.Mock.mock(fn - %{url: "https://mastodon.social/users/bastianallgeier"} -> - %Tesla.Env{ - status: 200, - body: File.read!("test/fixtures/bastianallgeier.json"), - headers: [{"content-type", "application/activity+json"}] - } - - %{url: "https://mastodon.social/users/bastianallgeier/collections/featured"} -> - %Tesla.Env{ - status: 200, - headers: [{"content-type", "application/activity+json"}], - body: - File.read!("test/fixtures/users_mock/masto_featured.json") - |> String.replace("{{domain}}", "mastodon.social") - |> String.replace("{{nickname}}", "bastianallgeier") - } - %{url: "https://phpc.social/users/denniskoch"} -> %Tesla.Env{ status: 200, @@ -161,136 +144,10 @@ defmodule Pleroma.Workers.ReceiverWorkerTest do |> String.replace("{{domain}}", "phpc.social") |> String.replace("{{nickname}}", "denniskoch") } - - %{url: "https://mastodon.social/users/bastianallgeier/statuses/112846516276907281"} -> - %Tesla.Env{ - status: 200, - headers: [{"content-type", "application/activity+json"}], - body: File.read!("test/fixtures/receiver_worker_signature_activity.json") - } end) - params = %{ - "@context" => [ - "https://www.w3.org/ns/activitystreams", - "https://w3id.org/security/v1", - %{ - "claim" => %{"@id" => "toot:claim", "@type" => "@id"}, - "memorial" => "toot:memorial", - "atomUri" => "ostatus:atomUri", - "manuallyApprovesFollowers" => "as:manuallyApprovesFollowers", - "blurhash" => "toot:blurhash", - "ostatus" => "http://ostatus.org#", - "discoverable" => "toot:discoverable", - "focalPoint" => %{"@container" => "@list", "@id" => "toot:focalPoint"}, - "votersCount" => "toot:votersCount", - "Hashtag" => "as:Hashtag", - "Emoji" => "toot:Emoji", - "alsoKnownAs" => %{"@id" => "as:alsoKnownAs", "@type" => "@id"}, - "sensitive" => "as:sensitive", - "movedTo" => %{"@id" => "as:movedTo", "@type" => "@id"}, - "inReplyToAtomUri" => "ostatus:inReplyToAtomUri", - "conversation" => "ostatus:conversation", - "Device" => "toot:Device", - "schema" => "http://schema.org#", - "toot" => "http://joinmastodon.org/ns#", - "cipherText" => "toot:cipherText", - "suspended" => "toot:suspended", - "messageType" => "toot:messageType", - "featuredTags" => %{"@id" => "toot:featuredTags", "@type" => "@id"}, - "Curve25519Key" => "toot:Curve25519Key", - "deviceId" => "toot:deviceId", - "Ed25519Signature" => "toot:Ed25519Signature", - "featured" => %{"@id" => "toot:featured", "@type" => "@id"}, - "devices" => %{"@id" => "toot:devices", "@type" => "@id"}, - "value" => "schema:value", - "PropertyValue" => "schema:PropertyValue", - "messageFranking" => "toot:messageFranking", - "publicKeyBase64" => "toot:publicKeyBase64", - "identityKey" => %{"@id" => "toot:identityKey", "@type" => "@id"}, - "Ed25519Key" => "toot:Ed25519Key", - "indexable" => "toot:indexable", - "EncryptedMessage" => "toot:EncryptedMessage", - "fingerprintKey" => %{"@id" => "toot:fingerprintKey", "@type" => "@id"} - } - ], - "actor" => "https://phpc.social/users/denniskoch", - "cc" => [ - "https://phpc.social/users/denniskoch/followers", - "https://mastodon.social/users/bastianallgeier", - "https://chaos.social/users/distantnative", - "https://fosstodon.org/users/kev" - ], - "id" => "https://phpc.social/users/denniskoch/statuses/112847382711461301/activity", - "object" => %{ - "atomUri" => "https://phpc.social/users/denniskoch/statuses/112847382711461301", - "attachment" => [], - "attributedTo" => "https://phpc.social/users/denniskoch", - "cc" => [ - "https://phpc.social/users/denniskoch/followers", - "https://mastodon.social/users/bastianallgeier", - "https://chaos.social/users/distantnative", - "https://fosstodon.org/users/kev" - ], - "content" => - "

@bastianallgeier @distantnative @kev Another main argument: Discord is popular. Many people have an account, so you can just join an server quickly. Also you know the app and how to get around.

", - "contentMap" => %{ - "en" => - "

@bastianallgeier @distantnative @kev Another main argument: Discord is popular. Many people have an account, so you can just join an server quickly. Also you know the app and how to get around.

" - }, - "conversation" => - "tag:mastodon.social,2024-07-25:objectId=760068442:objectType=Conversation", - "id" => "https://phpc.social/users/denniskoch/statuses/112847382711461301", - "inReplyTo" => - "https://mastodon.social/users/bastianallgeier/statuses/112846516276907281", - "inReplyToAtomUri" => - "https://mastodon.social/users/bastianallgeier/statuses/112846516276907281", - "published" => "2024-07-25T13:33:29Z", - "replies" => %{ - "first" => %{ - "items" => [], - "next" => - "https://phpc.social/users/denniskoch/statuses/112847382711461301/replies?only_other_accounts=true&page=true", - "partOf" => - "https://phpc.social/users/denniskoch/statuses/112847382711461301/replies", - "type" => "CollectionPage" - }, - "id" => "https://phpc.social/users/denniskoch/statuses/112847382711461301/replies", - "type" => "Collection" - }, - "sensitive" => false, - "tag" => [ - %{ - "href" => "https://mastodon.social/users/bastianallgeier", - "name" => "@bastianallgeier@mastodon.social", - "type" => "Mention" - }, - %{ - "href" => "https://chaos.social/users/distantnative", - "name" => "@distantnative@chaos.social", - "type" => "Mention" - }, - %{ - "href" => "https://fosstodon.org/users/kev", - "name" => "@kev@fosstodon.org", - "type" => "Mention" - } - ], - "to" => ["https://www.w3.org/ns/activitystreams#Public"], - "type" => "Note", - "url" => "https://phpc.social/@denniskoch/112847382711461301" - }, - "published" => "2024-07-25T13:33:29Z", - "signature" => %{ - "created" => "2024-07-25T13:33:29Z", - "creator" => "https://phpc.social/users/denniskoch#main-key", - "signatureValue" => - "slz9BKJzd2n1S44wdXGOU+bV/wsskdgAaUpwxj8R16mYOL8+DTpE6VnfSKoZGsBBJT8uG5gnVfVEz1YsTUYtymeUgLMh7cvd8VnJnZPS+oixbmBRVky/Myf91TEgQQE7G4vDmTdB4ii54hZrHcOOYYf5FKPNRSkMXboKA6LMqNtekhbI+JTUJYIB02WBBK6PUyo15f6B1RJ6HGWVgud9NE0y1EZXfrkqUt682p8/9D49ORf7AwjXUJibKic2RbPvhEBj70qUGfBm4vvgdWhSUn1IG46xh+U0+NrTSUED82j1ZVOeua/2k/igkGs8cSBkY35quXTkPz6gbqCCH66CuA==", - "type" => "RsaSignature2017" - }, - "to" => ["https://www.w3.org/ns/activitystreams#Public"], - "type" => "Create" - } + params = + File.read!("test/fixtures/receiver_worker_signature_activity.json") |> Jason.decode!() req_headers = [ ["accept-encoding", "gzip"], From 3dadb9ed086fb63a3e664a43be3bf30f9ffbfb2d Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 28 Aug 2024 16:37:46 -0400 Subject: [PATCH 023/117] Changelog --- changelog.d/oban-recevier-user-error.fix | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/oban-recevier-user-error.fix diff --git a/changelog.d/oban-recevier-user-error.fix b/changelog.d/oban-recevier-user-error.fix new file mode 100644 index 000000000..1ed0c5bb1 --- /dev/null +++ b/changelog.d/oban-recevier-user-error.fix @@ -0,0 +1 @@ +ReceiverWorker will cancel processing jobs instead of retrying if the user cannot be fetched due to 403, 404, or 410 errors. From bb2f4a76b3af4ad5f0e2950ef8dc2567c6ad69ff Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 28 Aug 2024 17:01:30 -0400 Subject: [PATCH 024/117] Add test for origin containment failures --- test/pleroma/workers/receiver_worker_test.exs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/test/pleroma/workers/receiver_worker_test.exs b/test/pleroma/workers/receiver_worker_test.exs index cb434f52e..995f765a1 100644 --- a/test/pleroma/workers/receiver_worker_test.exs +++ b/test/pleroma/workers/receiver_worker_test.exs @@ -177,4 +177,21 @@ defmodule Pleroma.Workers.ReceiverWorkerTest do assert {:ok, %Pleroma.Activity{}} = ReceiverWorker.perform(oban_job) end + + test "cancels due to origin containment" do + params = + insert(:note_activity).data + |> Map.put("id", "https://notorigindomain.com/activity") + + {:ok, oban_job} = + Federator.incoming_ap_doc(%{ + method: "POST", + req_headers: [], + request_path: "/inbox", + params: params, + query_string: "" + }) + + assert {:cancel, :origin_containment_failed} = ReceiverWorker.perform(oban_job) + end end From 6ae629cfe072d236453d256017618fe9a8c44755 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 28 Aug 2024 17:24:59 -0400 Subject: [PATCH 025/117] Cancel ReceiverWorker jobs if the user account has been disabled / deactivated --- lib/pleroma/workers/receiver_worker.ex | 4 ++- test/pleroma/workers/receiver_worker_test.exs | 26 +++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/workers/receiver_worker.ex b/lib/pleroma/workers/receiver_worker.ex index 7dce02a5f..80518f6fd 100644 --- a/lib/pleroma/workers/receiver_worker.ex +++ b/lib/pleroma/workers/receiver_worker.ex @@ -33,7 +33,8 @@ defmodule Pleroma.Workers.ReceiverWorker do query_string: query_string } - with {:ok, %User{} = _actor} <- User.get_or_fetch_by_ap_id(conn_data.params["actor"]), + with {:ok, %User{} = actor} <- User.get_or_fetch_by_ap_id(conn_data.params["actor"]), + {:user_active, true} <- {:user_active, match?(true, actor.is_active)}, {:ok, _public_key} <- Signature.refetch_public_key(conn_data), {:signature, true} <- {:signature, Signature.validate_signature(conn_data)}, {:ok, res} <- Federator.perform(:incoming_ap_doc, params) do @@ -70,6 +71,7 @@ defmodule Pleroma.Workers.ReceiverWorker do {:error, {:side_effects, {:error, :no_object_actor}} = reason} -> {:cancel, reason} {:error, :not_found} = reason -> {:cancel, reason} {:error, :forbidden} = reason -> {:cancel, reason} + {:user_active, false} = reason -> {:cancel, reason} {:error, _} = e -> e e -> {:error, e} end diff --git a/test/pleroma/workers/receiver_worker_test.exs b/test/pleroma/workers/receiver_worker_test.exs index 995f765a1..adf90ec86 100644 --- a/test/pleroma/workers/receiver_worker_test.exs +++ b/test/pleroma/workers/receiver_worker_test.exs @@ -9,6 +9,7 @@ defmodule Pleroma.Workers.ReceiverWorkerTest do import Mock import Pleroma.Factory + alias Pleroma.User alias Pleroma.Web.Federator alias Pleroma.Workers.ReceiverWorker @@ -124,6 +125,31 @@ defmodule Pleroma.Workers.ReceiverWorkerTest do assert {:cancel, {:error, :not_found}} = ReceiverWorker.perform(oban_job) end + + test "when user account is disabled" do + user = insert(:user) + + fake_activity = URI.parse(user.ap_id) |> Map.put(:path, "/fake-activity") |> to_string + + params = + insert(:note_activity, user: user).data + |> Map.put("id", fake_activity) + + {:ok, %User{}} = User.set_activation(user, false) + + {:ok, oban_job} = + ReceiverWorker.new(%{ + "op" => "incoming_ap_doc", + "method" => "POST", + "req_headers" => [], + "request_path" => "/inbox", + "params" => params, + "query_string" => "" + }) + |> Oban.insert() + + assert {:cancel, {:user_active, false}} = ReceiverWorker.perform(oban_job) + end end test "it can validate the signature" do From 2e9515578a689428027ca7084d5c9b0d0b4a60ba Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 28 Aug 2024 17:38:13 -0400 Subject: [PATCH 026/117] ReceiverWorker job canceled due to deleted object --- test/pleroma/workers/receiver_worker_test.exs | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/test/pleroma/workers/receiver_worker_test.exs b/test/pleroma/workers/receiver_worker_test.exs index adf90ec86..779e83eaa 100644 --- a/test/pleroma/workers/receiver_worker_test.exs +++ b/test/pleroma/workers/receiver_worker_test.exs @@ -220,4 +220,29 @@ defmodule Pleroma.Workers.ReceiverWorkerTest do assert {:cancel, :origin_containment_failed} = ReceiverWorker.perform(oban_job) end + + test "canceled due to deleted object" do + params = + insert(:announce_activity).data + |> Map.put("object", "http://localhost:4001/deleted") + + Tesla.Mock.mock(fn + %{url: "http://localhost:4001/deleted"} -> + %Tesla.Env{ + status: 404, + body: "" + } + end) + + {:ok, oban_job} = + Federator.incoming_ap_doc(%{ + method: "POST", + req_headers: [], + request_path: "/inbox", + params: params, + query_string: "" + }) + + assert {:cancel, _} = ReceiverWorker.perform(oban_job) + end end From 2346807ac93d5acb9901823cceaffe5c305c1e20 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 28 Aug 2024 17:44:33 -0400 Subject: [PATCH 027/117] Annotate error cases --- lib/pleroma/workers/receiver_worker.ex | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/lib/pleroma/workers/receiver_worker.ex b/lib/pleroma/workers/receiver_worker.ex index 80518f6fd..4b1f74a27 100644 --- a/lib/pleroma/workers/receiver_worker.ex +++ b/lib/pleroma/workers/receiver_worker.ex @@ -61,17 +61,21 @@ defmodule Pleroma.Workers.ReceiverWorker do defp process_errors(errors) do case errors do - {:error, :origin_containment_failed} -> {:cancel, :origin_containment_failed} - {:error, :already_present} -> {:cancel, :already_present} - {:error, {:validate_object, _} = reason} -> {:cancel, reason} - {:error, {:validate, {:error, _changeset} = reason}} -> {:cancel, reason} - {:error, {:reject, _} = reason} -> {:cancel, reason} - {:signature, false} -> {:cancel, :invalid_signature} - {:error, "Object has been deleted"} = reason -> {:cancel, reason} - {:error, {:side_effects, {:error, :no_object_actor}} = reason} -> {:cancel, reason} + # User fetch failures {:error, :not_found} = reason -> {:cancel, reason} {:error, :forbidden} = reason -> {:cancel, reason} + # Inactive user {:user_active, false} = reason -> {:cancel, reason} + # Validator will error and return a changeset error + # e.g., duplicate activities or if the object was deleted + {:error, {:validate, {:error, _changeset} = reason}} -> {:cancel, reason} + # MRFs will return a reject + {:error, {:reject, _} = reason} -> {:cancel, reason} + # HTTP Sigs + {:signature, false} -> {:cancel, :invalid_signature} + {:error, :origin_containment_failed} -> {:cancel, :origin_containment_failed} + {:error, {:validate_object, _} = reason} -> {:cancel, reason} + {:error, {:side_effects, {:error, :no_object_actor}} = reason} -> {:cancel, reason} {:error, _} = e -> e e -> {:error, e} end From 380a6a6df31a16a89f5c5cc497ddc1360cea3854 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 28 Aug 2024 17:45:31 -0400 Subject: [PATCH 028/117] :validate_object is not a real error returned from anywhere --- lib/pleroma/web/federator.ex | 5 ----- lib/pleroma/workers/receiver_worker.ex | 1 - 2 files changed, 6 deletions(-) diff --git a/lib/pleroma/web/federator.ex b/lib/pleroma/web/federator.ex index 2df716556..e812b1a46 100644 --- a/lib/pleroma/web/federator.ex +++ b/lib/pleroma/web/federator.ex @@ -121,11 +121,6 @@ defmodule Pleroma.Web.Federator do Logger.debug("Unhandled actor #{actor}, #{inspect(e)}") {:error, e} - {:error, {:validate_object, _}} = e -> - Logger.error("Incoming AP doc validation error: #{inspect(e)}") - Logger.debug(Jason.encode!(params, pretty: true)) - e - e -> # Just drop those for now Logger.debug(fn -> "Unhandled activity\n" <> Jason.encode!(params, pretty: true) end) diff --git a/lib/pleroma/workers/receiver_worker.ex b/lib/pleroma/workers/receiver_worker.ex index 4b1f74a27..c7e6bc5ea 100644 --- a/lib/pleroma/workers/receiver_worker.ex +++ b/lib/pleroma/workers/receiver_worker.ex @@ -74,7 +74,6 @@ defmodule Pleroma.Workers.ReceiverWorker do # HTTP Sigs {:signature, false} -> {:cancel, :invalid_signature} {:error, :origin_containment_failed} -> {:cancel, :origin_containment_failed} - {:error, {:validate_object, _} = reason} -> {:cancel, reason} {:error, {:side_effects, {:error, :no_object_actor}} = reason} -> {:cancel, reason} {:error, _} = e -> e e -> {:error, e} From c5ca806aa0023e25755947a3bf0d54242e45f65a Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 28 Aug 2024 17:57:34 -0400 Subject: [PATCH 029/117] Add back one of the duplicate checks to fix a test, document where it comes from --- lib/pleroma/workers/receiver_worker.ex | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/pleroma/workers/receiver_worker.ex b/lib/pleroma/workers/receiver_worker.ex index c7e6bc5ea..810fda67c 100644 --- a/lib/pleroma/workers/receiver_worker.ex +++ b/lib/pleroma/workers/receiver_worker.ex @@ -69,6 +69,8 @@ defmodule Pleroma.Workers.ReceiverWorker do # Validator will error and return a changeset error # e.g., duplicate activities or if the object was deleted {:error, {:validate, {:error, _changeset} = reason}} -> {:cancel, reason} + # Duplicate detection during Normalization + {:error, :already_present} -> {:cancel, :already_present} # MRFs will return a reject {:error, {:reject, _} = reason} -> {:cancel, reason} # HTTP Sigs From 8a3efa7152488460934c1fadc8ab86efd7d47c04 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 28 Aug 2024 18:02:35 -0400 Subject: [PATCH 030/117] More error annotations --- lib/pleroma/workers/receiver_worker.ex | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/pleroma/workers/receiver_worker.ex b/lib/pleroma/workers/receiver_worker.ex index 810fda67c..6787a59ef 100644 --- a/lib/pleroma/workers/receiver_worker.ex +++ b/lib/pleroma/workers/receiver_worker.ex @@ -75,8 +75,11 @@ defmodule Pleroma.Workers.ReceiverWorker do {:error, {:reject, _} = reason} -> {:cancel, reason} # HTTP Sigs {:signature, false} -> {:cancel, :invalid_signature} + # Origin / URL validation failed somewhere possibly due to spoofing {:error, :origin_containment_failed} -> {:cancel, :origin_containment_failed} + # Unclear if this can be reached {:error, {:side_effects, {:error, :no_object_actor}} = reason} -> {:cancel, reason} + # Catchall {:error, _} = e -> e e -> {:error, e} end From e498d252e44ddc1a85288b80dc65beefcd60edf2 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 28 Aug 2024 18:03:33 -0400 Subject: [PATCH 031/117] Changelog update --- changelog.d/oban-recevier-improvements.fix | 1 + changelog.d/oban-recevier-user-error.fix | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 changelog.d/oban-recevier-improvements.fix delete mode 100644 changelog.d/oban-recevier-user-error.fix diff --git a/changelog.d/oban-recevier-improvements.fix b/changelog.d/oban-recevier-improvements.fix new file mode 100644 index 000000000..f91502ed2 --- /dev/null +++ b/changelog.d/oban-recevier-improvements.fix @@ -0,0 +1 @@ +ReceiverWorker will cancel processing jobs instead of retrying if the user cannot be fetched due to 403, 404, or 410 errors or if the account is disabled locally. diff --git a/changelog.d/oban-recevier-user-error.fix b/changelog.d/oban-recevier-user-error.fix deleted file mode 100644 index 1ed0c5bb1..000000000 --- a/changelog.d/oban-recevier-user-error.fix +++ /dev/null @@ -1 +0,0 @@ -ReceiverWorker will cancel processing jobs instead of retrying if the user cannot be fetched due to 403, 404, or 410 errors. From 1821ef4f157980bdf64f7540ee5aa8e26fa3102e Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 28 Aug 2024 18:35:01 -0400 Subject: [PATCH 032/117] Move user active check into Federator.perform/1 --- lib/pleroma/web/federator.ex | 3 ++- lib/pleroma/workers/receiver_worker.ex | 5 ++--- test/pleroma/workers/receiver_worker_test.exs | 14 ++++++-------- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/lib/pleroma/web/federator.ex b/lib/pleroma/web/federator.ex index e812b1a46..58260afa8 100644 --- a/lib/pleroma/web/federator.ex +++ b/lib/pleroma/web/federator.ex @@ -102,7 +102,8 @@ defmodule Pleroma.Web.Federator do # NOTE: we use the actor ID to do the containment, this is fine because an # actor shouldn't be acting on objects outside their own AP server. - with {_, {:ok, _user}} <- {:actor, User.get_or_fetch_by_ap_id(actor)}, + with {_, {:ok, user}} <- {:actor, User.get_or_fetch_by_ap_id(actor)}, + {:user_active, true} <- {:user_active, match?(true, user.is_active)}, nil <- Activity.normalize(params["id"]), {_, :ok} <- {:correct_origin?, Containment.contain_origin_from_id(actor, params)}, diff --git a/lib/pleroma/workers/receiver_worker.ex b/lib/pleroma/workers/receiver_worker.ex index 6787a59ef..0373ec15f 100644 --- a/lib/pleroma/workers/receiver_worker.ex +++ b/lib/pleroma/workers/receiver_worker.ex @@ -33,8 +33,7 @@ defmodule Pleroma.Workers.ReceiverWorker do query_string: query_string } - with {:ok, %User{} = actor} <- User.get_or_fetch_by_ap_id(conn_data.params["actor"]), - {:user_active, true} <- {:user_active, match?(true, actor.is_active)}, + with {:ok, %User{}} <- User.get_or_fetch_by_ap_id(conn_data.params["actor"]), {:ok, _public_key} <- Signature.refetch_public_key(conn_data), {:signature, true} <- {:signature, Signature.validate_signature(conn_data)}, {:ok, res} <- Federator.perform(:incoming_ap_doc, params) do @@ -65,7 +64,7 @@ defmodule Pleroma.Workers.ReceiverWorker do {:error, :not_found} = reason -> {:cancel, reason} {:error, :forbidden} = reason -> {:cancel, reason} # Inactive user - {:user_active, false} = reason -> {:cancel, reason} + {:error, {:user_active, false} = reason} -> {:cancel, reason} # Validator will error and return a changeset error # e.g., duplicate activities or if the object was deleted {:error, {:validate, {:error, _changeset} = reason}} -> {:cancel, reason} diff --git a/test/pleroma/workers/receiver_worker_test.exs b/test/pleroma/workers/receiver_worker_test.exs index 779e83eaa..4d53c44ed 100644 --- a/test/pleroma/workers/receiver_worker_test.exs +++ b/test/pleroma/workers/receiver_worker_test.exs @@ -138,15 +138,13 @@ defmodule Pleroma.Workers.ReceiverWorkerTest do {:ok, %User{}} = User.set_activation(user, false) {:ok, oban_job} = - ReceiverWorker.new(%{ - "op" => "incoming_ap_doc", - "method" => "POST", - "req_headers" => [], - "request_path" => "/inbox", - "params" => params, - "query_string" => "" + Federator.incoming_ap_doc(%{ + method: "POST", + req_headers: [], + request_path: "/inbox", + params: params, + query_string: "" }) - |> Oban.insert() assert {:cancel, {:user_active, false}} = ReceiverWorker.perform(oban_job) end From 0bf82a1745a38a3752f5b7df645a7d266b8fd9c8 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 28 Aug 2024 19:50:51 -0400 Subject: [PATCH 033/117] Add an AdapterHelper for Finch so we can support streaming request bodies --- lib/pleroma/http/adapter_helper.ex | 2 ++ lib/pleroma/http/adapter_helper/finch.ex | 33 ++++++++++++++++++++++++ 2 files changed, 35 insertions(+) create mode 100644 lib/pleroma/http/adapter_helper/finch.ex diff --git a/lib/pleroma/http/adapter_helper.ex b/lib/pleroma/http/adapter_helper.ex index 4dbcccdcc..be00ba78a 100644 --- a/lib/pleroma/http/adapter_helper.ex +++ b/lib/pleroma/http/adapter_helper.ex @@ -52,6 +52,7 @@ defmodule Pleroma.HTTP.AdapterHelper do case adapter() do Tesla.Adapter.Gun -> AdapterHelper.Gun Tesla.Adapter.Hackney -> AdapterHelper.Hackney + {Tesla.Adapter.Finch, _} -> AdapterHelper.Finch _ -> AdapterHelper.Default end end @@ -124,6 +125,7 @@ defmodule Pleroma.HTTP.AdapterHelper do def can_stream? do case Application.get_env(:tesla, :adapter) do Tesla.Adapter.Gun -> true + {Tesla.Adapter.Finch, _} -> true _ -> false end end diff --git a/lib/pleroma/http/adapter_helper/finch.ex b/lib/pleroma/http/adapter_helper/finch.ex new file mode 100644 index 000000000..10a988901 --- /dev/null +++ b/lib/pleroma/http/adapter_helper/finch.ex @@ -0,0 +1,33 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.HTTP.AdapterHelper.Finch do + @behaviour Pleroma.HTTP.AdapterHelper + + alias Pleroma.Config + alias Pleroma.HTTP.AdapterHelper + + @spec options(keyword(), URI.t()) :: keyword() + def options(incoming_opts \\ [], %URI{} = _uri) do + proxy = + [:http, :proxy_url] + |> Config.get() + |> AdapterHelper.format_proxy() + + config_opts = Config.get([:http, :adapter], []) + + config_opts + |> Keyword.merge(incoming_opts) + |> AdapterHelper.maybe_add_proxy(proxy) + |> maybe_stream() + end + + # Tesla Finch adapter uses response: :stream + defp maybe_stream(opts) do + case Keyword.pop(opts, :stream, nil) do + {true, opts} -> Keyword.put(opts, :response, :stream) + {_, opts} -> opts + end + end +end From 8ab4dd20dfdd0cc92c18ade7d84bfb5364785a15 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 28 Aug 2024 19:52:29 -0400 Subject: [PATCH 034/117] Update comments, remove solved TODO --- lib/pleroma/http/adapter_helper.ex | 1 - lib/pleroma/http/adapter_helper/finch.ex | 2 +- lib/pleroma/http/adapter_helper/gun.ex | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/http/adapter_helper.ex b/lib/pleroma/http/adapter_helper.ex index be00ba78a..32c1080f7 100644 --- a/lib/pleroma/http/adapter_helper.ex +++ b/lib/pleroma/http/adapter_helper.ex @@ -120,7 +120,6 @@ defmodule Pleroma.HTTP.AdapterHelper do end end - # TODO add Finch support once we have an AdapterHelper for it @spec can_stream? :: bool() def can_stream? do case Application.get_env(:tesla, :adapter) do diff --git a/lib/pleroma/http/adapter_helper/finch.ex b/lib/pleroma/http/adapter_helper/finch.ex index 10a988901..181caed7e 100644 --- a/lib/pleroma/http/adapter_helper/finch.ex +++ b/lib/pleroma/http/adapter_helper/finch.ex @@ -23,7 +23,7 @@ defmodule Pleroma.HTTP.AdapterHelper.Finch do |> maybe_stream() end - # Tesla Finch adapter uses response: :stream + # Finch uses [response: :stream] defp maybe_stream(opts) do case Keyword.pop(opts, :stream, nil) do {true, opts} -> Keyword.put(opts, :response, :stream) diff --git a/lib/pleroma/http/adapter_helper/gun.ex b/lib/pleroma/http/adapter_helper/gun.ex index f9a8180f2..30ba26765 100644 --- a/lib/pleroma/http/adapter_helper/gun.ex +++ b/lib/pleroma/http/adapter_helper/gun.ex @@ -48,7 +48,7 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do Keyword.put(opts, :timeout, recv_timeout) end - # Tesla Gun adapter uses body_as: :stream + # Gun uses [body_as: :stream] defp maybe_stream(opts) do case Keyword.pop(opts, :stream, nil) do {true, opts} -> Keyword.put(opts, :body_as, :stream) From d01569822e0dc45349c321ad306f6e19b4e967af Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 28 Aug 2024 19:56:09 -0400 Subject: [PATCH 035/117] Changelog --- changelog.d/rich-media-no-heads.change | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/rich-media-no-heads.change diff --git a/changelog.d/rich-media-no-heads.change b/changelog.d/rich-media-no-heads.change new file mode 100644 index 000000000..0bab323aa --- /dev/null +++ b/changelog.d/rich-media-no-heads.change @@ -0,0 +1 @@ +Rich Media preview fetching will skip making an HTTP HEAD request to check a URL for allowed content type and length if the Tesla adapter is Gun or Finch From c17a78c55a6b288c271923f730dc69aaf27e6556 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 29 Aug 2024 09:37:10 -0400 Subject: [PATCH 036/117] Rich Media: add stream byte counting as an extra protection against malicious URLs --- lib/pleroma/web/rich_media/helpers.ex | 34 +++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/web/rich_media/helpers.ex b/lib/pleroma/web/rich_media/helpers.ex index a242ca640..d4be97957 100644 --- a/lib/pleroma/web/rich_media/helpers.ex +++ b/lib/pleroma/web/rich_media/helpers.ex @@ -23,7 +23,7 @@ defmodule Pleroma.Web.RichMedia.Helpers do {:get, Pleroma.HTTP.get(url, req_headers(), http_options())}, {_, :ok} <- {:content_type, check_content_type(headers)}, {_, :ok} <- {:content_length, check_content_length(headers)}, - body <- Enum.into(stream_body, <<>>) do + {:read_stream, {:ok, body}} <- {:read_stream, read_stream(stream_body)} do {:ok, body} end end @@ -52,8 +52,12 @@ defmodule Pleroma.Web.RichMedia.Helpers do Logger.debug("Rich media error for #{url}: content-type is #{type}") {:error, :content_type} - {:content_length, {_, length}} -> - Logger.debug("Rich media error for #{url}: content-length is #{length}") + {:content_length, :error} -> + Logger.debug("Rich media error for #{url}: content-length exceeded") + {:error, :body_too_large} + + {:read_stream, :error} -> + Logger.debug("Rich media error for #{url}: content-length exceeded") {:error, :body_too_large} {:get, _} -> @@ -82,7 +86,7 @@ defmodule Pleroma.Web.RichMedia.Helpers do {_, maybe_content_length} -> case Integer.parse(maybe_content_length) do {content_length, ""} when content_length <= max_body -> :ok - {_, ""} -> {:error, maybe_content_length} + {_, ""} -> :error _ -> :ok end @@ -91,6 +95,28 @@ defmodule Pleroma.Web.RichMedia.Helpers do end end + defp read_stream(stream) do + max_body = Keyword.get(http_options(), :max_body) + + try do + result = + Stream.transform(stream, 0, fn chunk, total_bytes -> + new_total = total_bytes + byte_size(chunk) + + if new_total > max_body do + raise("Exceeds max body limit of #{max_body}") + else + {[chunk], new_total} + end + end) + |> Enum.into(<<>>) + + {:ok, result} + rescue + _ -> :error + end + end + defp http_options do [ pool: :rich_media, From ceffb8a8918b83d482e9c1da64fec22b428a61f3 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 23 Aug 2024 13:52:19 -0400 Subject: [PATCH 037/117] Drop incoming Delete activities from unknown actors --- changelog.d/drop-unknown-deletes.change | 1 + lib/pleroma/workers/receiver_worker.ex | 16 +++++++++++++- test/pleroma/workers/receiver_worker_test.exs | 22 +++++++++++++++++++ 3 files changed, 38 insertions(+), 1 deletion(-) create mode 100644 changelog.d/drop-unknown-deletes.change diff --git a/changelog.d/drop-unknown-deletes.change b/changelog.d/drop-unknown-deletes.change new file mode 100644 index 000000000..0be7f5450 --- /dev/null +++ b/changelog.d/drop-unknown-deletes.change @@ -0,0 +1 @@ +Drop incoming Delete activities from unknown actors diff --git a/lib/pleroma/workers/receiver_worker.ex b/lib/pleroma/workers/receiver_worker.ex index d4db97b63..ea86a3a12 100644 --- a/lib/pleroma/workers/receiver_worker.ex +++ b/lib/pleroma/workers/receiver_worker.ex @@ -33,7 +33,8 @@ defmodule Pleroma.Workers.ReceiverWorker do query_string: query_string } - with {:ok, %User{} = _actor} <- User.get_or_fetch_by_ap_id(conn_data.params["actor"]), + with {_, false} <- {:unknown_delete, unknown_delete?(params)}, + User.get_or_fetch_by_ap_id(conn_data.params["actor"]), {:ok, _public_key} <- Signature.refetch_public_key(conn_data), {:signature, true} <- {:signature, Signature.validate_signature(conn_data)}, {:ok, res} <- Federator.perform(:incoming_ap_doc, params) do @@ -58,6 +59,7 @@ defmodule Pleroma.Workers.ReceiverWorker do defp process_errors(errors) do case errors do + {:unknown_delete, true} -> {:cancel, "Delete from unknown actor"} {:error, :origin_containment_failed} -> {:cancel, :origin_containment_failed} {:error, :already_present} -> {:cancel, :already_present} {:error, {:validate_object, _} = reason} -> {:cancel, reason} @@ -71,4 +73,16 @@ defmodule Pleroma.Workers.ReceiverWorker do e -> {:error, e} end end + + defp unknown_delete?(%{ + "type" => "Delete", + "actor" => actor + }) do + case User.get_cached_by_ap_id(actor) do + %User{} -> false + _ -> true + end + end + + defp unknown_delete?(_), do: false end diff --git a/test/pleroma/workers/receiver_worker_test.exs b/test/pleroma/workers/receiver_worker_test.exs index 33be91085..91fbb1fe8 100644 --- a/test/pleroma/workers/receiver_worker_test.exs +++ b/test/pleroma/workers/receiver_worker_test.exs @@ -245,4 +245,26 @@ defmodule Pleroma.Workers.ReceiverWorkerTest do assert {:ok, %Pleroma.Activity{}} = ReceiverWorker.perform(oban_job) end + + # When activity is delivered to the inbox and we cannot immediately verify signature + # we capture all the params and process it later in the Oban job. + # This requires we replicate the same scenario by including additional fields in the params + test "Deletes cancelled for an unknown actor" do + params = %{ + "type" => "Delete", + "actor" => "https://unknown.mastodon.instance/users/somebody" + } + + assert {:cancel, "Delete from unknown actor"} = + ReceiverWorker.perform(%Oban.Job{ + args: %{ + "op" => "incoming_ap_doc", + "method" => :post, + "req_headers" => [], + "request_path" => "/inbox", + "query_string" => "", + "params" => params + } + }) + end end From 4bc6f334f49c27e1f466052fee6fa2f3d5d2ee74 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 23 Aug 2024 14:18:04 -0400 Subject: [PATCH 038/117] Revert unintentional change --- lib/pleroma/workers/receiver_worker.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/workers/receiver_worker.ex b/lib/pleroma/workers/receiver_worker.ex index ea86a3a12..4033fec0f 100644 --- a/lib/pleroma/workers/receiver_worker.ex +++ b/lib/pleroma/workers/receiver_worker.ex @@ -34,7 +34,7 @@ defmodule Pleroma.Workers.ReceiverWorker do } with {_, false} <- {:unknown_delete, unknown_delete?(params)}, - User.get_or_fetch_by_ap_id(conn_data.params["actor"]), + {:ok, %User{} = _actor} <- User.get_or_fetch_by_ap_id(conn_data.params["actor"]), {:ok, _public_key} <- Signature.refetch_public_key(conn_data), {:signature, true} <- {:signature, Signature.validate_signature(conn_data)}, {:ok, res} <- Federator.perform(:incoming_ap_doc, params) do From 1c394dd18c5d61dc716a9b9cda981a359de32456 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 23 Aug 2024 14:24:04 -0400 Subject: [PATCH 039/117] Move the check to the inbox --- .../activity_pub/activity_pub_controller.ex | 32 +++++++++++++++---- lib/pleroma/workers/receiver_worker.ex | 15 +-------- 2 files changed, 26 insertions(+), 21 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index cdd054e1a..a24dcfc9c 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -294,13 +294,19 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do end def inbox(%{assigns: %{valid_signature: false}} = conn, params) do - Federator.incoming_ap_doc(%{ - method: conn.method, - req_headers: conn.req_headers, - request_path: conn.request_path, - params: params, - query_string: conn.query_string - }) + case unknown_delete?(params) do + true -> + :ok + + false -> + Federator.incoming_ap_doc(%{ + method: conn.method, + req_headers: conn.req_headers, + request_path: conn.request_path, + params: params, + query_string: conn.query_string + }) + end json(conn, "ok") end @@ -558,4 +564,16 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do |> json(UserView.render("featured.json", %{user: user})) end end + + defp unknown_delete?(%{ + "type" => "Delete", + "actor" => actor + }) do + case User.get_cached_by_ap_id(actor) do + %User{} -> false + _ -> true + end + end + + defp unknown_delete?(_), do: false end diff --git a/lib/pleroma/workers/receiver_worker.ex b/lib/pleroma/workers/receiver_worker.ex index 4033fec0f..ce92ef9dd 100644 --- a/lib/pleroma/workers/receiver_worker.ex +++ b/lib/pleroma/workers/receiver_worker.ex @@ -33,8 +33,7 @@ defmodule Pleroma.Workers.ReceiverWorker do query_string: query_string } - with {_, false} <- {:unknown_delete, unknown_delete?(params)}, - {:ok, %User{} = _actor} <- User.get_or_fetch_by_ap_id(conn_data.params["actor"]), + with {:ok, %User{} = _actor} <- User.get_or_fetch_by_ap_id(conn_data.params["actor"]), {:ok, _public_key} <- Signature.refetch_public_key(conn_data), {:signature, true} <- {:signature, Signature.validate_signature(conn_data)}, {:ok, res} <- Federator.perform(:incoming_ap_doc, params) do @@ -73,16 +72,4 @@ defmodule Pleroma.Workers.ReceiverWorker do e -> {:error, e} end end - - defp unknown_delete?(%{ - "type" => "Delete", - "actor" => actor - }) do - case User.get_cached_by_ap_id(actor) do - %User{} -> false - _ -> true - end - end - - defp unknown_delete?(_), do: false end From 27fcc421719062d5de9bf4dc90f3349595eb278d Mon Sep 17 00:00:00 2001 From: feld Date: Sat, 24 Aug 2024 16:53:22 +0000 Subject: [PATCH 040/117] Use Pleroma.Object.Containment.get_actor/1 to reliably find the actor of an incoming activity or object --- lib/pleroma/web/activity_pub/activity_pub_controller.ex | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index a24dcfc9c..77ec26f14 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -567,9 +567,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do defp unknown_delete?(%{ "type" => "Delete", - "actor" => actor - }) do - case User.get_cached_by_ap_id(actor) do + } = data) do + case data |> Pleroma.Object.Containment.get_actor() |> User.get_cached_by_ap_id() do %User{} -> false _ -> true end From 7bcc21ad6f1fdf9dbc16990e9891f9de7a21011d Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 24 Aug 2024 13:01:28 -0400 Subject: [PATCH 041/117] Switch test to the inbox --- .../activity_pub_controller_test.exs | 21 ++++++++++++++++++ test/pleroma/workers/receiver_worker_test.exs | 22 ------------------- 2 files changed, 21 insertions(+), 22 deletions(-) diff --git a/test/pleroma/web/activity_pub/activity_pub_controller_test.exs b/test/pleroma/web/activity_pub/activity_pub_controller_test.exs index af1a32fed..b9067533c 100644 --- a/test/pleroma/web/activity_pub/activity_pub_controller_test.exs +++ b/test/pleroma/web/activity_pub/activity_pub_controller_test.exs @@ -684,6 +684,27 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do |> json_response(400) end + # When activity is delivered to the inbox and we cannot immediately verify signature + # we capture all the params and process it later in the Oban job. + # Once we begin processing it through Oban we risk fetching the actor to validate the + # activity which just leads to inserting a new user to process a Delete not relevant to us. + test "Deletes from an unknown actor are discarded", %{conn: conn} do + params = + %{ + "type" => "Delete", + "actor" => "https://unknown.mastodon.instance/users/somebody" + } + |> Jason.encode!() + + conn + |> assign(:valid_signature, false) + |> put_req_header("content-type", "application/activity+json") + |> post("/inbox", params) + |> json_response(200) + + assert all_enqueued() == [] + end + test "accepts Add/Remove activities", %{conn: conn} do object_id = "c61d6733-e256-4fe1-ab13-1e369789423f" diff --git a/test/pleroma/workers/receiver_worker_test.exs b/test/pleroma/workers/receiver_worker_test.exs index 91fbb1fe8..33be91085 100644 --- a/test/pleroma/workers/receiver_worker_test.exs +++ b/test/pleroma/workers/receiver_worker_test.exs @@ -245,26 +245,4 @@ defmodule Pleroma.Workers.ReceiverWorkerTest do assert {:ok, %Pleroma.Activity{}} = ReceiverWorker.perform(oban_job) end - - # When activity is delivered to the inbox and we cannot immediately verify signature - # we capture all the params and process it later in the Oban job. - # This requires we replicate the same scenario by including additional fields in the params - test "Deletes cancelled for an unknown actor" do - params = %{ - "type" => "Delete", - "actor" => "https://unknown.mastodon.instance/users/somebody" - } - - assert {:cancel, "Delete from unknown actor"} = - ReceiverWorker.perform(%Oban.Job{ - args: %{ - "op" => "incoming_ap_doc", - "method" => :post, - "req_headers" => [], - "request_path" => "/inbox", - "query_string" => "", - "params" => params - } - }) - end end From 06deacd58e6aa676847530f24c6799162ed06777 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 24 Aug 2024 20:03:50 -0400 Subject: [PATCH 042/117] Formatting --- lib/pleroma/web/activity_pub/activity_pub_controller.ex | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index 77ec26f14..4c83d1927 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -565,9 +565,11 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do end end - defp unknown_delete?(%{ - "type" => "Delete", - } = data) do + defp unknown_delete?( + %{ + "type" => "Delete" + } = data + ) do case data |> Pleroma.Object.Containment.get_actor() |> User.get_cached_by_ap_id() do %User{} -> false _ -> true From 16a9b34876f3a9289c02253e802bffdac4901ac0 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 29 Aug 2024 14:16:59 -0400 Subject: [PATCH 043/117] Convert to an Plug called InboxGuard --- lib/pleroma/constants.ex | 12 ++++ .../activity_pub/activity_pub_controller.ex | 33 ++-------- lib/pleroma/web/plugs/inbox_guard_plug.ex | 66 +++++++++++++++++++ lib/pleroma/web/router.ex | 6 +- .../activity_pub_controller_test.exs | 2 +- 5 files changed, 91 insertions(+), 28 deletions(-) create mode 100644 lib/pleroma/web/plugs/inbox_guard_plug.ex diff --git a/lib/pleroma/constants.ex b/lib/pleroma/constants.ex index 3a5e35301..f969bfdbb 100644 --- a/lib/pleroma/constants.ex +++ b/lib/pleroma/constants.ex @@ -85,6 +85,18 @@ defmodule Pleroma.Constants do ] ) + const(allowed_activity_types_from_strangers, + do: [ + "Block", + "Create", + "Flag", + "Follow", + "Like", + "Move", + "React" + ] + ) + # basic regex, just there to weed out potential mistakes # https://datatracker.ietf.org/doc/html/rfc2045#section-5.1 const(mime_regex, diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index 4c83d1927..cdd054e1a 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -294,19 +294,13 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do end def inbox(%{assigns: %{valid_signature: false}} = conn, params) do - case unknown_delete?(params) do - true -> - :ok - - false -> - Federator.incoming_ap_doc(%{ - method: conn.method, - req_headers: conn.req_headers, - request_path: conn.request_path, - params: params, - query_string: conn.query_string - }) - end + Federator.incoming_ap_doc(%{ + method: conn.method, + req_headers: conn.req_headers, + request_path: conn.request_path, + params: params, + query_string: conn.query_string + }) json(conn, "ok") end @@ -564,17 +558,4 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do |> json(UserView.render("featured.json", %{user: user})) end end - - defp unknown_delete?( - %{ - "type" => "Delete" - } = data - ) do - case data |> Pleroma.Object.Containment.get_actor() |> User.get_cached_by_ap_id() do - %User{} -> false - _ -> true - end - end - - defp unknown_delete?(_), do: false end diff --git a/lib/pleroma/web/plugs/inbox_guard_plug.ex b/lib/pleroma/web/plugs/inbox_guard_plug.ex new file mode 100644 index 000000000..643b586d4 --- /dev/null +++ b/lib/pleroma/web/plugs/inbox_guard_plug.ex @@ -0,0 +1,66 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.Plugs.InboxGuardPlug do + import Plug.Conn + import Pleroma.Constants, only: [allowed_activity_types_from_strangers: 0] + + alias Pleroma.Config + alias Pleroma.User + + def init(options) do + options + end + + def call(%{assigns: %{valid_signature: true}} = conn, _opts) do + conn + end + + def call(conn, _opts) do + with {_, true} <- {:federating, Config.get!([:instance, :federating])}, + true <- known_actor?(conn) do + conn + else + {:federating, false} -> + conn + |> json(403, "Not federating") + + _ -> + conn + |> filter_from_strangers() + end + end + + # If signature failed but we know this actor we should + # accept it as we may only need to refetch their public key + # during processing + defp known_actor?(%{body_params: data}) do + case Pleroma.Object.Containment.get_actor(data) |> User.get_cached_by_ap_id() do + %User{} -> true + _ -> false + end + end + + # Only permit a subset of activity types from strangers + # or else it will add actors you've never interacted with + # to the database + defp filter_from_strangers(%{body_params: %{"type" => type}} = conn) do + with true <- type in allowed_activity_types_from_strangers() do + conn + else + _ -> + conn + |> json(400, "Invalid activity type for an unknown actor") + end + end + + defp json(conn, status, resp) do + json_resp = Jason.encode!(resp) + + conn + |> put_resp_content_type("application/json") + |> resp(status, json_resp) + |> halt() + end +end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 6492e3861..9b9ee421c 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -217,6 +217,10 @@ defmodule Pleroma.Web.Router do plug(Pleroma.Web.Plugs.MappedSignatureToIdentityPlug) end + pipeline :inbox_guard do + plug(Pleroma.Web.Plugs.InboxGuardPlug) + end + pipeline :static_fe do plug(Pleroma.Web.Plugs.StaticFEPlug) end @@ -920,7 +924,7 @@ defmodule Pleroma.Web.Router do end scope "/", Pleroma.Web.ActivityPub do - pipe_through(:activitypub) + pipe_through([:activitypub, :inbox_guard]) post("/inbox", ActivityPubController, :inbox) post("/users/:nickname/inbox", ActivityPubController, :inbox) end diff --git a/test/pleroma/web/activity_pub/activity_pub_controller_test.exs b/test/pleroma/web/activity_pub/activity_pub_controller_test.exs index b9067533c..1b959f324 100644 --- a/test/pleroma/web/activity_pub/activity_pub_controller_test.exs +++ b/test/pleroma/web/activity_pub/activity_pub_controller_test.exs @@ -700,7 +700,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do |> assign(:valid_signature, false) |> put_req_header("content-type", "application/activity+json") |> post("/inbox", params) - |> json_response(200) + |> json_response(400) assert all_enqueued() == [] end From e2cdae2c88eb22588209923d83c2a9c52d16c48c Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 29 Aug 2024 14:23:07 -0400 Subject: [PATCH 044/117] Change relay inbox response when not federating to a 403 for consistency --- lib/pleroma/web/activity_pub/activity_pub_controller.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index cdd054e1a..a08eda5f4 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -311,7 +311,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do post_inbox_relayed_create(conn, params) else conn - |> put_status(:bad_request) + |> put_status(403) |> json("Not federating") end end From 990b2058df11cc32f260d0ffcf7dd0f342d435b4 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 29 Aug 2024 14:30:23 -0400 Subject: [PATCH 045/117] Remove unnecessary error match in ReceiverWorker --- lib/pleroma/workers/receiver_worker.ex | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/pleroma/workers/receiver_worker.ex b/lib/pleroma/workers/receiver_worker.ex index ce92ef9dd..d4db97b63 100644 --- a/lib/pleroma/workers/receiver_worker.ex +++ b/lib/pleroma/workers/receiver_worker.ex @@ -58,7 +58,6 @@ defmodule Pleroma.Workers.ReceiverWorker do defp process_errors(errors) do case errors do - {:unknown_delete, true} -> {:cancel, "Delete from unknown actor"} {:error, :origin_containment_failed} -> {:cancel, :origin_containment_failed} {:error, :already_present} -> {:cancel, :already_present} {:error, {:validate_object, _} = reason} -> {:cancel, reason} From 2b39956acbc3ccd87a43cd4ddbd5976adcac5936 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 29 Aug 2024 14:40:31 -0400 Subject: [PATCH 046/117] Fix test title to be more specific as it has a broader but incorrect meaning --- test/pleroma/web/activity_pub/activity_pub_controller_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/pleroma/web/activity_pub/activity_pub_controller_test.exs b/test/pleroma/web/activity_pub/activity_pub_controller_test.exs index 1b959f324..762fca0a1 100644 --- a/test/pleroma/web/activity_pub/activity_pub_controller_test.exs +++ b/test/pleroma/web/activity_pub/activity_pub_controller_test.exs @@ -657,7 +657,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do end test "without valid signature, " <> - "it only accepts Create activities and requires enabled federation", + "it accepts Create activities and requires enabled federation", %{conn: conn} do data = File.read!("test/fixtures/mastodon-post-activity.json") |> Jason.decode!() non_create_data = File.read!("test/fixtures/mastodon-announce.json") |> Jason.decode!() From 012132303f79c0d693a8fba7236433443261b757 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 29 Aug 2024 14:40:45 -0400 Subject: [PATCH 047/117] Test more types we do not want to receive from strangers --- .../activity_pub_controller_test.exs | 30 +++++++++++-------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/test/pleroma/web/activity_pub/activity_pub_controller_test.exs b/test/pleroma/web/activity_pub/activity_pub_controller_test.exs index 762fca0a1..453dbaf0c 100644 --- a/test/pleroma/web/activity_pub/activity_pub_controller_test.exs +++ b/test/pleroma/web/activity_pub/activity_pub_controller_test.exs @@ -688,21 +688,25 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do # we capture all the params and process it later in the Oban job. # Once we begin processing it through Oban we risk fetching the actor to validate the # activity which just leads to inserting a new user to process a Delete not relevant to us. - test "Deletes from an unknown actor are discarded", %{conn: conn} do - params = - %{ - "type" => "Delete", - "actor" => "https://unknown.mastodon.instance/users/somebody" - } - |> Jason.encode!() + test "Activities of certain types from an unknown actor are discarded", %{conn: conn} do + example_bad_types = ["Announce", "Delete", "Undo"] - conn - |> assign(:valid_signature, false) - |> put_req_header("content-type", "application/activity+json") - |> post("/inbox", params) - |> json_response(400) + Enum.each(example_bad_types, fn bad_type -> + params = + %{ + "type" => bad_type, + "actor" => "https://unknown.mastodon.instance/users/somebody" + } + |> Jason.encode!() - assert all_enqueued() == [] + conn + |> assign(:valid_signature, false) + |> put_req_header("content-type", "application/activity+json") + |> post("/inbox", params) + |> json_response(400) + + assert all_enqueued() == [] + end) end test "accepts Add/Remove activities", %{conn: conn} do From 094da5d6343507eb1540f0a6357accc67573e02e Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 29 Aug 2024 14:44:52 -0400 Subject: [PATCH 048/117] Update changelog --- changelog.d/drop-unknown-deletes.change | 1 - changelog.d/drop-unwanted.change | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) delete mode 100644 changelog.d/drop-unknown-deletes.change create mode 100644 changelog.d/drop-unwanted.change diff --git a/changelog.d/drop-unknown-deletes.change b/changelog.d/drop-unknown-deletes.change deleted file mode 100644 index 0be7f5450..000000000 --- a/changelog.d/drop-unknown-deletes.change +++ /dev/null @@ -1 +0,0 @@ -Drop incoming Delete activities from unknown actors diff --git a/changelog.d/drop-unwanted.change b/changelog.d/drop-unwanted.change new file mode 100644 index 000000000..59447d68f --- /dev/null +++ b/changelog.d/drop-unwanted.change @@ -0,0 +1 @@ +Restrict incoming activities from unknown actors to a subset that does not imply a previous relationship From 5205e846eb5cfbd0adfa5031ad75e96fccbc86d8 Mon Sep 17 00:00:00 2001 From: feld Date: Fri, 30 Aug 2024 13:14:05 +0000 Subject: [PATCH 049/117] Update allowed activity types from strangers Move is emitted from the old account EmojiReact is ~ Like Announced TBD --- lib/pleroma/constants.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/constants.ex b/lib/pleroma/constants.ex index f969bfdbb..bbd183683 100644 --- a/lib/pleroma/constants.ex +++ b/lib/pleroma/constants.ex @@ -92,8 +92,8 @@ defmodule Pleroma.Constants do "Flag", "Follow", "Like", - "Move", - "React" + "EmojiReact", + "Announce" ] ) From e38f5f1a817d6da30e9a128ec74a2a7c78faf174 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 30 Aug 2024 09:35:04 -0400 Subject: [PATCH 050/117] Add recognized activity types to a constant and use it in the test --- lib/pleroma/constants.ex | 18 ++++++++++++++++++ .../activity_pub_controller_test.exs | 4 +++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/constants.ex b/lib/pleroma/constants.ex index bbd183683..5268ebe7a 100644 --- a/lib/pleroma/constants.ex +++ b/lib/pleroma/constants.ex @@ -85,6 +85,24 @@ defmodule Pleroma.Constants do ] ) + const(activity_types, + do: [ + "Create", + "Update", + "Delete", + "Follow", + "Accept", + "Reject", + "Add", + "Remove", + "Like", + "Announce", + "Undo", + "Flag", + "EmojiReact" + ] + ) + const(allowed_activity_types_from_strangers, do: [ "Block", diff --git a/test/pleroma/web/activity_pub/activity_pub_controller_test.exs b/test/pleroma/web/activity_pub/activity_pub_controller_test.exs index 453dbaf0c..c32f6c1a3 100644 --- a/test/pleroma/web/activity_pub/activity_pub_controller_test.exs +++ b/test/pleroma/web/activity_pub/activity_pub_controller_test.exs @@ -689,7 +689,9 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do # Once we begin processing it through Oban we risk fetching the actor to validate the # activity which just leads to inserting a new user to process a Delete not relevant to us. test "Activities of certain types from an unknown actor are discarded", %{conn: conn} do - example_bad_types = ["Announce", "Delete", "Undo"] + example_bad_types = + Pleroma.Constants.activity_types() -- + Pleroma.Constants.allowed_activity_types_from_strangers() Enum.each(example_bad_types, fn bad_type -> params = From 11ee94ae17094a2bc33505a31671b8c705f768a4 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 30 Aug 2024 09:46:10 -0400 Subject: [PATCH 051/117] InboxGuardPlug: Add early rejection of unknown activity types --- lib/pleroma/web/plugs/inbox_guard_plug.ex | 31 ++++++++++++++++--- .../activity_pub_controller_test.exs | 21 +++++++++++++ 2 files changed, 48 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/web/plugs/inbox_guard_plug.ex b/lib/pleroma/web/plugs/inbox_guard_plug.ex index 643b586d4..0064cce76 100644 --- a/lib/pleroma/web/plugs/inbox_guard_plug.ex +++ b/lib/pleroma/web/plugs/inbox_guard_plug.ex @@ -4,7 +4,7 @@ defmodule Pleroma.Web.Plugs.InboxGuardPlug do import Plug.Conn - import Pleroma.Constants, only: [allowed_activity_types_from_strangers: 0] + import Pleroma.Constants, only: [activity_types: 0, allowed_activity_types_from_strangers: 0] alias Pleroma.Config alias Pleroma.User @@ -14,24 +14,46 @@ defmodule Pleroma.Web.Plugs.InboxGuardPlug do end def call(%{assigns: %{valid_signature: true}} = conn, _opts) do - conn + with {_, true} <- {:federating, Config.get!([:instance, :federating])} do + conn + |> filter_activity_types() + else + {:federating, false} -> + conn + |> json(403, "Not federating") + |> halt() + end end def call(conn, _opts) do with {_, true} <- {:federating, Config.get!([:instance, :federating])}, - true <- known_actor?(conn) do + conn = filter_activity_types(conn), + {:known, true} <- {:known, known_actor?(conn)} do conn else {:federating, false} -> conn |> json(403, "Not federating") + |> halt() - _ -> + {:known, false} -> conn |> filter_from_strangers() end end + # Early rejection of unrecognized types + defp filter_activity_types(%{body_params: %{"type" => type}} = conn) do + with true <- type in activity_types() do + conn + else + _ -> + conn + |> json(400, "Invalid activity type") + |> halt() + end + end + # If signature failed but we know this actor we should # accept it as we may only need to refetch their public key # during processing @@ -52,6 +74,7 @@ defmodule Pleroma.Web.Plugs.InboxGuardPlug do _ -> conn |> json(400, "Invalid activity type for an unknown actor") + |> halt() end end diff --git a/test/pleroma/web/activity_pub/activity_pub_controller_test.exs b/test/pleroma/web/activity_pub/activity_pub_controller_test.exs index c32f6c1a3..3bd589f49 100644 --- a/test/pleroma/web/activity_pub/activity_pub_controller_test.exs +++ b/test/pleroma/web/activity_pub/activity_pub_controller_test.exs @@ -711,6 +711,27 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do end) end + test "Unknown activity types are discarded", %{conn: conn} do + unknown_types = ["Poke", "Read", "Dazzle"] + + Enum.each(unknown_types, fn bad_type -> + params = + %{ + "type" => bad_type, + "actor" => "https://unknown.mastodon.instance/users/somebody" + } + |> Jason.encode!() + + conn + |> assign(:valid_signature, true) + |> put_req_header("content-type", "application/activity+json") + |> post("/inbox", params) + |> json_response(400) + + assert all_enqueued() == [] + end) + end + test "accepts Add/Remove activities", %{conn: conn} do object_id = "c61d6733-e256-4fe1-ab13-1e369789423f" From bb235f913fb88f925abc791285808afe63d14bca Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 30 Aug 2024 10:03:51 -0400 Subject: [PATCH 052/117] Update changelog --- changelog.d/drop-unwanted.change | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.d/drop-unwanted.change b/changelog.d/drop-unwanted.change index 59447d68f..459d4bfe6 100644 --- a/changelog.d/drop-unwanted.change +++ b/changelog.d/drop-unwanted.change @@ -1 +1 @@ -Restrict incoming activities from unknown actors to a subset that does not imply a previous relationship +Restrict incoming activities from unknown actors to a subset that does not imply a previous relationship and early rejection of unrecognized activity types. From 4ae17c62944eab89acfa96a0912819093c872436 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 30 Aug 2024 15:25:21 -0400 Subject: [PATCH 053/117] NodeInfo: Accept application/activity+json requests --- changelog.d/well-known.change | 1 + lib/pleroma/web/router.ex | 2 +- test/pleroma/web/node_info_test.exs | 13 +++++++++++++ 3 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 changelog.d/well-known.change diff --git a/changelog.d/well-known.change b/changelog.d/well-known.change new file mode 100644 index 000000000..e928124fb --- /dev/null +++ b/changelog.d/well-known.change @@ -0,0 +1 @@ +Accept application/activity+json for requests to .well-known/nodeinfo diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 6492e3861..9e4b403e0 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -189,7 +189,7 @@ defmodule Pleroma.Web.Router do end pipeline :well_known do - plug(:accepts, ["json", "jrd", "jrd+json", "xml", "xrd+xml"]) + plug(:accepts, ["activity+json", "json", "jrd", "jrd+json", "xml", "xrd+xml"]) end pipeline :config do diff --git a/test/pleroma/web/node_info_test.exs b/test/pleroma/web/node_info_test.exs index f474220be..afe4ebb36 100644 --- a/test/pleroma/web/node_info_test.exs +++ b/test/pleroma/web/node_info_test.exs @@ -24,6 +24,19 @@ defmodule Pleroma.Web.NodeInfoTest do |> get(href) |> json_response(200) end) + + accept_types = [ + "application/activity+json", + "application/json", + "application/jrd+json" + ] + + for type <- accept_types do + conn + |> put_req_header("accept", type) + |> get("/.well-known/nodeinfo") + |> json_response(200) + end end test "nodeinfo shows staff accounts", %{conn: conn} do From 5a1144208d1007af2a2d2279c582adf9d2fa7246 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sun, 1 Sep 2024 12:26:59 -0400 Subject: [PATCH 054/117] Prevent OAuth App flow from creating duplicate entries --- changelog.d/oauth-app.fix | 1 + .../controllers/app_controller.ex | 4 +- .../controllers/app_controller_test.exs | 47 +++++++++++++++++++ 3 files changed, 49 insertions(+), 3 deletions(-) create mode 100644 changelog.d/oauth-app.fix diff --git a/changelog.d/oauth-app.fix b/changelog.d/oauth-app.fix new file mode 100644 index 000000000..eb917462f --- /dev/null +++ b/changelog.d/oauth-app.fix @@ -0,0 +1 @@ +Prevent OAuth App flow from creating duplicate entries diff --git a/lib/pleroma/web/mastodon_api/controllers/app_controller.ex b/lib/pleroma/web/mastodon_api/controllers/app_controller.ex index 844673ae0..e5e8ea8f5 100644 --- a/lib/pleroma/web/mastodon_api/controllers/app_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/app_controller.ex @@ -33,11 +33,9 @@ defmodule Pleroma.Web.MastodonAPI.AppController do app_attrs = params |> Map.take([:client_name, :redirect_uris, :website]) - |> Map.put(:scopes, scopes) |> Maps.put_if_present(:user_id, user_id) - with cs <- App.register_changeset(%App{}, app_attrs), - {:ok, app} <- Repo.insert(cs) do + with {:ok, app} <- App.get_or_make(app_attrs, scopes) do render(conn, "show.json", app: app) end end diff --git a/test/pleroma/web/mastodon_api/controllers/app_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/app_controller_test.exs index bc9d4048c..1e2e68791 100644 --- a/test/pleroma/web/mastodon_api/controllers/app_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/app_controller_test.exs @@ -89,4 +89,51 @@ defmodule Pleroma.Web.MastodonAPI.AppControllerTest do assert expected == json_response_and_validate_schema(conn, 200) assert app.user_id == user.id end + + test "creates an oauth app without a user", %{conn: conn} do + app_attrs = build(:oauth_app) + + conn = + conn + |> put_req_header("content-type", "application/json") + |> post("/api/v1/apps", %{ + client_name: app_attrs.client_name, + redirect_uris: app_attrs.redirect_uris + }) + + [app] = Repo.all(App) + + expected = %{ + "name" => app.client_name, + "website" => app.website, + "client_id" => app.client_id, + "client_secret" => app.client_secret, + "id" => app.id |> to_string(), + "redirect_uri" => app.redirect_uris, + "vapid_key" => Push.vapid_config() |> Keyword.get(:public_key) + } + + assert expected == json_response_and_validate_schema(conn, 200) + end + + test "does not duplicate apps with the same client name", %{conn: conn} do + client_name = "BleromaSE" + redirect_uris = "https://bleroma.app/oauth-callback" + + for _i <- 1..3 do + conn + |> put_req_header("content-type", "application/json") + |> post("/api/v1/apps", %{ + client_name: client_name, + redirect_uris: redirect_uris + }) + |> json_response_and_validate_schema(200) + end + + apps = Repo.all(App) + + assert length(apps) == 1 + assert List.first(apps).client_name == client_name + assert List.first(apps).redirect_uris == redirect_uris + end end From e3a7c1d906698d2f36661b60de4bdab5a475b871 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sun, 1 Sep 2024 12:37:59 -0400 Subject: [PATCH 055/117] Test that app scopes can be updated --- .../controllers/app_controller_test.exs | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/test/pleroma/web/mastodon_api/controllers/app_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/app_controller_test.exs index 1e2e68791..26c431673 100644 --- a/test/pleroma/web/mastodon_api/controllers/app_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/app_controller_test.exs @@ -136,4 +136,37 @@ defmodule Pleroma.Web.MastodonAPI.AppControllerTest do assert List.first(apps).client_name == client_name assert List.first(apps).redirect_uris == redirect_uris end + + test "app scopes can be updated", %{conn: conn} do + client_name = "BleromaSE" + redirect_uris = "https://bleroma.app/oauth-callback" + website = "https://bleromase.com" + scopes = "read write" + + conn + |> put_req_header("content-type", "application/json") + |> post("/api/v1/apps", %{ + client_name: client_name, + redirect_uris: redirect_uris, + website: website, + scopes: scopes + }) + |> json_response_and_validate_schema(200) + + assert List.first(Repo.all(App)).scopes == String.split(scopes, " ") + + updated_scopes = "read write push" + + conn + |> put_req_header("content-type", "application/json") + |> post("/api/v1/apps", %{ + client_name: client_name, + redirect_uris: redirect_uris, + website: website, + scopes: updated_scopes + }) + |> json_response_and_validate_schema(200) + + assert List.first(Repo.all(App)).scopes == String.split(updated_scopes, " ") + end end From 751d63d4bb05caececf52a3a3b134182e57a059d Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sun, 1 Sep 2024 13:34:30 -0400 Subject: [PATCH 056/117] Support OAuth App updating the website URL --- .../controllers/app_controller.ex | 3 +- lib/pleroma/web/o_auth/app.ex | 24 +++++---------- .../controllers/app_controller_test.exs | 30 +++++++++++++++++++ test/pleroma/web/o_auth/app_test.exs | 15 ++++++---- 4 files changed, 49 insertions(+), 23 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/controllers/app_controller.ex b/lib/pleroma/web/mastodon_api/controllers/app_controller.ex index e5e8ea8f5..4677ac40a 100644 --- a/lib/pleroma/web/mastodon_api/controllers/app_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/app_controller.ex @@ -33,9 +33,10 @@ defmodule Pleroma.Web.MastodonAPI.AppController do app_attrs = params |> Map.take([:client_name, :redirect_uris, :website]) + |> Map.put(:scopes, scopes) |> Maps.put_if_present(:user_id, user_id) - with {:ok, app} <- App.get_or_make(app_attrs, scopes) do + with {:ok, app} <- App.get_or_make(app_attrs) do render(conn, "show.json", app: app) end end diff --git a/lib/pleroma/web/o_auth/app.ex b/lib/pleroma/web/o_auth/app.ex index d1bf6dd18..889850c73 100644 --- a/lib/pleroma/web/o_auth/app.ex +++ b/lib/pleroma/web/o_auth/app.ex @@ -67,35 +67,27 @@ defmodule Pleroma.Web.OAuth.App do with %__MODULE__{} = app <- Repo.get(__MODULE__, id) do app |> changeset(params) + |> validate_required([:scopes]) |> Repo.update() end end @doc """ - Gets app by attrs or create new with attrs. - And updates the scopes if need. + Gets app by attrs or create new with attrs. + Updates the attrs if needed. """ - @spec get_or_make(map(), list(String.t())) :: {:ok, t()} | {:error, Ecto.Changeset.t()} - def get_or_make(attrs, scopes) do - with %__MODULE__{} = app <- Repo.get_by(__MODULE__, attrs) do - update_scopes(app, scopes) + @spec get_or_make(map()) :: {:ok, t()} | {:error, Ecto.Changeset.t()} + def get_or_make(attrs) do + with %__MODULE__{} = app <- Repo.get_by(__MODULE__, client_name: attrs.client_name) do + __MODULE__.update(app.id, Map.take(attrs, [:scopes, :website])) else _e -> %__MODULE__{} - |> register_changeset(Map.put(attrs, :scopes, scopes)) + |> register_changeset(attrs) |> Repo.insert() end end - defp update_scopes(%__MODULE__{} = app, []), do: {:ok, app} - defp update_scopes(%__MODULE__{scopes: scopes} = app, scopes), do: {:ok, app} - - defp update_scopes(%__MODULE__{} = app, scopes) do - app - |> change(%{scopes: scopes}) - |> Repo.update() - end - @spec search(map()) :: {:ok, [t()], non_neg_integer()} def search(params) do query = from(a in __MODULE__) diff --git a/test/pleroma/web/mastodon_api/controllers/app_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/app_controller_test.exs index 26c431673..df28f2010 100644 --- a/test/pleroma/web/mastodon_api/controllers/app_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/app_controller_test.exs @@ -169,4 +169,34 @@ defmodule Pleroma.Web.MastodonAPI.AppControllerTest do assert List.first(Repo.all(App)).scopes == String.split(updated_scopes, " ") end + + test "app website URL can be updated", %{conn: conn} do + client_name = "BleromaSE" + redirect_uris = "https://bleroma.app/oauth-callback" + website = "https://bleromase.com" + + conn + |> put_req_header("content-type", "application/json") + |> post("/api/v1/apps", %{ + client_name: client_name, + redirect_uris: redirect_uris, + website: website + }) + |> json_response_and_validate_schema(200) + + assert List.first(Repo.all(App)).website == website + + updated_website = "https://bleromase2ultimateedition.com" + + conn + |> put_req_header("content-type", "application/json") + |> post("/api/v1/apps", %{ + client_name: client_name, + redirect_uris: redirect_uris, + website: updated_website + }) + |> json_response_and_validate_schema(200) + + assert List.first(Repo.all(App)).website == updated_website + end end diff --git a/test/pleroma/web/o_auth/app_test.exs b/test/pleroma/web/o_auth/app_test.exs index 96a67de6b..423b660ea 100644 --- a/test/pleroma/web/o_auth/app_test.exs +++ b/test/pleroma/web/o_auth/app_test.exs @@ -12,20 +12,23 @@ defmodule Pleroma.Web.OAuth.AppTest do test "gets exist app" do attrs = %{client_name: "Mastodon-Local", redirect_uris: "."} app = insert(:oauth_app, Map.merge(attrs, %{scopes: ["read", "write"]})) - {:ok, %App{} = exist_app} = App.get_or_make(attrs, []) + {:ok, %App{} = exist_app} = App.get_or_make(attrs) assert exist_app == app end test "make app" do - attrs = %{client_name: "Mastodon-Local", redirect_uris: "."} - {:ok, %App{} = app} = App.get_or_make(attrs, ["write"]) + attrs = %{client_name: "Mastodon-Local", redirect_uris: ".", scopes: ["write"]} + {:ok, %App{} = app} = App.get_or_make(attrs) assert app.scopes == ["write"] end test "gets exist app and updates scopes" do - attrs = %{client_name: "Mastodon-Local", redirect_uris: "."} - app = insert(:oauth_app, Map.merge(attrs, %{scopes: ["read", "write"]})) - {:ok, %App{} = exist_app} = App.get_or_make(attrs, ["read", "write", "follow", "push"]) + attrs = %{client_name: "Mastodon-Local", redirect_uris: ".", scopes: ["read", "write"]} + app = insert(:oauth_app, attrs) + + {:ok, %App{} = exist_app} = + App.get_or_make(%{attrs | scopes: ["read", "write", "follow", "push"]}) + assert exist_app.id == app.id assert exist_app.scopes == ["read", "write", "follow", "push"] end From 37397a43be7e73efeb1004f59fc9e69bc75da927 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Mon, 2 Sep 2024 12:39:29 +0200 Subject: [PATCH 057/117] scrubbers/default: Allow "mention hashtag" classes used by Mastodon MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- changelog.d/scrubbers-allow-mention-hashtag.add | 1 + priv/scrubbers/default.ex | 3 ++- priv/scrubbers/twitter_text.ex | 3 ++- 3 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 changelog.d/scrubbers-allow-mention-hashtag.add diff --git a/changelog.d/scrubbers-allow-mention-hashtag.add b/changelog.d/scrubbers-allow-mention-hashtag.add new file mode 100644 index 000000000..c12ab1ffb --- /dev/null +++ b/changelog.d/scrubbers-allow-mention-hashtag.add @@ -0,0 +1 @@ +scrubbers/default: Allow "mention hashtag" classes used by Mastodon \ No newline at end of file diff --git a/priv/scrubbers/default.ex b/priv/scrubbers/default.ex index a75a6465d..dad9dc1a1 100644 --- a/priv/scrubbers/default.ex +++ b/priv/scrubbers/default.ex @@ -22,7 +22,8 @@ defmodule Pleroma.HTML.Scrubber.Default do "u-url", "mention", "u-url mention", - "mention u-url" + "mention u-url", + "mention hashtag" ]) Meta.allow_tag_with_this_attribute_values(:a, "rel", [ diff --git a/priv/scrubbers/twitter_text.ex b/priv/scrubbers/twitter_text.ex index 6e23b3efb..4df840735 100644 --- a/priv/scrubbers/twitter_text.ex +++ b/priv/scrubbers/twitter_text.ex @@ -23,7 +23,8 @@ defmodule Pleroma.HTML.Scrubber.TwitterText do "u-url", "mention", "u-url mention", - "mention u-url" + "mention u-url", + "mention hashtag" ]) Meta.allow_tag_with_this_attribute_values(:a, "rel", [ From 6d5ae4d2e91f8ea75115c0ffcdd1d94a24c9dc31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Tue, 3 Sep 2024 15:17:45 +0200 Subject: [PATCH 058/117] Include list id in StatusView MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- changelog.d/list-id-visibility.add | 1 + .../API/differences_in_mastoapi_responses.md | 1 + lib/pleroma/web/api_spec/schemas/status.ex | 6 ++++++ lib/pleroma/web/mastodon_api/views/status_view.ex | 13 ++++++++++++- .../web/mastodon_api/views/status_view_test.exs | 8 +++++++- 5 files changed, 27 insertions(+), 2 deletions(-) create mode 100644 changelog.d/list-id-visibility.add diff --git a/changelog.d/list-id-visibility.add b/changelog.d/list-id-visibility.add new file mode 100644 index 000000000..2fea2d771 --- /dev/null +++ b/changelog.d/list-id-visibility.add @@ -0,0 +1 @@ +Include list id in StatusView \ No newline at end of file diff --git a/docs/development/API/differences_in_mastoapi_responses.md b/docs/development/API/differences_in_mastoapi_responses.md index 41464e802..3bf8637f8 100644 --- a/docs/development/API/differences_in_mastoapi_responses.md +++ b/docs/development/API/differences_in_mastoapi_responses.md @@ -42,6 +42,7 @@ Has these additional fields under the `pleroma` object: - `quotes_count`: the count of status quotes. - `non_anonymous`: true if the source post specifies the poll results are not anonymous. Currently only implemented by Smithereen. - `bookmark_folder`: the ID of the folder bookmark is stored within (if any). +- `list_id`: the ID of the list the post is addressed to (if any, only returned to author). The `GET /api/v1/statuses/:id/source` endpoint additionally has the following attributes: diff --git a/lib/pleroma/web/api_spec/schemas/status.ex b/lib/pleroma/web/api_spec/schemas/status.ex index 6e537b5da..25548d75b 100644 --- a/lib/pleroma/web/api_spec/schemas/status.ex +++ b/lib/pleroma/web/api_spec/schemas/status.ex @@ -249,6 +249,12 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Status do nullable: true, description: "A datetime (ISO 8601) that states when the post was pinned or `null` if the post is not pinned" + }, + list_id: %Schema{ + type: :integer, + nullable: true, + description: + "The ID of the list the post is addressed to (if any, only returned to author)" } } }, diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 1b78477d0..3bf870c24 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -465,7 +465,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do parent_visible: visible_for_user?(reply_to, opts[:for]), pinned_at: pinned_at, quotes_count: object.data["quotesCount"] || 0, - bookmark_folder: bookmark_folder + bookmark_folder: bookmark_folder, + list_id: get_list_id(object, client_posted_this_activity) } } end @@ -835,4 +836,14 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do nil end end + + defp get_list_id(object, client_posted_this_activity) do + with true <- client_posted_this_activity, + %{data: %{"listMessage" => list_ap_id}} when is_binary(list_ap_id) <- object, + %{id: list_id} <- Pleroma.List.get_by_ap_id(list_ap_id) do + list_id + else + _ -> nil + end + end end diff --git a/test/pleroma/web/mastodon_api/views/status_view_test.exs b/test/pleroma/web/mastodon_api/views/status_view_test.exs index afe0ccb28..bc6dec32a 100644 --- a/test/pleroma/web/mastodon_api/views/status_view_test.exs +++ b/test/pleroma/web/mastodon_api/views/status_view_test.exs @@ -342,7 +342,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do parent_visible: false, pinned_at: nil, quotes_count: 0, - bookmark_folder: nil + bookmark_folder: nil, + list_id: nil } } @@ -912,6 +913,11 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do status = StatusView.render("show.json", activity: activity) assert status.visibility == "list" + assert status.pleroma.list_id == nil + + status = StatusView.render("show.json", activity: activity, for: user) + + assert status.pleroma.list_id == list.id end test "has a field for parent visibility" do From 92d5f0ac141e679af048ebcbec9cb91df68fd929 Mon Sep 17 00:00:00 2001 From: feld Date: Wed, 4 Sep 2024 02:22:25 +0000 Subject: [PATCH 059/117] Revert "Merge branch 'oauth-app-spam' into 'develop'" This reverts merge request !4244 --- changelog.d/oauth-app.fix | 1 - .../controllers/app_controller.ex | 3 +- lib/pleroma/web/o_auth/app.ex | 24 ++-- .../controllers/app_controller_test.exs | 110 ------------------ test/pleroma/web/o_auth/app_test.exs | 15 +-- 5 files changed, 24 insertions(+), 129 deletions(-) delete mode 100644 changelog.d/oauth-app.fix diff --git a/changelog.d/oauth-app.fix b/changelog.d/oauth-app.fix deleted file mode 100644 index eb917462f..000000000 --- a/changelog.d/oauth-app.fix +++ /dev/null @@ -1 +0,0 @@ -Prevent OAuth App flow from creating duplicate entries diff --git a/lib/pleroma/web/mastodon_api/controllers/app_controller.ex b/lib/pleroma/web/mastodon_api/controllers/app_controller.ex index 4677ac40a..844673ae0 100644 --- a/lib/pleroma/web/mastodon_api/controllers/app_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/app_controller.ex @@ -36,7 +36,8 @@ defmodule Pleroma.Web.MastodonAPI.AppController do |> Map.put(:scopes, scopes) |> Maps.put_if_present(:user_id, user_id) - with {:ok, app} <- App.get_or_make(app_attrs) do + with cs <- App.register_changeset(%App{}, app_attrs), + {:ok, app} <- Repo.insert(cs) do render(conn, "show.json", app: app) end end diff --git a/lib/pleroma/web/o_auth/app.ex b/lib/pleroma/web/o_auth/app.ex index 889850c73..d1bf6dd18 100644 --- a/lib/pleroma/web/o_auth/app.ex +++ b/lib/pleroma/web/o_auth/app.ex @@ -67,27 +67,35 @@ defmodule Pleroma.Web.OAuth.App do with %__MODULE__{} = app <- Repo.get(__MODULE__, id) do app |> changeset(params) - |> validate_required([:scopes]) |> Repo.update() end end @doc """ - Gets app by attrs or create new with attrs. - Updates the attrs if needed. + Gets app by attrs or create new with attrs. + And updates the scopes if need. """ - @spec get_or_make(map()) :: {:ok, t()} | {:error, Ecto.Changeset.t()} - def get_or_make(attrs) do - with %__MODULE__{} = app <- Repo.get_by(__MODULE__, client_name: attrs.client_name) do - __MODULE__.update(app.id, Map.take(attrs, [:scopes, :website])) + @spec get_or_make(map(), list(String.t())) :: {:ok, t()} | {:error, Ecto.Changeset.t()} + def get_or_make(attrs, scopes) do + with %__MODULE__{} = app <- Repo.get_by(__MODULE__, attrs) do + update_scopes(app, scopes) else _e -> %__MODULE__{} - |> register_changeset(attrs) + |> register_changeset(Map.put(attrs, :scopes, scopes)) |> Repo.insert() end end + defp update_scopes(%__MODULE__{} = app, []), do: {:ok, app} + defp update_scopes(%__MODULE__{scopes: scopes} = app, scopes), do: {:ok, app} + + defp update_scopes(%__MODULE__{} = app, scopes) do + app + |> change(%{scopes: scopes}) + |> Repo.update() + end + @spec search(map()) :: {:ok, [t()], non_neg_integer()} def search(params) do query = from(a in __MODULE__) diff --git a/test/pleroma/web/mastodon_api/controllers/app_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/app_controller_test.exs index df28f2010..bc9d4048c 100644 --- a/test/pleroma/web/mastodon_api/controllers/app_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/app_controller_test.exs @@ -89,114 +89,4 @@ defmodule Pleroma.Web.MastodonAPI.AppControllerTest do assert expected == json_response_and_validate_schema(conn, 200) assert app.user_id == user.id end - - test "creates an oauth app without a user", %{conn: conn} do - app_attrs = build(:oauth_app) - - conn = - conn - |> put_req_header("content-type", "application/json") - |> post("/api/v1/apps", %{ - client_name: app_attrs.client_name, - redirect_uris: app_attrs.redirect_uris - }) - - [app] = Repo.all(App) - - expected = %{ - "name" => app.client_name, - "website" => app.website, - "client_id" => app.client_id, - "client_secret" => app.client_secret, - "id" => app.id |> to_string(), - "redirect_uri" => app.redirect_uris, - "vapid_key" => Push.vapid_config() |> Keyword.get(:public_key) - } - - assert expected == json_response_and_validate_schema(conn, 200) - end - - test "does not duplicate apps with the same client name", %{conn: conn} do - client_name = "BleromaSE" - redirect_uris = "https://bleroma.app/oauth-callback" - - for _i <- 1..3 do - conn - |> put_req_header("content-type", "application/json") - |> post("/api/v1/apps", %{ - client_name: client_name, - redirect_uris: redirect_uris - }) - |> json_response_and_validate_schema(200) - end - - apps = Repo.all(App) - - assert length(apps) == 1 - assert List.first(apps).client_name == client_name - assert List.first(apps).redirect_uris == redirect_uris - end - - test "app scopes can be updated", %{conn: conn} do - client_name = "BleromaSE" - redirect_uris = "https://bleroma.app/oauth-callback" - website = "https://bleromase.com" - scopes = "read write" - - conn - |> put_req_header("content-type", "application/json") - |> post("/api/v1/apps", %{ - client_name: client_name, - redirect_uris: redirect_uris, - website: website, - scopes: scopes - }) - |> json_response_and_validate_schema(200) - - assert List.first(Repo.all(App)).scopes == String.split(scopes, " ") - - updated_scopes = "read write push" - - conn - |> put_req_header("content-type", "application/json") - |> post("/api/v1/apps", %{ - client_name: client_name, - redirect_uris: redirect_uris, - website: website, - scopes: updated_scopes - }) - |> json_response_and_validate_schema(200) - - assert List.first(Repo.all(App)).scopes == String.split(updated_scopes, " ") - end - - test "app website URL can be updated", %{conn: conn} do - client_name = "BleromaSE" - redirect_uris = "https://bleroma.app/oauth-callback" - website = "https://bleromase.com" - - conn - |> put_req_header("content-type", "application/json") - |> post("/api/v1/apps", %{ - client_name: client_name, - redirect_uris: redirect_uris, - website: website - }) - |> json_response_and_validate_schema(200) - - assert List.first(Repo.all(App)).website == website - - updated_website = "https://bleromase2ultimateedition.com" - - conn - |> put_req_header("content-type", "application/json") - |> post("/api/v1/apps", %{ - client_name: client_name, - redirect_uris: redirect_uris, - website: updated_website - }) - |> json_response_and_validate_schema(200) - - assert List.first(Repo.all(App)).website == updated_website - end end diff --git a/test/pleroma/web/o_auth/app_test.exs b/test/pleroma/web/o_auth/app_test.exs index 423b660ea..96a67de6b 100644 --- a/test/pleroma/web/o_auth/app_test.exs +++ b/test/pleroma/web/o_auth/app_test.exs @@ -12,23 +12,20 @@ defmodule Pleroma.Web.OAuth.AppTest do test "gets exist app" do attrs = %{client_name: "Mastodon-Local", redirect_uris: "."} app = insert(:oauth_app, Map.merge(attrs, %{scopes: ["read", "write"]})) - {:ok, %App{} = exist_app} = App.get_or_make(attrs) + {:ok, %App{} = exist_app} = App.get_or_make(attrs, []) assert exist_app == app end test "make app" do - attrs = %{client_name: "Mastodon-Local", redirect_uris: ".", scopes: ["write"]} - {:ok, %App{} = app} = App.get_or_make(attrs) + attrs = %{client_name: "Mastodon-Local", redirect_uris: "."} + {:ok, %App{} = app} = App.get_or_make(attrs, ["write"]) assert app.scopes == ["write"] end test "gets exist app and updates scopes" do - attrs = %{client_name: "Mastodon-Local", redirect_uris: ".", scopes: ["read", "write"]} - app = insert(:oauth_app, attrs) - - {:ok, %App{} = exist_app} = - App.get_or_make(%{attrs | scopes: ["read", "write", "follow", "push"]}) - + attrs = %{client_name: "Mastodon-Local", redirect_uris: "."} + app = insert(:oauth_app, Map.merge(attrs, %{scopes: ["read", "write"]})) + {:ok, %App{} = exist_app} = App.get_or_make(attrs, ["read", "write", "follow", "push"]) assert exist_app.id == app.id assert exist_app.scopes == ["read", "write", "follow", "push"] end From 427da7a99a30ebc7a7deb54e7704b5d8dffea199 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 4 Sep 2024 09:19:07 -0400 Subject: [PATCH 060/117] Rate Limit the OAuth App spam --- changelog.d/oauth-app-spam.fix | 1 + config/config.exs | 1 + lib/pleroma/web/mastodon_api/controllers/app_controller.ex | 2 ++ 3 files changed, 4 insertions(+) create mode 100644 changelog.d/oauth-app-spam.fix diff --git a/changelog.d/oauth-app-spam.fix b/changelog.d/oauth-app-spam.fix new file mode 100644 index 000000000..0e95c01d7 --- /dev/null +++ b/changelog.d/oauth-app-spam.fix @@ -0,0 +1 @@ +Add a rate limiter to the OAuth App creation endpoint diff --git a/config/config.exs b/config/config.exs index ad6b1cb94..a4fedff45 100644 --- a/config/config.exs +++ b/config/config.exs @@ -711,6 +711,7 @@ config :pleroma, :rate_limit, timeline: {500, 3}, search: [{1000, 10}, {1000, 30}], app_account_creation: {1_800_000, 25}, + oauth_app_creation: {900_000, 5}, relations_actions: {10_000, 10}, relation_id_action: {60_000, 2}, statuses_actions: {10_000, 15}, diff --git a/lib/pleroma/web/mastodon_api/controllers/app_controller.ex b/lib/pleroma/web/mastodon_api/controllers/app_controller.ex index 844673ae0..6cfeb712e 100644 --- a/lib/pleroma/web/mastodon_api/controllers/app_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/app_controller.ex @@ -19,6 +19,8 @@ defmodule Pleroma.Web.MastodonAPI.AppController do action_fallback(Pleroma.Web.MastodonAPI.FallbackController) + plug(Pleroma.Web.Plugs.RateLimiter, [name: :oauth_app_creation] when action == :create) + plug(:skip_auth when action in [:create, :verify_credentials]) plug(Pleroma.Web.ApiSpec.CastAndValidate) From 7bd0750787859cb30382d90162d70380441abc05 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 4 Sep 2024 10:40:37 -0400 Subject: [PATCH 061/117] Ensure apps are assigned to users --- changelog.d/oauth-app-spam.fix | 2 +- lib/pleroma/web/o_auth/app.ex | 10 +++++++++ lib/pleroma/web/o_auth/o_auth_controller.ex | 2 ++ .../20240904142434_assign_app_user.exs | 21 +++++++++++++++++++ .../web/o_auth/o_auth_controller_test.exs | 8 +++++++ 5 files changed, 42 insertions(+), 1 deletion(-) create mode 100644 priv/repo/migrations/20240904142434_assign_app_user.exs diff --git a/changelog.d/oauth-app-spam.fix b/changelog.d/oauth-app-spam.fix index 0e95c01d7..cdc2e816d 100644 --- a/changelog.d/oauth-app-spam.fix +++ b/changelog.d/oauth-app-spam.fix @@ -1 +1 @@ -Add a rate limiter to the OAuth App creation endpoint +Add a rate limiter to the OAuth App creation endpoint and ensure registered apps are assigned to users. diff --git a/lib/pleroma/web/o_auth/app.ex b/lib/pleroma/web/o_auth/app.ex index d1bf6dd18..f0f54bb46 100644 --- a/lib/pleroma/web/o_auth/app.ex +++ b/lib/pleroma/web/o_auth/app.ex @@ -8,6 +8,7 @@ defmodule Pleroma.Web.OAuth.App do import Ecto.Query alias Pleroma.Repo alias Pleroma.User + alias Pleroma.Web.OAuth.Token @type t :: %__MODULE__{} @@ -155,4 +156,13 @@ defmodule Pleroma.Web.OAuth.App do Map.put(acc, key, error) end) end + + @spec maybe_update_owner(Token.t()) :: :ok + def maybe_update_owner(%Token{app_id: app_id, user_id: user_id}) when not is_nil(user_id) do + __MODULE__.update(app_id, %{user_id: user_id}) + + :ok + end + + def maybe_update_owner(_), do: :ok end diff --git a/lib/pleroma/web/o_auth/o_auth_controller.ex b/lib/pleroma/web/o_auth/o_auth_controller.ex index 47b03215f..0b3de5481 100644 --- a/lib/pleroma/web/o_auth/o_auth_controller.ex +++ b/lib/pleroma/web/o_auth/o_auth_controller.ex @@ -318,6 +318,8 @@ defmodule Pleroma.Web.OAuth.OAuthController do def token_exchange(%Plug.Conn{} = conn, params), do: bad_request(conn, params) def after_token_exchange(%Plug.Conn{} = conn, %{token: token} = view_params) do + App.maybe_update_owner(token) + conn |> AuthHelper.put_session_token(token.token) |> json(OAuthView.render("token.json", view_params)) diff --git a/priv/repo/migrations/20240904142434_assign_app_user.exs b/priv/repo/migrations/20240904142434_assign_app_user.exs new file mode 100644 index 000000000..11bec529b --- /dev/null +++ b/priv/repo/migrations/20240904142434_assign_app_user.exs @@ -0,0 +1,21 @@ +defmodule Pleroma.Repo.Migrations.AssignAppUser do + use Ecto.Migration + + alias Pleroma.Repo + alias Pleroma.Web.OAuth.App + alias Pleroma.Web.OAuth.Token + + def up do + Repo.all(Token) + |> Enum.group_by(fn x -> Map.get(x, :app_id) end) + |> Enum.each(fn {_app_id, tokens} -> + token = + Enum.filter(tokens, fn x -> not is_nil(x.user_id) end) + |> List.first() + + App.maybe_update_owner(token) + end) + end + + def down, do: :ok +end diff --git a/test/pleroma/web/o_auth/o_auth_controller_test.exs b/test/pleroma/web/o_auth/o_auth_controller_test.exs index 83a08d9fc..260442771 100644 --- a/test/pleroma/web/o_auth/o_auth_controller_test.exs +++ b/test/pleroma/web/o_auth/o_auth_controller_test.exs @@ -12,6 +12,7 @@ defmodule Pleroma.Web.OAuth.OAuthControllerTest do alias Pleroma.MFA.TOTP alias Pleroma.Repo alias Pleroma.User + alias Pleroma.Web.OAuth.App alias Pleroma.Web.OAuth.Authorization alias Pleroma.Web.OAuth.OAuthController alias Pleroma.Web.OAuth.Token @@ -770,6 +771,9 @@ defmodule Pleroma.Web.OAuth.OAuthControllerTest do {:ok, auth} = Authorization.create_authorization(app, user, ["write"]) + # Verify app has no associated user yet + assert %Pleroma.Web.OAuth.App{user_id: nil} = Repo.get_by(App, %{id: app.id}) + conn = build_conn() |> post("/oauth/token", %{ @@ -786,6 +790,10 @@ defmodule Pleroma.Web.OAuth.OAuthControllerTest do assert token assert token.scopes == auth.scopes assert user.ap_id == ap_id + + # Verify app has an associated user now + user_id = user.id + assert %Pleroma.Web.OAuth.App{user_id: ^user_id} = Repo.get_by(App, %{id: app.id}) end test "issues a token for `password` grant_type with valid credentials, with full permissions by default" do From a1951f3af7e1d5c4d53819962c3e68df5ba4475b Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 4 Sep 2024 10:59:58 -0400 Subject: [PATCH 062/117] Add Cron worker to clean up orphaned apps hourly --- config/config.exs | 3 ++- lib/pleroma/web/o_auth/app.ex | 6 ++++++ .../workers/cron/app_cleanup_worker.ex | 21 +++++++++++++++++++ test/pleroma/web/o_auth/app_test.exs | 12 +++++++++++ 4 files changed, 41 insertions(+), 1 deletion(-) create mode 100644 lib/pleroma/workers/cron/app_cleanup_worker.ex diff --git a/config/config.exs b/config/config.exs index a4fedff45..2bc28b256 100644 --- a/config/config.exs +++ b/config/config.exs @@ -597,7 +597,8 @@ config :pleroma, Oban, plugins: [{Oban.Plugins.Pruner, max_age: 900}], crontab: [ {"0 0 * * 0", Pleroma.Workers.Cron.DigestEmailsWorker}, - {"0 0 * * *", Pleroma.Workers.Cron.NewUsersDigestWorker} + {"0 0 * * *", Pleroma.Workers.Cron.NewUsersDigestWorker}, + {"0 0 * * *", Pleroma.Workers.Cron.AppCleanupWorker} ] config :pleroma, Pleroma.Formatter, diff --git a/lib/pleroma/web/o_auth/app.ex b/lib/pleroma/web/o_auth/app.ex index f0f54bb46..6ae419d09 100644 --- a/lib/pleroma/web/o_auth/app.ex +++ b/lib/pleroma/web/o_auth/app.ex @@ -165,4 +165,10 @@ defmodule Pleroma.Web.OAuth.App do end def maybe_update_owner(_), do: :ok + + @spec remove_orphans() :: :ok + def remove_orphans() do + from(a in __MODULE__, where: is_nil(a.user_id)) + |> Repo.delete_all() + end end diff --git a/lib/pleroma/workers/cron/app_cleanup_worker.ex b/lib/pleroma/workers/cron/app_cleanup_worker.ex new file mode 100644 index 000000000..ee71cd7b6 --- /dev/null +++ b/lib/pleroma/workers/cron/app_cleanup_worker.ex @@ -0,0 +1,21 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Workers.Cron.AppCleanupWorker do + @moduledoc """ + Cleans up registered apps that were never associated with a user. + """ + + use Oban.Worker, queue: "background" + + alias Pleroma.Web.OAuth.App + + @impl true + def perform(_job) do + App.remove_orphans() + end + + @impl true + def timeout(_job), do: :timer.seconds(30) +end diff --git a/test/pleroma/web/o_auth/app_test.exs b/test/pleroma/web/o_auth/app_test.exs index 96a67de6b..725ea3eb8 100644 --- a/test/pleroma/web/o_auth/app_test.exs +++ b/test/pleroma/web/o_auth/app_test.exs @@ -53,4 +53,16 @@ defmodule Pleroma.Web.OAuth.AppTest do assert Enum.sort(App.get_user_apps(user)) == Enum.sort(apps) end + + test "removes orphaned apps" do + attrs = %{client_name: "Mastodon-Local", redirect_uris: "."} + {:ok, %App{} = app} = App.get_or_make(attrs, ["write"]) + assert app.scopes == ["write"] + + assert app == Pleroma.Repo.get_by(App, %{id: app.id}) + + App.remove_orphans() + + assert nil == Pleroma.Repo.get_by(App, %{id: app.id}) + end end From 53744bf146f157ee1ecfc9ba4de9e5d65fa80784 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 4 Sep 2024 11:43:43 -0400 Subject: [PATCH 063/117] Limit the number of orphaned to delete at 100 every 10 mins due to the cascading queries that have to check oauth_authorizations and oauth_tokens tables. This should keep ahead of most app registration spam and not overwhelm lower powered servers. --- config/config.exs | 2 +- lib/pleroma/web/o_auth/app.ex | 13 +++++++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/config/config.exs b/config/config.exs index 2bc28b256..80a3b8d57 100644 --- a/config/config.exs +++ b/config/config.exs @@ -598,7 +598,7 @@ config :pleroma, Oban, crontab: [ {"0 0 * * 0", Pleroma.Workers.Cron.DigestEmailsWorker}, {"0 0 * * *", Pleroma.Workers.Cron.NewUsersDigestWorker}, - {"0 0 * * *", Pleroma.Workers.Cron.AppCleanupWorker} + {"*/10 * * * *", Pleroma.Workers.Cron.AppCleanupWorker} ] config :pleroma, Pleroma.Formatter, diff --git a/lib/pleroma/web/o_auth/app.ex b/lib/pleroma/web/o_auth/app.ex index 6ae419d09..032011433 100644 --- a/lib/pleroma/web/o_auth/app.ex +++ b/lib/pleroma/web/o_auth/app.ex @@ -166,9 +166,14 @@ defmodule Pleroma.Web.OAuth.App do def maybe_update_owner(_), do: :ok - @spec remove_orphans() :: :ok - def remove_orphans() do - from(a in __MODULE__, where: is_nil(a.user_id)) - |> Repo.delete_all() + @spec remove_orphans(pos_integer()) :: :ok + def remove_orphans(limit \\ 100) do + Repo.transaction(fn -> + from(a in __MODULE__, where: is_nil(a.user_id), limit: ^limit) + |> Repo.all() + |> Enum.each(&Repo.delete(&1)) + end) + + :ok end end From fb376ce0056cf977d3673c99bca4e77c564b4c47 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 5 Sep 2024 15:27:43 -0400 Subject: [PATCH 064/117] Test Account View does not indicate following if a FollowingRelationship is missing --- .../mastodon_api/views/account_view_test.exs | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/test/pleroma/web/mastodon_api/views/account_view_test.exs b/test/pleroma/web/mastodon_api/views/account_view_test.exs index dca64853d..5d2f55c6d 100644 --- a/test/pleroma/web/mastodon_api/views/account_view_test.exs +++ b/test/pleroma/web/mastodon_api/views/account_view_test.exs @@ -456,6 +456,44 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do test_relationship_rendering(user, other_user, expected) end + test "relationship does not indicate following if a FollowingRelationship is missing" do + user = insert(:user) + other_user = insert(:user, local: false) + + # Create a follow relationship with the real Follow Activity and Accept it + assert {:ok, _, _, _} = CommonAPI.follow(other_user, user) + assert {:ok, _} = CommonAPI.accept_follow_request(user, other_user) + + assert %{data: %{"state" => "accept"}} = + Pleroma.Web.ActivityPub.Utils.fetch_latest_follow(user, other_user) + + # Fetch the relationship and forcibly delete it to simulate a Follow Accept that did not complete processing + %{following_relationships: [relationship]} = + Pleroma.UserRelationship.view_relationships_option(user, [other_user]) + + assert {:ok, _} = Pleroma.Repo.delete(relationship) + + assert %{following_relationships: [], user_relationships: []} == + Pleroma.UserRelationship.view_relationships_option(user, [other_user]) + + expected = + Map.merge( + @blank_response, + %{ + following: false, + followed_by: false, + muting: false, + muting_notifications: false, + subscribing: false, + notifying: false, + showing_reblogs: true, + id: to_string(other_user.id) + } + ) + + test_relationship_rendering(user, other_user, expected) + end + test "represent a relationship for the blocking and blocked user" do user = insert(:user) other_user = insert(:user) From 4d76692db36d6779fddacee3a7690739064eae0c Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 5 Sep 2024 11:43:48 -0400 Subject: [PATCH 065/117] Fix Following status bug --- changelog.d/following-state.fix | 1 + .../web/mastodon_api/views/account_view.ex | 19 +++++++++---------- 2 files changed, 10 insertions(+), 10 deletions(-) create mode 100644 changelog.d/following-state.fix diff --git a/changelog.d/following-state.fix b/changelog.d/following-state.fix new file mode 100644 index 000000000..314ea6210 --- /dev/null +++ b/changelog.d/following-state.fix @@ -0,0 +1 @@ +Resolved edge case where the API can report you are following a user but the relationship is not fully established. diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index 6976ca6e5..298c73986 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -92,14 +92,13 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do User.get_follow_state(reading_user, target) end - followed_by = - if following_relationships do - case FollowingRelationship.find(following_relationships, target, reading_user) do - %{state: :follow_accept} -> true - _ -> false - end - else - User.following?(target, reading_user) + followed_by = FollowingRelationship.following?(target, reading_user) + following = FollowingRelationship.following?(reading_user, target) + + requested = + cond do + following -> false + true -> match?(:follow_pending, follow_state) end subscribing = @@ -114,7 +113,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do # NOTE: adjust UserRelationship.view_relationships_option/2 on new relation-related flags %{ id: to_string(target.id), - following: follow_state == :follow_accept, + following: following, followed_by: followed_by, blocking: UserRelationship.exists?( @@ -150,7 +149,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do ), subscribing: subscribing, notifying: subscribing, - requested: follow_state == :follow_pending, + requested: requested, domain_blocking: User.blocks_domain?(reading_user, target), showing_reblogs: not UserRelationship.exists?( From 1797f5958a92f78dc79c5bf313755b16319c5d2d Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 5 Sep 2024 20:55:28 +0000 Subject: [PATCH 066/117] App orphans should only be removed if they are older than 15 mins --- lib/pleroma/web/o_auth/app.ex | 7 ++++++- test/pleroma/web/o_auth/app_test.exs | 13 +++++++++---- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/web/o_auth/app.ex b/lib/pleroma/web/o_auth/app.ex index 032011433..7661c2566 100644 --- a/lib/pleroma/web/o_auth/app.ex +++ b/lib/pleroma/web/o_auth/app.ex @@ -168,8 +168,13 @@ defmodule Pleroma.Web.OAuth.App do @spec remove_orphans(pos_integer()) :: :ok def remove_orphans(limit \\ 100) do + fifteen_mins_ago = DateTime.add(DateTime.utc_now(), -900, :second) + Repo.transaction(fn -> - from(a in __MODULE__, where: is_nil(a.user_id), limit: ^limit) + from(a in __MODULE__, + where: is_nil(a.user_id) and a.inserted_at < ^fifteen_mins_ago, + limit: ^limit + ) |> Repo.all() |> Enum.each(&Repo.delete(&1)) end) diff --git a/test/pleroma/web/o_auth/app_test.exs b/test/pleroma/web/o_auth/app_test.exs index 725ea3eb8..44219cf90 100644 --- a/test/pleroma/web/o_auth/app_test.exs +++ b/test/pleroma/web/o_auth/app_test.exs @@ -56,13 +56,18 @@ defmodule Pleroma.Web.OAuth.AppTest do test "removes orphaned apps" do attrs = %{client_name: "Mastodon-Local", redirect_uris: "."} - {:ok, %App{} = app} = App.get_or_make(attrs, ["write"]) - assert app.scopes == ["write"] + {:ok, %App{} = old_app} = App.get_or_make(attrs, ["write"]) - assert app == Pleroma.Repo.get_by(App, %{id: app.id}) + attrs = %{client_name: "PleromaFE", redirect_uris: "."} + {:ok, %App{} = app} = App.get_or_make(attrs, ["write"]) + + # backdate the old app so it's within the threshold for being cleaned up + {:ok, _} = + "UPDATE apps SET inserted_at = now() - interval '1 hour' WHERE id = #{old_app.id}" + |> Pleroma.Repo.query() App.remove_orphans() - assert nil == Pleroma.Repo.get_by(App, %{id: app.id}) + assert [app] == Pleroma.Repo.all(App) end end From e51cd31a576715d8e7d991e4fce850edcf4c8a1e Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 5 Sep 2024 17:06:53 -0400 Subject: [PATCH 067/117] Bump credo to prevent it from crashing --- mix.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mix.lock b/mix.lock index 07c1122aa..865e09a4c 100644 --- a/mix.lock +++ b/mix.lock @@ -22,7 +22,7 @@ "cowboy": {:hex, :cowboy, "2.12.0", "f276d521a1ff88b2b9b4c54d0e753da6c66dd7be6c9fca3d9418b561828a3731", [:make, :rebar3], [{:cowlib, "2.13.0", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "1.8.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "8a7abe6d183372ceb21caa2709bec928ab2b72e18a3911aa1771639bef82651e"}, "cowboy_telemetry": {:hex, :cowboy_telemetry, "0.4.0", "f239f68b588efa7707abce16a84d0d2acf3a0f50571f8bb7f56a15865aae820c", [:rebar3], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "7d98bac1ee4565d31b62d59f8823dfd8356a169e7fcbb83831b8a5397404c9de"}, "cowlib": {:hex, :cowlib, "2.13.0", "db8f7505d8332d98ef50a3ef34b34c1afddec7506e4ee4dd4a3a266285d282ca", [:make, :rebar3], [], "hexpm", "e1e1284dc3fc030a64b1ad0d8382ae7e99da46c3246b815318a4b848873800a4"}, - "credo": {:hex, :credo, "1.7.3", "05bb11eaf2f2b8db370ecaa6a6bda2ec49b2acd5e0418bc106b73b07128c0436", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "35ea675a094c934c22fb1dca3696f3c31f2728ae6ef5a53b5d648c11180a4535"}, + "credo": {:hex, :credo, "1.7.7", "771445037228f763f9b2afd612b6aa2fd8e28432a95dbbc60d8e03ce71ba4446", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "8bc87496c9aaacdc3f90f01b7b0582467b69b4bd2441fe8aae3109d843cc2f2e"}, "crontab": {:hex, :crontab, "1.1.8", "2ce0e74777dfcadb28a1debbea707e58b879e6aa0ffbf9c9bb540887bce43617", [:mix], [{:ecto, "~> 1.0 or ~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm"}, "custom_base": {:hex, :custom_base, "0.2.1", "4a832a42ea0552299d81652aa0b1f775d462175293e99dfbe4d7dbaab785a706", [:mix], [], "hexpm", "8df019facc5ec9603e94f7270f1ac73ddf339f56ade76a721eaa57c1493ba463"}, "db_connection": {:hex, :db_connection, "2.7.0", "b99faa9291bb09892c7da373bb82cba59aefa9b36300f6145c5f201c7adf48ec", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "dcf08f31b2701f857dfc787fbad78223d61a32204f217f15e881dd93e4bdd3ff"}, From 5f573b4095ade584a590b756c83f13f89336cd04 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 5 Sep 2024 17:11:02 -0400 Subject: [PATCH 068/117] Credo: comment line length --- test/pleroma/web/mastodon_api/views/account_view_test.exs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/pleroma/web/mastodon_api/views/account_view_test.exs b/test/pleroma/web/mastodon_api/views/account_view_test.exs index 5d2f55c6d..f88b90955 100644 --- a/test/pleroma/web/mastodon_api/views/account_view_test.exs +++ b/test/pleroma/web/mastodon_api/views/account_view_test.exs @@ -467,7 +467,8 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do assert %{data: %{"state" => "accept"}} = Pleroma.Web.ActivityPub.Utils.fetch_latest_follow(user, other_user) - # Fetch the relationship and forcibly delete it to simulate a Follow Accept that did not complete processing + # Fetch the relationship and forcibly delete it to simulate + # a Follow Accept that did not complete processing %{following_relationships: [relationship]} = Pleroma.UserRelationship.view_relationships_option(user, [other_user]) From a887188890a6b8c9e97c6cafe1776bb151e63843 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 6 Sep 2024 09:42:53 -0400 Subject: [PATCH 069/117] Oban: more unique job constraints --- changelog.d/oban-uniques.change | 1 + lib/pleroma/workers/receiver_worker.ex | 2 +- lib/pleroma/workers/remote_fetcher_worker.ex | 2 +- lib/pleroma/workers/rich_media_worker.ex | 2 +- lib/pleroma/workers/user_refresh_worker.ex | 2 +- lib/pleroma/workers/web_pusher_worker.ex | 2 +- 6 files changed, 6 insertions(+), 5 deletions(-) create mode 100644 changelog.d/oban-uniques.change diff --git a/changelog.d/oban-uniques.change b/changelog.d/oban-uniques.change new file mode 100644 index 000000000..d9deb4696 --- /dev/null +++ b/changelog.d/oban-uniques.change @@ -0,0 +1 @@ +Adjust more Oban workers to enforce unique job constraints. diff --git a/lib/pleroma/workers/receiver_worker.ex b/lib/pleroma/workers/receiver_worker.ex index 0373ec15f..11b672bef 100644 --- a/lib/pleroma/workers/receiver_worker.ex +++ b/lib/pleroma/workers/receiver_worker.ex @@ -7,7 +7,7 @@ defmodule Pleroma.Workers.ReceiverWorker do alias Pleroma.User alias Pleroma.Web.Federator - use Oban.Worker, queue: :federator_incoming, max_attempts: 5 + use Oban.Worker, queue: :federator_incoming, max_attempts: 5, unique: [period: :infinity] @impl true diff --git a/lib/pleroma/workers/remote_fetcher_worker.ex b/lib/pleroma/workers/remote_fetcher_worker.ex index 9d3f1ec53..aa09362f5 100644 --- a/lib/pleroma/workers/remote_fetcher_worker.ex +++ b/lib/pleroma/workers/remote_fetcher_worker.ex @@ -5,7 +5,7 @@ defmodule Pleroma.Workers.RemoteFetcherWorker do alias Pleroma.Object.Fetcher - use Oban.Worker, queue: :background + use Oban.Worker, queue: :background, unique: [period: :infinity] @impl true def perform(%Job{args: %{"op" => "fetch_remote", "id" => id} = args}) do diff --git a/lib/pleroma/workers/rich_media_worker.ex b/lib/pleroma/workers/rich_media_worker.ex index d5ba7b63e..e351ecd6e 100644 --- a/lib/pleroma/workers/rich_media_worker.ex +++ b/lib/pleroma/workers/rich_media_worker.ex @@ -7,7 +7,7 @@ defmodule Pleroma.Workers.RichMediaWorker do alias Pleroma.Web.RichMedia.Backfill alias Pleroma.Web.RichMedia.Card - use Oban.Worker, queue: :background, max_attempts: 3, unique: [period: 300] + use Oban.Worker, queue: :background, max_attempts: 3, unique: [period: :infinity] @impl true def perform(%Job{args: %{"op" => "expire", "url" => url} = _args}) do diff --git a/lib/pleroma/workers/user_refresh_worker.ex b/lib/pleroma/workers/user_refresh_worker.ex index 222a4a8f7..ee276774b 100644 --- a/lib/pleroma/workers/user_refresh_worker.ex +++ b/lib/pleroma/workers/user_refresh_worker.ex @@ -3,7 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Workers.UserRefreshWorker do - use Oban.Worker, queue: :background, max_attempts: 1, unique: [period: 300] + use Oban.Worker, queue: :background, max_attempts: 1, unique: [period: :infinity] alias Pleroma.User diff --git a/lib/pleroma/workers/web_pusher_worker.ex b/lib/pleroma/workers/web_pusher_worker.ex index f4232d02a..879b26cc3 100644 --- a/lib/pleroma/workers/web_pusher_worker.ex +++ b/lib/pleroma/workers/web_pusher_worker.ex @@ -7,7 +7,7 @@ defmodule Pleroma.Workers.WebPusherWorker do alias Pleroma.Repo alias Pleroma.Web.Push.Impl - use Oban.Worker, queue: :web_push + use Oban.Worker, queue: :web_push, unique: [period: :infinity] @impl true def perform(%Job{args: %{"op" => "web_push", "notification_id" => notification_id}}) do From fc3ea94a1c17e84033fa593c0ea987fbfa545447 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 6 Sep 2024 09:58:03 -0400 Subject: [PATCH 070/117] Dialyzer: the pattern can never match the type --- lib/pleroma/object/fetcher.ex | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index 9d9a201ca..ff7aa539f 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -58,8 +58,11 @@ defmodule Pleroma.Object.Fetcher do end end + @typep fetcher_errors :: + :error | :reject | :allowed_depth | :fetch | :containment | :transmogrifier + # Note: will create a Create activity, which we need internally at the moment. - @spec fetch_object_from_id(String.t(), list()) :: {:ok, Object.t()} | {:error | :reject, any()} + @spec fetch_object_from_id(String.t(), list()) :: {:ok, Object.t()} | {fetcher_errors(), any()} def fetch_object_from_id(id, options \\ []) do with {_, nil} <- {:fetch_object, Object.get_cached_by_ap_id(id)}, {_, true} <- {:allowed_depth, Federator.allowed_thread_distance?(options[:depth])}, From bc16f09d7b9c1a15c8c9be6d99092b9a8d867d18 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 6 Sep 2024 11:12:13 -0400 Subject: [PATCH 071/117] Dialyzer: the pattern can never match the type The original error was for the chat controller: lib/pleroma/web/pleroma_api/controllers/chat_controller.ex:104:pattern_match The pattern can never match the type {:error, :content_too_long | :forbidden | :no_content | :not_found} | {:user, nil}. Improve typespecs for the Pipeline and apply them where it could be encountered --- lib/pleroma/object/fetcher.ex | 3 +- lib/pleroma/web/activity_pub/pipeline.ex | 19 ++++++---- lib/pleroma/web/common_api.ex | 44 ++++++++++++++---------- 3 files changed, 39 insertions(+), 27 deletions(-) diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index ff7aa539f..69a5f3268 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -62,7 +62,8 @@ defmodule Pleroma.Object.Fetcher do :error | :reject | :allowed_depth | :fetch | :containment | :transmogrifier # Note: will create a Create activity, which we need internally at the moment. - @spec fetch_object_from_id(String.t(), list()) :: {:ok, Object.t()} | {fetcher_errors(), any()} + @spec fetch_object_from_id(String.t(), list()) :: + {:ok, Object.t()} | {fetcher_errors(), any()} | Pipeline.errors() def fetch_object_from_id(id, options \\ []) do with {_, nil} <- {:fetch_object, Object.get_cached_by_ap_id(id)}, {_, true} <- {:allowed_depth, Federator.allowed_thread_distance?(options[:depth])}, diff --git a/lib/pleroma/web/activity_pub/pipeline.ex b/lib/pleroma/web/activity_pub/pipeline.ex index 7f11a4d67..fc36935d5 100644 --- a/lib/pleroma/web/activity_pub/pipeline.ex +++ b/lib/pleroma/web/activity_pub/pipeline.ex @@ -22,22 +22,27 @@ defmodule Pleroma.Web.ActivityPub.Pipeline do defp activity_pub, do: Config.get([:pipeline, :activity_pub], ActivityPub) defp config, do: Config.get([:pipeline, :config], Config) - @spec common_pipeline(map(), keyword()) :: - {:ok, Activity.t() | Object.t(), keyword()} | {:error | :reject, any()} + @type results :: {:ok, Activity.t() | Object.t(), keyword()} + @type errors :: {:error | :reject, any()} + + # The Repo.transaction will wrap the result in an {:ok, _} + # and only returns an {:error, _} if the error encountered was related + # to the SQL transaction + @spec common_pipeline(map(), keyword()) :: results() | errors() def common_pipeline(object, meta) do case Repo.transaction(fn -> do_common_pipeline(object, meta) end, Utils.query_timeout()) do {:ok, {:ok, activity, meta}} -> side_effects().handle_after_transaction(meta) {:ok, activity, meta} - {:ok, value} -> - value + {:ok, {:error, _} = error} -> + error + + {:ok, {:reject, _} = error} -> + error {:error, e} -> {:error, e} - - {:reject, e} -> - {:reject, e} end end diff --git a/lib/pleroma/web/common_api.ex b/lib/pleroma/web/common_api.ex index 921e414c3..412424dae 100644 --- a/lib/pleroma/web/common_api.ex +++ b/lib/pleroma/web/common_api.ex @@ -26,7 +26,7 @@ defmodule Pleroma.Web.CommonAPI do require Pleroma.Constants require Logger - @spec block(User.t(), User.t()) :: {:ok, Activity.t()} | {:error, any()} + @spec block(User.t(), User.t()) :: {:ok, Activity.t()} | Pipeline.errors() def block(blocked, blocker) do with {:ok, block_data, _} <- Builder.block(blocker, blocked), {:ok, block, _} <- Pipeline.common_pipeline(block_data, local: true) do @@ -35,7 +35,7 @@ defmodule Pleroma.Web.CommonAPI do end @spec post_chat_message(User.t(), User.t(), String.t(), list()) :: - {:ok, Activity.t()} | {:error, any()} + {:ok, Activity.t()} | Pipeline.errors() def post_chat_message(%User{} = user, %User{} = recipient, content, opts \\ []) do with maybe_attachment <- opts[:media_id] && Object.get_by_id(opts[:media_id]), :ok <- validate_chat_attachment_attribution(maybe_attachment, user), @@ -58,7 +58,7 @@ defmodule Pleroma.Web.CommonAPI do )} do {:ok, activity} else - {:common_pipeline, {:reject, _} = e} -> e + {:common_pipeline, e} -> e e -> e end end @@ -99,7 +99,8 @@ defmodule Pleroma.Web.CommonAPI do end end - @spec unblock(User.t(), User.t()) :: {:ok, Activity.t()} | {:error, any()} + @spec unblock(User.t(), User.t()) :: + {:ok, Activity.t()} | {:ok, :no_activity} | Pipeline.errors() | {:error, :not_blocking} def unblock(blocked, blocker) do with {_, %Activity{} = block} <- {:fetch_block, Utils.fetch_latest_block(blocker, blocked)}, {:ok, unblock_data, _} <- Builder.undo(blocker, block), @@ -120,7 +121,9 @@ defmodule Pleroma.Web.CommonAPI do end @spec follow(User.t(), User.t()) :: - {:ok, User.t(), User.t(), Activity.t() | Object.t()} | {:error, :rejected} + {:ok, User.t(), User.t(), Activity.t() | Object.t()} + | {:error, :rejected} + | Pipeline.errors() def follow(followed, follower) do timeout = Pleroma.Config.get([:activitypub, :follow_handshake_timeout]) @@ -145,7 +148,7 @@ defmodule Pleroma.Web.CommonAPI do end end - @spec accept_follow_request(User.t(), User.t()) :: {:ok, User.t()} | {:error, any()} + @spec accept_follow_request(User.t(), User.t()) :: {:ok, User.t()} | Pipeline.errors() def accept_follow_request(follower, followed) do with %Activity{} = follow_activity <- Utils.fetch_latest_follow(follower, followed), {:ok, accept_data, _} <- Builder.accept(followed, follow_activity), @@ -154,7 +157,7 @@ defmodule Pleroma.Web.CommonAPI do end end - @spec reject_follow_request(User.t(), User.t()) :: {:ok, User.t()} | {:error, any()} | nil + @spec reject_follow_request(User.t(), User.t()) :: {:ok, User.t()} | Pipeline.errors() | nil def reject_follow_request(follower, followed) do with %Activity{} = follow_activity <- Utils.fetch_latest_follow(follower, followed), {:ok, reject_data, _} <- Builder.reject(followed, follow_activity), @@ -163,7 +166,8 @@ defmodule Pleroma.Web.CommonAPI do end end - @spec delete(String.t(), User.t()) :: {:ok, Activity.t()} | {:error, any()} + @spec delete(String.t(), User.t()) :: + {:ok, Activity.t()} | Pipeline.errors() | {:error, :not_found | String.t()} def delete(activity_id, user) do with {_, %Activity{data: %{"object" => _, "type" => "Create"}} = activity} <- {:find_activity, Activity.get_by_id(activity_id, filter: [])}, @@ -213,7 +217,7 @@ defmodule Pleroma.Web.CommonAPI do end end - @spec repeat(String.t(), User.t(), map()) :: {:ok, Activity.t()} | {:error, any()} + @spec repeat(String.t(), User.t(), map()) :: {:ok, Activity.t()} | {:error, :not_found} def repeat(id, user, params \\ %{}) do with %Activity{data: %{"type" => "Create"}} = activity <- Activity.get_by_id(id), object = %Object{} <- Object.normalize(activity, fetch: false), @@ -231,7 +235,7 @@ defmodule Pleroma.Web.CommonAPI do end end - @spec unrepeat(String.t(), User.t()) :: {:ok, Activity.t()} | {:error, any()} + @spec unrepeat(String.t(), User.t()) :: {:ok, Activity.t()} | {:error, :not_found | String.t()} def unrepeat(id, user) do with {_, %Activity{data: %{"type" => "Create"}} = activity} <- {:find_activity, Activity.get_by_id(id)}, @@ -247,7 +251,8 @@ defmodule Pleroma.Web.CommonAPI do end end - @spec favorite(String.t(), User.t()) :: {:ok, Activity.t()} | {:error, any()} + @spec favorite(String.t(), User.t()) :: + {:ok, Activity.t()} | {:ok, :already_liked} | {:error, :not_found | String.t()} def favorite(id, %User{} = user) do case favorite_helper(user, id) do {:ok, _} = res -> @@ -285,7 +290,8 @@ defmodule Pleroma.Web.CommonAPI do end end - @spec unfavorite(String.t(), User.t()) :: {:ok, Activity.t()} | {:error, any()} + @spec unfavorite(String.t(), User.t()) :: + {:ok, Activity.t()} | {:error, :not_found | String.t()} def unfavorite(id, user) do with {_, %Activity{data: %{"type" => "Create"}} = activity} <- {:find_activity, Activity.get_by_id(id)}, @@ -302,7 +308,7 @@ defmodule Pleroma.Web.CommonAPI do end @spec react_with_emoji(String.t(), User.t(), String.t()) :: - {:ok, Activity.t()} | {:error, any()} + {:ok, Activity.t()} | {:error, String.t()} def react_with_emoji(id, user, emoji) do with %Activity{} = activity <- Activity.get_by_id(id), object <- Object.normalize(activity, fetch: false), @@ -316,7 +322,7 @@ defmodule Pleroma.Web.CommonAPI do end @spec unreact_with_emoji(String.t(), User.t(), String.t()) :: - {:ok, Activity.t()} | {:error, any()} + {:ok, Activity.t()} | {:error, String.t()} def unreact_with_emoji(id, user, emoji) do with %Activity{} = reaction_activity <- Utils.get_latest_reaction(id, user, emoji), {_, {:ok, _}} <- {:cancel_jobs, maybe_cancel_jobs(reaction_activity)}, @@ -329,7 +335,7 @@ defmodule Pleroma.Web.CommonAPI do end end - @spec vote(Object.t(), User.t(), list()) :: {:ok, list(), Object.t()} | {:error, any()} + @spec vote(Object.t(), User.t(), list()) :: {:ok, list(), Object.t()} | Pipeline.errors() def vote(%Object{data: %{"type" => "Question"}} = object, %User{} = user, choices) do with :ok <- validate_not_author(object, user), :ok <- validate_existing_votes(user, object), @@ -461,7 +467,7 @@ defmodule Pleroma.Web.CommonAPI do end end - @spec update(Activity.t(), User.t(), map()) :: {:ok, Activity.t()} | {:error, any()} + @spec update(Activity.t(), User.t(), map()) :: {:ok, Activity.t()} | {:error, nil} def update(orig_activity, %User{} = user, changes) do with orig_object <- Object.normalize(orig_activity), {:ok, new_object} <- make_update_data(user, orig_object, changes), @@ -497,7 +503,7 @@ defmodule Pleroma.Web.CommonAPI do end end - @spec pin(String.t(), User.t()) :: {:ok, Activity.t()} | {:error, term()} + @spec pin(String.t(), User.t()) :: {:ok, Activity.t()} | Pipeline.errors() def pin(id, %User{} = user) do with %Activity{} = activity <- create_activity_by_id(id), true <- activity_belongs_to_actor(activity, user.ap_id), @@ -537,7 +543,7 @@ defmodule Pleroma.Web.CommonAPI do end end - @spec unpin(String.t(), User.t()) :: {:ok, Activity.t()} | {:error, term()} + @spec unpin(String.t(), User.t()) :: {:ok, Activity.t()} | Pipeline.errors() def unpin(id, user) do with %Activity{} = activity <- create_activity_by_id(id), {:ok, unpin_data, _} <- Builder.unpin(user, activity.object), @@ -552,7 +558,7 @@ defmodule Pleroma.Web.CommonAPI do end end - @spec add_mute(Activity.t(), User.t(), map()) :: {:ok, Activity.t()} | {:error, any()} + @spec add_mute(Activity.t(), User.t(), map()) :: {:ok, Activity.t()} | {:error, String.t()} def add_mute(activity, user, params \\ %{}) do expires_in = Map.get(params, :expires_in, 0) From 7eb579c1911f2eac175c2030f6bb80685b4ab4f8 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 6 Sep 2024 11:18:12 -0400 Subject: [PATCH 072/117] Dialyzer: invalid contract --- lib/pleroma/user/import.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/user/import.ex b/lib/pleroma/user/import.ex index b79fa88eb..ab6bdb8d4 100644 --- a/lib/pleroma/user/import.ex +++ b/lib/pleroma/user/import.ex @@ -12,7 +12,7 @@ defmodule Pleroma.User.Import do require Logger - @spec perform(atom(), User.t(), list()) :: :ok | list() | {:error, any()} + @spec perform(atom(), User.t(), String.t()) :: :ok | {:error, any()} def perform(:mute_import, %User{} = user, actor) do with {:ok, %User{} = muted_user} <- User.get_or_fetch(actor), {_, false} <- {:existing_mute, User.mutes_user?(user, muted_user)}, @@ -49,7 +49,7 @@ defmodule Pleroma.User.Import do defp handle_error(op, user_id, error) do Logger.debug("#{op} failed for #{user_id} with: #{inspect(error)}") - error + {:error, error} end def blocks_import(%User{} = user, [_ | _] = actors) do From 06d6febff960e5aafd44709d0a61311b45892a81 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 6 Sep 2024 11:19:24 -0400 Subject: [PATCH 073/117] Dialyzer: The pattern variable _e@1 can never match the type, because it is covered by previous clauses. --- lib/pleroma/user/backup.ex | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/pleroma/user/backup.ex b/lib/pleroma/user/backup.ex index 7feaa22bf..70cf5b2a1 100644 --- a/lib/pleroma/user/backup.ex +++ b/lib/pleroma/user/backup.ex @@ -92,9 +92,6 @@ defmodule Pleroma.User.Backup do else true -> {:error, "Backup is missing id. Please insert it into the Repo first."} - - e -> - {:error, e} end end From 1d0e3b1355c5a5883be1522f0f925c398c0e87a4 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 6 Sep 2024 11:24:37 -0400 Subject: [PATCH 074/117] Dialyzer: The pattern variable _ can never match the type, because it is covered by previous clauses. --- lib/pleroma/user/backup.ex | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/pleroma/user/backup.ex b/lib/pleroma/user/backup.ex index 70cf5b2a1..c5038c8f4 100644 --- a/lib/pleroma/user/backup.ex +++ b/lib/pleroma/user/backup.ex @@ -294,9 +294,6 @@ defmodule Pleroma.User.Backup do ) acc - - _ -> - acc end end) From 06ce5e3b43b4a6809397bdb0eb192a82e7243e93 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 6 Sep 2024 11:27:07 -0400 Subject: [PATCH 075/117] Dialyzer: pattern_match The pattern can never match the type {:diff, false}. --- lib/pleroma/user/backup.ex | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/user/backup.ex b/lib/pleroma/user/backup.ex index c5038c8f4..d77d49890 100644 --- a/lib/pleroma/user/backup.ex +++ b/lib/pleroma/user/backup.ex @@ -118,14 +118,13 @@ defmodule Pleroma.User.Backup do end defp permitted?(user) do - with {_, %__MODULE__{inserted_at: inserted_at}} <- {:last, get_last(user)}, - days = Config.get([__MODULE__, :limit_days]), - diff = Timex.diff(NaiveDateTime.utc_now(), inserted_at, :days), - {_, true} <- {:diff, diff > days} do - true + with {_, %__MODULE__{inserted_at: inserted_at}} <- {:last, get_last(user)} do + days = Config.get([__MODULE__, :limit_days]) + diff = Timex.diff(NaiveDateTime.utc_now(), inserted_at, :days) + + diff > days else {:last, nil} -> true - {:diff, false} -> false end end From 5b26c56624ad281987a18091a6ae245833d2fde1 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 6 Sep 2024 11:34:06 -0400 Subject: [PATCH 076/117] Changelog --- changelog.d/dialyzer.skip | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 changelog.d/dialyzer.skip diff --git a/changelog.d/dialyzer.skip b/changelog.d/dialyzer.skip new file mode 100644 index 000000000..e69de29bb From 1afcfd4845fb71e111b7cbcf18858bb100863f8a Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 6 Sep 2024 11:51:16 -0400 Subject: [PATCH 077/117] Add tests for Mastodon mention hashtag class --- test/pleroma/html_test.exs | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/test/pleroma/html_test.exs b/test/pleroma/html_test.exs index 1be161971..d17b07540 100644 --- a/test/pleroma/html_test.exs +++ b/test/pleroma/html_test.exs @@ -41,6 +41,10 @@ defmodule Pleroma.HTMLTest do @foo """ + @mention_hashtags_sample """ + + """ + describe "StripTags scrubber" do test "works as expected" do expected = """ @@ -126,6 +130,15 @@ defmodule Pleroma.HTMLTest do Pleroma.HTML.Scrubber.TwitterText ) end + + test "does allow mention hashtags" do + expected = """ + + """ + + assert expected == + HTML.filter_tags(@mention_hashtags_sample, Pleroma.HTML.Scrubber.Default) + end end describe "default scrubber" do @@ -189,6 +202,15 @@ defmodule Pleroma.HTMLTest do Pleroma.HTML.Scrubber.Default ) end + + test "does allow mention hashtags" do + expected = """ + + """ + + assert expected == + HTML.filter_tags(@mention_hashtags_sample, Pleroma.HTML.Scrubber.Default) + end end describe "extract_first_external_url_from_object" do From c9b28eaf9a484fa1f2c27d00855c997575369782 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Sun, 8 Sep 2024 05:23:46 +0300 Subject: [PATCH 078/117] Argon2 password support --- lib/pleroma/web/plugs/authentication_plug.ex | 4 ++++ mix.exs | 1 + mix.lock | 1 + 3 files changed, 6 insertions(+) diff --git a/lib/pleroma/web/plugs/authentication_plug.ex b/lib/pleroma/web/plugs/authentication_plug.ex index f912a1542..3fc6e7b51 100644 --- a/lib/pleroma/web/plugs/authentication_plug.ex +++ b/lib/pleroma/web/plugs/authentication_plug.ex @@ -47,6 +47,10 @@ defmodule Pleroma.Web.Plugs.AuthenticationPlug do Pleroma.Password.Pbkdf2.verify_pass(password, password_hash) end + def checkpw(password, "$argon2" <> _ = password_hash) do + Argon2.verify_pass(password, password_hash) + end + def checkpw(_password, _password_hash) do Logger.error("Password hash not recognized") false diff --git a/mix.exs b/mix.exs index df44934d7..0d49a6b45 100644 --- a/mix.exs +++ b/mix.exs @@ -203,6 +203,7 @@ defmodule Pleroma.Mixfile do {:websock_adapter, "~> 0.5.6"}, {:oban_live_dashboard, "~> 0.1.1"}, {:multipart, "~> 0.4.0", optional: true}, + {:argon2_elixir, "~> 4.0"}, ## dev & test {:phoenix_live_reload, "~> 1.3.3", only: :dev}, diff --git a/mix.lock b/mix.lock index 865e09a4c..01f2eef98 100644 --- a/mix.lock +++ b/mix.lock @@ -1,5 +1,6 @@ %{ "accept": {:hex, :accept, "0.3.5", "b33b127abca7cc948bbe6caa4c263369abf1347cfa9d8e699c6d214660f10cd1", [:rebar3], [], "hexpm", "11b18c220bcc2eab63b5470c038ef10eb6783bcb1fcdb11aa4137defa5ac1bb8"}, + "argon2_elixir": {:hex, :argon2_elixir, "4.0.0", "7f6cd2e4a93a37f61d58a367d82f830ad9527082ff3c820b8197a8a736648941", [:make, :mix], [{:comeonin, "~> 5.3", [hex: :comeonin, repo: "hexpm", optional: false]}, {:elixir_make, "~> 0.6", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm", "f9da27cf060c9ea61b1bd47837a28d7e48a8f6fa13a745e252556c14f9132c7f"}, "bandit": {:hex, :bandit, "1.5.5", "df28f1c41f745401fe9e85a6882033f5f3442ab6d30c8a2948554062a4ab56e0", [:mix], [{:hpax, "~> 0.2.0", [hex: :hpax, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:thousand_island, "~> 1.0", [hex: :thousand_island, repo: "hexpm", optional: false]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "f21579a29ea4bc08440343b2b5f16f7cddf2fea5725d31b72cf973ec729079e1"}, "base62": {:hex, :base62, "1.2.2", "85c6627eb609317b70f555294045895ffaaeb1758666ab9ef9ca38865b11e629", [:mix], [{:custom_base, "~> 0.2.1", [hex: :custom_base, repo: "hexpm", optional: false]}], "hexpm", "d41336bda8eaa5be197f1e4592400513ee60518e5b9f4dcf38f4b4dae6f377bb"}, "bbcode_pleroma": {:hex, :bbcode_pleroma, "0.2.0", "d36f5bca6e2f62261c45be30fa9b92725c0655ad45c99025cb1c3e28e25803ef", [:mix], [{:nimble_parsec, "~> 0.5", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "19851074419a5fedb4ef49e1f01b30df504bb5dbb6d6adfc135238063bebd1c3"}, From 9de522ce5048bd72dd083a1661506b563be27cc1 Mon Sep 17 00:00:00 2001 From: Mint Date: Sun, 8 Sep 2024 05:32:40 +0300 Subject: [PATCH 079/117] Authentication: convert argon2 passwords, add tests --- lib/pleroma/web/plugs/authentication_plug.ex | 5 ++++ .../web/plugs/authentication_plug_test.exs | 26 +++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/lib/pleroma/web/plugs/authentication_plug.ex b/lib/pleroma/web/plugs/authentication_plug.ex index 3fc6e7b51..af7d7f45a 100644 --- a/lib/pleroma/web/plugs/authentication_plug.ex +++ b/lib/pleroma/web/plugs/authentication_plug.ex @@ -48,6 +48,7 @@ defmodule Pleroma.Web.Plugs.AuthenticationPlug do end def checkpw(password, "$argon2" <> _ = password_hash) do + # Handle argon2 passwords for Akkoma migration Argon2.verify_pass(password, password_hash) end @@ -60,6 +61,10 @@ defmodule Pleroma.Web.Plugs.AuthenticationPlug do do_update_password(user, password) end + def maybe_update_password(%User{password_hash: "$argon2" <> _} = user, password) do + do_update_password(user, password) + end + def maybe_update_password(user, _), do: {:ok, user} defp do_update_password(user, password) do diff --git a/test/pleroma/web/plugs/authentication_plug_test.exs b/test/pleroma/web/plugs/authentication_plug_test.exs index b8acd01c5..bdbf3de32 100644 --- a/test/pleroma/web/plugs/authentication_plug_test.exs +++ b/test/pleroma/web/plugs/authentication_plug_test.exs @@ -70,6 +70,24 @@ defmodule Pleroma.Web.Plugs.AuthenticationPlugTest do assert "$pbkdf2" <> _ = user.password_hash end + test "with an argon2 hash, it updates to a pkbdf2 hash", %{conn: conn} do + user = insert(:user, password_hash: Argon2.hash_pwd_salt("123")) + assert "$argon2" <> _ = user.password_hash + + conn = + conn + |> assign(:auth_user, user) + |> assign(:auth_credentials, %{password: "123"}) + |> AuthenticationPlug.call(%{}) + + assert conn.assigns.user.id == conn.assigns.auth_user.id + assert conn.assigns.token == nil + assert PlugHelper.plug_skipped?(conn, OAuthScopesPlug) + + user = User.get_by_id(user.id) + assert "$pbkdf2" <> _ = user.password_hash + end + describe "checkpw/2" do test "check pbkdf2 hash" do hash = @@ -86,6 +104,14 @@ defmodule Pleroma.Web.Plugs.AuthenticationPlugTest do refute AuthenticationPlug.checkpw("password1", hash) end + test "check argon2 hash" do + hash = + "$argon2id$v=19$m=65536,t=8,p=2$zEMMsTuK5KkL5AFWbX7jyQ$VyaQD7PF6e9btz0oH1YiAkWwIGZ7WNDZP8l+a/O171g" + + assert AuthenticationPlug.checkpw("password", hash) + refute AuthenticationPlug.checkpw("password1", hash) + end + test "it returns false when hash invalid" do hash = "psBWV8gxkGOZWBz$PmfCycChoxeJ3GgGzwvhlgacb9mUoZ.KUXNCssekER4SJ7bOK53uXrHNb2e4i8yPFgSKyzaW9CcmrDXWIEMtD1" From 7e91c3a306b8f050e6a88e888fd439b579c1f125 Mon Sep 17 00:00:00 2001 From: Mint Date: Sun, 8 Sep 2024 05:41:48 +0300 Subject: [PATCH 080/117] Changelog --- changelog.d/argon2-passwords.add | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/argon2-passwords.add diff --git a/changelog.d/argon2-passwords.add b/changelog.d/argon2-passwords.add new file mode 100644 index 000000000..36fd7faf2 --- /dev/null +++ b/changelog.d/argon2-passwords.add @@ -0,0 +1 @@ +Added support for argon2 passwords and their conversion for migration from Akkoma fork to upstream. From 7def11d7c352f13ce0f12715649359344cbba9a6 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 11 Sep 2024 12:45:33 -0400 Subject: [PATCH 081/117] LDAP Auth: fix TLS certificate verification Currently we only support STARTTLS and it was not verifying certificate and hostname correctly. We must pass a custom fqdn_fun/1 function so it knows what value to compare against. --- changelog.d/ldap-tls.fix | 1 + lib/pleroma/web/auth/ldap_authenticator.ex | 12 +++++++++++- mix.exs | 1 + 3 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 changelog.d/ldap-tls.fix diff --git a/changelog.d/ldap-tls.fix b/changelog.d/ldap-tls.fix new file mode 100644 index 000000000..b15137d77 --- /dev/null +++ b/changelog.d/ldap-tls.fix @@ -0,0 +1 @@ +STARTTLS certificate and hostname verification for LDAP authentication diff --git a/lib/pleroma/web/auth/ldap_authenticator.ex b/lib/pleroma/web/auth/ldap_authenticator.ex index ea5620cf6..d31f34747 100644 --- a/lib/pleroma/web/auth/ldap_authenticator.ex +++ b/lib/pleroma/web/auth/ldap_authenticator.ex @@ -41,6 +41,7 @@ defmodule Pleroma.Web.Auth.LDAPAuthenticator do port = Keyword.get(ldap, :port, 389) ssl = Keyword.get(ldap, :ssl, false) sslopts = Keyword.get(ldap, :sslopts, []) + tlsopts = Keyword.get(ldap, :tlsopts, []) options = [{:port, port}, {:ssl, ssl}, {:timeout, @connection_timeout}] ++ @@ -54,7 +55,16 @@ defmodule Pleroma.Web.Auth.LDAPAuthenticator do case :eldap.start_tls( connection, - Keyword.get(ldap, :tlsopts, []), + Keyword.merge( + [ + verify: :verify_peer, + cacerts: :certifi.cacerts(), + customize_hostname_check: [ + fqdn_fun: fn _ -> to_charlist(host) end + ] + ], + tlsopts + ), @connection_timeout ) do :ok -> diff --git a/mix.exs b/mix.exs index 0d49a6b45..9a261547f 100644 --- a/mix.exs +++ b/mix.exs @@ -204,6 +204,7 @@ defmodule Pleroma.Mixfile do {:oban_live_dashboard, "~> 0.1.1"}, {:multipart, "~> 0.4.0", optional: true}, {:argon2_elixir, "~> 4.0"}, + {:certifi, "~> 2.12"}, ## dev & test {:phoenix_live_reload, "~> 1.3.3", only: :dev}, From affdcdb68daabb15f8fad2e7b6406606e8086e75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Thu, 12 Sep 2024 11:27:29 +0200 Subject: [PATCH 082/117] Manifest: declare /static/logo.svg as 512x512 to match one provided by pleroma-fe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- changelog.d/manifest-icon-size.skip | 0 config/config.exs | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 changelog.d/manifest-icon-size.skip diff --git a/changelog.d/manifest-icon-size.skip b/changelog.d/manifest-icon-size.skip new file mode 100644 index 000000000..e69de29bb diff --git a/config/config.exs b/config/config.exs index 80a3b8d57..cd9a2539f 100644 --- a/config/config.exs +++ b/config/config.exs @@ -344,7 +344,7 @@ config :pleroma, :manifest, icons: [ %{ src: "/static/logo.svg", - sizes: "144x144", + sizes: "512x512", purpose: "any", type: "image/svg+xml" } From 17b69c43d5ed6ba867f5fb1da15f6af9aa7c5d00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Thu, 12 Sep 2024 14:37:37 +0200 Subject: [PATCH 083/117] Add `group_key` to notifications MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- changelog.d/notifications-group-key.add | 1 + .../api_spec/operations/notification_operation.ex | 5 +++++ .../web/mastodon_api/views/notification_view.ex | 1 + .../mastodon_api/views/notification_view_test.exs | 13 +++++++++++++ 4 files changed, 20 insertions(+) create mode 100644 changelog.d/notifications-group-key.add diff --git a/changelog.d/notifications-group-key.add b/changelog.d/notifications-group-key.add new file mode 100644 index 000000000..386927f4a --- /dev/null +++ b/changelog.d/notifications-group-key.add @@ -0,0 +1 @@ +Add `group_key` to notifications \ No newline at end of file diff --git a/lib/pleroma/web/api_spec/operations/notification_operation.ex b/lib/pleroma/web/api_spec/operations/notification_operation.ex index 2dc0f66df..94d1f6b82 100644 --- a/lib/pleroma/web/api_spec/operations/notification_operation.ex +++ b/lib/pleroma/web/api_spec/operations/notification_operation.ex @@ -158,6 +158,10 @@ defmodule Pleroma.Web.ApiSpec.NotificationOperation do type: :object, properties: %{ id: %Schema{type: :string}, + group_key: %Schema{ + type: :string, + description: "Group key shared by similar notifications" + }, type: notification_type(), created_at: %Schema{type: :string, format: :"date-time"}, account: %Schema{ @@ -180,6 +184,7 @@ defmodule Pleroma.Web.ApiSpec.NotificationOperation do }, example: %{ "id" => "34975861", + "group-key" => "ungrouped-34975861", "type" => "mention", "created_at" => "2019-11-23T07:49:02.064Z", "account" => Account.schema().example, diff --git a/lib/pleroma/web/mastodon_api/views/notification_view.ex b/lib/pleroma/web/mastodon_api/views/notification_view.ex index 3f2478719..c277af98b 100644 --- a/lib/pleroma/web/mastodon_api/views/notification_view.ex +++ b/lib/pleroma/web/mastodon_api/views/notification_view.ex @@ -95,6 +95,7 @@ defmodule Pleroma.Web.MastodonAPI.NotificationView do response = %{ id: to_string(notification.id), + group_key: "ungrouped-" <> to_string(notification.id), type: notification.type, created_at: CommonAPI.Utils.to_masto_date(notification.inserted_at), account: account, diff --git a/test/pleroma/web/mastodon_api/views/notification_view_test.exs b/test/pleroma/web/mastodon_api/views/notification_view_test.exs index 75ab375aa..b1f3523ac 100644 --- a/test/pleroma/web/mastodon_api/views/notification_view_test.exs +++ b/test/pleroma/web/mastodon_api/views/notification_view_test.exs @@ -56,6 +56,7 @@ defmodule Pleroma.Web.MastodonAPI.NotificationViewTest do expected = %{ id: to_string(notification.id), + group_key: "ungrouped-#{to_string(notification.id)}", pleroma: %{is_seen: false, is_muted: false}, type: "pleroma:chat_mention", account: AccountView.render("show.json", %{user: user, for: recipient}), @@ -75,6 +76,7 @@ defmodule Pleroma.Web.MastodonAPI.NotificationViewTest do expected = %{ id: to_string(notification.id), + group_key: "ungrouped-#{to_string(notification.id)}", pleroma: %{is_seen: false, is_muted: false}, type: "mention", account: @@ -99,6 +101,7 @@ defmodule Pleroma.Web.MastodonAPI.NotificationViewTest do expected = %{ id: to_string(notification.id), + group_key: "ungrouped-#{to_string(notification.id)}", pleroma: %{is_seen: false, is_muted: false}, type: "favourite", account: AccountView.render("show.json", %{user: another_user, for: user}), @@ -119,6 +122,7 @@ defmodule Pleroma.Web.MastodonAPI.NotificationViewTest do expected = %{ id: to_string(notification.id), + group_key: "ungrouped-#{to_string(notification.id)}", pleroma: %{is_seen: false, is_muted: false}, type: "reblog", account: AccountView.render("show.json", %{user: another_user, for: user}), @@ -137,6 +141,7 @@ defmodule Pleroma.Web.MastodonAPI.NotificationViewTest do expected = %{ id: to_string(notification.id), + group_key: "ungrouped-#{to_string(notification.id)}", pleroma: %{is_seen: false, is_muted: false}, type: "follow", account: AccountView.render("show.json", %{user: follower, for: followed}), @@ -165,6 +170,7 @@ defmodule Pleroma.Web.MastodonAPI.NotificationViewTest do expected = %{ id: to_string(notification.id), + group_key: "ungrouped-#{to_string(notification.id)}", pleroma: %{is_seen: false, is_muted: false}, type: "move", account: AccountView.render("show.json", %{user: old_user, for: follower}), @@ -190,6 +196,7 @@ defmodule Pleroma.Web.MastodonAPI.NotificationViewTest do expected = %{ id: to_string(notification.id), + group_key: "ungrouped-#{to_string(notification.id)}", pleroma: %{is_seen: false, is_muted: false}, type: "pleroma:emoji_reaction", emoji: "☕", @@ -229,6 +236,7 @@ defmodule Pleroma.Web.MastodonAPI.NotificationViewTest do expected = %{ id: to_string(notification.id), + group_key: "ungrouped-#{to_string(notification.id)}", pleroma: %{is_seen: false, is_muted: false}, type: "pleroma:emoji_reaction", emoji: ":dinosaur:", @@ -248,6 +256,7 @@ defmodule Pleroma.Web.MastodonAPI.NotificationViewTest do expected = %{ id: to_string(notification.id), + group_key: "ungrouped-#{to_string(notification.id)}", pleroma: %{is_seen: false, is_muted: false}, type: "poll", account: @@ -274,6 +283,7 @@ defmodule Pleroma.Web.MastodonAPI.NotificationViewTest do expected = %{ id: to_string(notification.id), + group_key: "ungrouped-#{to_string(notification.id)}", pleroma: %{is_seen: false, is_muted: false}, type: "pleroma:report", account: AccountView.render("show.json", %{user: reporting_user, for: moderator_user}), @@ -300,6 +310,7 @@ defmodule Pleroma.Web.MastodonAPI.NotificationViewTest do expected = %{ id: to_string(notification.id), + group_key: "ungrouped-#{to_string(notification.id)}", pleroma: %{is_seen: false, is_muted: false}, type: "update", account: AccountView.render("show.json", %{user: user, for: repeat_user}), @@ -322,6 +333,7 @@ defmodule Pleroma.Web.MastodonAPI.NotificationViewTest do expected = %{ id: to_string(notification.id), + group_key: "ungrouped-#{to_string(notification.id)}", pleroma: %{is_seen: true, is_muted: true}, type: "favourite", account: AccountView.render("show.json", %{user: another_user, for: user}), @@ -345,6 +357,7 @@ defmodule Pleroma.Web.MastodonAPI.NotificationViewTest do expected = %{ id: to_string(notification.id), + group_key: "ungrouped-#{to_string(notification.id)}", pleroma: %{is_seen: false, is_muted: false}, type: "status", account: From e10db52e0a1c9cc24803a406998a9cfe75b7f9f2 Mon Sep 17 00:00:00 2001 From: Mint Date: Fri, 13 Sep 2024 02:58:59 +0300 Subject: [PATCH 084/117] Add dependencies for Swoosh's Mua mail adapter --- changelog.d/swoosh-mua.add | 1 + mix.exs | 4 +++- mix.lock | 4 +++- 3 files changed, 7 insertions(+), 2 deletions(-) create mode 100644 changelog.d/swoosh-mua.add diff --git a/changelog.d/swoosh-mua.add b/changelog.d/swoosh-mua.add new file mode 100644 index 000000000..d4c4bbd08 --- /dev/null +++ b/changelog.d/swoosh-mua.add @@ -0,0 +1 @@ +Added dependencies for Swoosh's Mua mail adapter diff --git a/mix.exs b/mix.exs index 0d49a6b45..ceae5c26d 100644 --- a/mix.exs +++ b/mix.exs @@ -153,7 +153,7 @@ defmodule Pleroma.Mixfile do {:calendar, "~> 1.0"}, {:cachex, "~> 3.2"}, {:tesla, "~> 1.11"}, - {:castore, "~> 0.1"}, + {:castore, "~> 1.0"}, {:cowlib, "~> 2.9", override: true}, {:gun, "~> 2.0.0-rc.1", override: true}, {:finch, "~> 0.15"}, @@ -169,6 +169,8 @@ defmodule Pleroma.Mixfile do {:swoosh, "~> 1.16.9"}, {:phoenix_swoosh, "~> 1.1"}, {:gen_smtp, "~> 0.13"}, + {:mua, "~> 0.2.0"}, + {:mail, "~> 0.3.0"}, {:ex_syslogger, "~> 1.4"}, {:floki, "~> 0.35"}, {:timex, "~> 3.6"}, diff --git a/mix.lock b/mix.lock index 01f2eef98..2cf44862b 100644 --- a/mix.lock +++ b/mix.lock @@ -11,7 +11,7 @@ "cachex": {:hex, :cachex, "3.6.0", "14a1bfbeee060dd9bec25a5b6f4e4691e3670ebda28c8ba2884b12fe30b36bf8", [:mix], [{:eternal, "~> 1.2", [hex: :eternal, repo: "hexpm", optional: false]}, {:jumper, "~> 1.0", [hex: :jumper, repo: "hexpm", optional: false]}, {:sleeplocks, "~> 1.1", [hex: :sleeplocks, repo: "hexpm", optional: false]}, {:unsafe, "~> 1.0", [hex: :unsafe, repo: "hexpm", optional: false]}], "hexpm", "ebf24e373883bc8e0c8d894a63bbe102ae13d918f790121f5cfe6e485cc8e2e2"}, "calendar": {:hex, :calendar, "1.0.0", "f52073a708528482ec33d0a171954ca610fe2bd28f1e871f247dc7f1565fa807", [:mix], [{:tzdata, "~> 0.5.20 or ~> 0.1.201603 or ~> 1.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm", "990e9581920c82912a5ee50e62ff5ef96da6b15949a2ee4734f935fdef0f0a6f"}, "captcha": {:git, "https://git.pleroma.social/pleroma/elixir-libraries/elixir-captcha.git", "6630c42aaaab124e697b4e513190c89d8b64e410", [ref: "6630c42aaaab124e697b4e513190c89d8b64e410"]}, - "castore": {:hex, :castore, "0.1.22", "4127549e411bedd012ca3a308dede574f43819fe9394254ca55ab4895abfa1a2", [:mix], [], "hexpm", "c17576df47eb5aa1ee40cc4134316a99f5cad3e215d5c77b8dd3cfef12a22cac"}, + "castore": {:hex, :castore, "1.0.8", "dedcf20ea746694647f883590b82d9e96014057aff1d44d03ec90f36a5c0dc6e", [:mix], [], "hexpm", "0b2b66d2ee742cb1d9cb8c8be3b43c3a70ee8651f37b75a8b982e036752983f1"}, "cc_precompiler": {:hex, :cc_precompiler, "0.1.9", "e8d3364f310da6ce6463c3dd20cf90ae7bbecbf6c5203b98bf9b48035592649b", [:mix], [{:elixir_make, "~> 0.7", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm", "9dcab3d0f3038621f1601f13539e7a9ee99843862e66ad62827b0c42b2f58a54"}, "certifi": {:hex, :certifi, "2.12.0", "2d1cca2ec95f59643862af91f001478c9863c2ac9cb6e2f89780bfd8de987329", [:rebar3], [], "hexpm", "ee68d85df22e554040cdb4be100f33873ac6051387baf6a8f6ce82272340ff1c"}, "combine": {:hex, :combine, "0.10.0", "eff8224eeb56498a2af13011d142c5e7997a80c8f5b97c499f84c841032e429f", [:mix], [], "hexpm", "1b1dbc1790073076580d0d1d64e42eae2366583e7aecd455d1215b0d16f2451b"}, @@ -72,6 +72,7 @@ "jumper": {:hex, :jumper, "1.0.2", "68cdcd84472a00ac596b4e6459a41b3062d4427cbd4f1e8c8793c5b54f1406a7", [:mix], [], "hexpm", "9b7782409021e01ab3c08270e26f36eb62976a38c1aa64b2eaf6348422f165e1"}, "linkify": {:hex, :linkify, "0.5.3", "5f8143d8f61f5ff08d3aeeff47ef6509492b4948d8f08007fbf66e4d2246a7f2", [:mix], [], "hexpm", "3ef35a1377d47c25506e07c1c005ea9d38d700699d92ee92825f024434258177"}, "logger_backends": {:hex, :logger_backends, "1.0.0", "09c4fad6202e08cb0fbd37f328282f16539aca380f512523ce9472b28edc6bdf", [:mix], [], "hexpm", "1faceb3e7ec3ef66a8f5746c5afd020e63996df6fd4eb8cdb789e5665ae6c9ce"}, + "mail": {:hex, :mail, "0.3.1", "cb0a14e4ed8904e4e5a08214e686ccf6f9099346885db17d8c309381f865cc5c", [:mix], [], "hexpm", "1db701e89865c1d5fa296b2b57b1cd587587cca8d8a1a22892b35ef5a8e352a6"}, "majic": {:hex, :majic, "1.0.0", "37e50648db5f5c2ff0c9fb46454d034d11596c03683807b9fb3850676ffdaab3", [:make, :mix], [{:elixir_make, "~> 0.6.1", [hex: :elixir_make, repo: "hexpm", optional: false]}, {:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 0.2", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "7905858f76650d49695f14ea55cd9aaaee0c6654fa391671d4cf305c275a0a9e"}, "makeup": {:hex, :makeup, "1.0.5", "d5a830bc42c9800ce07dd97fa94669dfb93d3bf5fcf6ea7a0c67b2e0e4a7f26c", [:mix], [{:nimble_parsec, "~> 0.5 or ~> 1.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "cfa158c02d3f5c0c665d0af11512fed3fba0144cf1aadee0f2ce17747fba2ca9"}, "makeup_elixir": {:hex, :makeup_elixir, "0.14.1", "4f0e96847c63c17841d42c08107405a005a2680eb9c7ccadfd757bd31dabccfb", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "f2438b1a80eaec9ede832b5c41cd4f373b38fd7aa33e3b22d9db79e640cbde11"}, @@ -85,6 +86,7 @@ "mock": {:hex, :mock, "0.3.8", "7046a306b71db2488ef54395eeb74df0a7f335a7caca4a3d3875d1fc81c884dd", [:mix], [{:meck, "~> 0.9.2", [hex: :meck, repo: "hexpm", optional: false]}], "hexpm", "7fa82364c97617d79bb7d15571193fc0c4fe5afd0c932cef09426b3ee6fe2022"}, "mogrify": {:hex, :mogrify, "0.9.3", "238c782f00271dace01369ad35ae2e9dd020feee3443b9299ea5ea6bed559841", [:mix], [], "hexpm", "0189b1e1de27455f2b9ae8cf88239cefd23d38de9276eb5add7159aea51731e6"}, "mox": {:hex, :mox, "1.1.0", "0f5e399649ce9ab7602f72e718305c0f9cdc351190f72844599545e4996af73c", [:mix], [], "hexpm", "d44474c50be02d5b72131070281a5d3895c0e7a95c780e90bc0cfe712f633a13"}, + "mua": {:hex, :mua, "0.2.3", "46b29b7b2bb14105c0b7be9526f7c452df17a7841b30b69871c024a822ff551c", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}], "hexpm", "7fe861a87fcc06a980d3941bbcb2634e5f0f30fd6ad15ef6c0423ff9dc7e46de"}, "multipart": {:hex, :multipart, "0.4.0", "634880a2148d4555d050963373d0e3bbb44a55b2badd87fa8623166172e9cda0", [:mix], [{:mime, "~> 1.2 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}], "hexpm", "3c5604bc2fb17b3137e5d2abdf5dacc2647e60c5cc6634b102cf1aef75a06f0a"}, "nimble_options": {:hex, :nimble_options, "1.1.1", "e3a492d54d85fc3fd7c5baf411d9d2852922f66e69476317787a7b2bb000a61b", [:mix], [], "hexpm", "821b2470ca9442c4b6984882fe9bb0389371b8ddec4d45a9504f00a66f650b44"}, "nimble_parsec": {:hex, :nimble_parsec, "0.6.0", "32111b3bf39137144abd7ba1cce0914533b2d16ef35e8abc5ec8be6122944263", [:mix], [], "hexpm", "27eac315a94909d4dc68bc07a4a83e06c8379237c5ea528a9acff4ca1c873c52"}, From 1a120d013019fc15b3f440f7db71d3eb328bc798 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Sat, 14 Sep 2024 20:17:08 +0200 Subject: [PATCH 085/117] Federate avatar/header descriptions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- changelog.d/profile-image-descriptions.skip | 0 lib/pleroma/user.ex | 5 +++ lib/pleroma/web/activity_pub/activity_pub.ex | 9 ++++- .../web/activity_pub/views/user_view.ex | 36 +++++++++++++++---- .../web/mastodon_api/views/account_view.ex | 8 ++--- .../web/activity_pub/activity_pub_test.exs | 35 ++++++++++++++++-- .../web/activity_pub/views/user_view_test.exs | 17 +++++++++ 7 files changed, 94 insertions(+), 16 deletions(-) create mode 100644 changelog.d/profile-image-descriptions.skip diff --git a/changelog.d/profile-image-descriptions.skip b/changelog.d/profile-image-descriptions.skip new file mode 100644 index 000000000..e69de29bb diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 517009253..7a36ece77 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -419,6 +419,11 @@ defmodule Pleroma.User do end end + def image_description(image, default \\ "") + + def image_description(%{"name" => name}, _default), do: name + def image_description(_, default), do: default + # Should probably be renamed or removed @spec ap_id(User.t()) :: String.t() def ap_id(%User{nickname: nickname}), do: "#{Endpoint.url()}/users/#{nickname}" diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index a2a94a0ff..df8795fe4 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -1542,16 +1542,23 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp get_actor_url(_url), do: nil - defp normalize_image(%{"url" => url}) do + defp normalize_image(%{"url" => url} = data) do %{ "type" => "Image", "url" => [%{"href" => url}] } + |> maybe_put_description(data) end defp normalize_image(urls) when is_list(urls), do: urls |> List.first() |> normalize_image() defp normalize_image(_), do: nil + defp maybe_put_description(map, %{"name" => description}) when is_binary(description) do + Map.put(map, "name", description) + end + + defp maybe_put_description(map, _), do: map + defp object_to_user_data(data, additional) do fields = data diff --git a/lib/pleroma/web/activity_pub/views/user_view.ex b/lib/pleroma/web/activity_pub/views/user_view.ex index 937e4fd67..cd485ed64 100644 --- a/lib/pleroma/web/activity_pub/views/user_view.ex +++ b/lib/pleroma/web/activity_pub/views/user_view.ex @@ -129,8 +129,22 @@ defmodule Pleroma.Web.ActivityPub.UserView do "vcard:bday" => birthday, "webfinger" => "acct:#{User.full_nickname(user)}" } - |> Map.merge(maybe_make_image(&User.avatar_url/2, "icon", user)) - |> Map.merge(maybe_make_image(&User.banner_url/2, "image", user)) + |> Map.merge( + maybe_make_image( + &User.avatar_url/2, + User.image_description(user.avatar, nil), + "icon", + user + ) + ) + |> Map.merge( + maybe_make_image( + &User.banner_url/2, + User.image_description(user.banner, nil), + "image", + user + ) + ) |> Map.merge(Utils.make_json_ld_header()) end @@ -305,16 +319,24 @@ defmodule Pleroma.Web.ActivityPub.UserView do end end - defp maybe_make_image(func, key, user) do + defp maybe_make_image(func, description, key, user) do if image = func.(user, no_default: true) do %{ - key => %{ - "type" => "Image", - "url" => image - } + key => + %{ + "type" => "Image", + "url" => image + } + |> maybe_put_description(description) } else %{} end end + + defp maybe_put_description(map, description) when is_binary(description) do + Map.put(map, "name", description) + end + + defp maybe_put_description(map, _description), do: map end diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index 7de6745d4..f6727d29d 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -219,10 +219,10 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do avatar = User.avatar_url(user) |> MediaProxy.url() avatar_static = User.avatar_url(user) |> MediaProxy.preview_url(static: true) - avatar_description = image_description(user.avatar) + avatar_description = User.image_description(user.avatar) header = User.banner_url(user) |> MediaProxy.url() header_static = User.banner_url(user) |> MediaProxy.preview_url(static: true) - header_description = image_description(user.banner) + header_description = User.image_description(user.banner) following_count = if !user.hide_follows_count or !user.hide_follows or self, @@ -349,10 +349,6 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do defp username_from_nickname(_), do: nil - defp image_description(%{"name" => name}), do: name - - defp image_description(_), do: "" - defp maybe_put_follow_requests_count( data, %User{id: user_id} = user, diff --git a/test/pleroma/web/activity_pub/activity_pub_test.exs b/test/pleroma/web/activity_pub/activity_pub_test.exs index b4f6fb68a..72222ae88 100644 --- a/test/pleroma/web/activity_pub/activity_pub_test.exs +++ b/test/pleroma/web/activity_pub/activity_pub_test.exs @@ -232,12 +232,14 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do assert user.avatar == %{ "type" => "Image", - "url" => [%{"href" => "https://jk.nipponalba.scot/images/profile.jpg"}] + "url" => [%{"href" => "https://jk.nipponalba.scot/images/profile.jpg"}], + "name" => "profile picture" } assert user.banner == %{ "type" => "Image", - "url" => [%{"href" => "https://jk.nipponalba.scot/images/profile.jpg"}] + "url" => [%{"href" => "https://jk.nipponalba.scot/images/profile.jpg"}], + "name" => "profile picture" } end @@ -432,6 +434,35 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do assert user.birthday == ~D[2001-02-12] end + + test "fetches avatar description" do + user_id = "https://example.com/users/marcin" + + user_data = + "test/fixtures/users_mock/user.json" + |> File.read!() + |> String.replace("{{nickname}}", "marcin") + |> Jason.decode!() + |> Map.delete("featured") + |> Map.update("icon", %{}, fn image -> Map.put(image, "name", "image description") end) + |> Jason.encode!() + + Tesla.Mock.mock(fn + %{ + method: :get, + url: ^user_id + } -> + %Tesla.Env{ + status: 200, + body: user_data, + headers: [{"content-type", "application/activity+json"}] + } + end) + + {:ok, user} = ActivityPub.make_user_from_ap_id(user_id) + + assert user.avatar["name"] == "image description" + end end test "it fetches the appropriate tag-restricted posts" do diff --git a/test/pleroma/web/activity_pub/views/user_view_test.exs b/test/pleroma/web/activity_pub/views/user_view_test.exs index 651e535ac..a32e72829 100644 --- a/test/pleroma/web/activity_pub/views/user_view_test.exs +++ b/test/pleroma/web/activity_pub/views/user_view_test.exs @@ -68,6 +68,23 @@ defmodule Pleroma.Web.ActivityPub.UserViewTest do result = UserView.render("user.json", %{user: user}) assert result["icon"]["url"] == "https://someurl" assert result["image"]["url"] == "https://somebanner" + + refute result["icon"]["name"] + refute result["image"]["name"] + end + + test "Avatar has a description if the user set one" do + user = + insert(:user, + avatar: %{ + "url" => [%{"href" => "https://someurl"}], + "name" => "a drawing of pleroma-tan using pleroma groups" + } + ) + + result = UserView.render("user.json", %{user: user}) + + assert result["icon"]["name"] == "a drawing of pleroma-tan using pleroma groups" end test "renders an invisible user with the invisible property set to true" do From 5539fea3bb0d272b4cefc2b72755cb3cd285cc67 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 14 Sep 2024 20:03:26 -0400 Subject: [PATCH 086/117] LDAP: permit overriding the CA root --- changelog.d/ldap-ca.add | 1 + config/config.exs | 4 +++- docs/configuration/cheatsheet.md | 1 + lib/pleroma/web/auth/ldap_authenticator.ex | 17 ++++++++++++++++- mix.exs | 1 - 5 files changed, 21 insertions(+), 3 deletions(-) create mode 100644 changelog.d/ldap-ca.add diff --git a/changelog.d/ldap-ca.add b/changelog.d/ldap-ca.add new file mode 100644 index 000000000..32ecbb5c0 --- /dev/null +++ b/changelog.d/ldap-ca.add @@ -0,0 +1 @@ +LDAP configuration now permits overriding the CA root certificate file for TLS validation. diff --git a/config/config.exs b/config/config.exs index 80a3b8d57..237928503 100644 --- a/config/config.exs +++ b/config/config.exs @@ -619,7 +619,9 @@ config :pleroma, :ldap, tls: System.get_env("LDAP_TLS") == "true", tlsopts: [], base: System.get_env("LDAP_BASE") || "dc=example,dc=com", - uid: System.get_env("LDAP_UID") || "cn" + uid: System.get_env("LDAP_UID") || "cn", + # defaults to CAStore's Mozilla roots + cacertfile: nil oauth_consumer_strategies = System.get_env("OAUTH_CONSUMER_STRATEGIES") diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index 0b4e53b6f..4cbde696e 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -974,6 +974,7 @@ Pleroma account will be created with the same name as the LDAP user name. * `tlsopts`: additional TLS options * `base`: LDAP base, e.g. "dc=example,dc=com" * `uid`: LDAP attribute name to authenticate the user, e.g. when "cn", the filter will be "cn=username,base" +* `cacertfile`: Path to alternate CA root certificates file Note, if your LDAP server is an Active Directory server the correct value is commonly `uid: "cn"`, but if you use an OpenLDAP server the value may be `uid: "uid"`. diff --git a/lib/pleroma/web/auth/ldap_authenticator.ex b/lib/pleroma/web/auth/ldap_authenticator.ex index d31f34747..7f2cd3d69 100644 --- a/lib/pleroma/web/auth/ldap_authenticator.ex +++ b/lib/pleroma/web/auth/ldap_authenticator.ex @@ -42,11 +42,14 @@ defmodule Pleroma.Web.Auth.LDAPAuthenticator do ssl = Keyword.get(ldap, :ssl, false) sslopts = Keyword.get(ldap, :sslopts, []) tlsopts = Keyword.get(ldap, :tlsopts, []) + cacertfile = Keyword.get(ldap, :cacertfile) || CAStore.file_path() options = [{:port, port}, {:ssl, ssl}, {:timeout, @connection_timeout}] ++ if sslopts != [], do: [{:sslopts, sslopts}], else: [] + cacerts = decode_certfile(cacertfile) + case :eldap.open([to_charlist(host)], options) do {:ok, connection} -> try do @@ -58,7 +61,7 @@ defmodule Pleroma.Web.Auth.LDAPAuthenticator do Keyword.merge( [ verify: :verify_peer, - cacerts: :certifi.cacerts(), + cacerts: cacerts, customize_hostname_check: [ fqdn_fun: fn _ -> to_charlist(host) end ] @@ -147,4 +150,16 @@ defmodule Pleroma.Web.Auth.LDAPAuthenticator do error -> error end end + + defp decode_certfile(file) do + with {:ok, data} <- File.read(file) do + data + |> :public_key.pem_decode() + |> Enum.map(fn {_, b, _} -> b end) + else + _ -> + Logger.error("Unable to read certfile: #{file}") + [] + end + end end diff --git a/mix.exs b/mix.exs index 9a261547f..0d49a6b45 100644 --- a/mix.exs +++ b/mix.exs @@ -204,7 +204,6 @@ defmodule Pleroma.Mixfile do {:oban_live_dashboard, "~> 0.1.1"}, {:multipart, "~> 0.4.0", optional: true}, {:argon2_elixir, "~> 4.0"}, - {:certifi, "~> 2.12"}, ## dev & test {:phoenix_live_reload, "~> 1.3.3", only: :dev}, From af3bf8a4628c0b2981d69f624e3be298adc7dfe6 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sun, 15 Sep 2024 13:56:16 -0400 Subject: [PATCH 087/117] Support implicit TLS connections Update docs to clarify that the :ssl option is also for modern TLS, but the :tls option is only for STARTTLS These options may benefit from being renamed but they match upstream terminology. --- changelog.d/ldaps.fix | 1 + docs/configuration/cheatsheet.md | 4 +- lib/pleroma/web/auth/ldap_authenticator.ex | 50 ++++++++++++---------- 3 files changed, 31 insertions(+), 24 deletions(-) create mode 100644 changelog.d/ldaps.fix diff --git a/changelog.d/ldaps.fix b/changelog.d/ldaps.fix new file mode 100644 index 000000000..a1dc901ab --- /dev/null +++ b/changelog.d/ldaps.fix @@ -0,0 +1 @@ +LDAPS connections (implicit TLS) are now supported. diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index 4cbde696e..6a535e054 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -968,9 +968,9 @@ Pleroma account will be created with the same name as the LDAP user name. * `enabled`: enables LDAP authentication * `host`: LDAP server hostname * `port`: LDAP port, e.g. 389 or 636 -* `ssl`: true to use SSL, usually implies the port 636 +* `ssl`: true to use implicit SSL/TLS, usually port 636 * `sslopts`: additional SSL options -* `tls`: true to start TLS, usually implies the port 389 +* `tls`: true to use explicit TLS (STARTTLS), usually port 389 * `tlsopts`: additional TLS options * `base`: LDAP base, e.g. "dc=example,dc=com" * `uid`: LDAP attribute name to authenticate the user, e.g. when "cn", the filter will be "cn=username,base" diff --git a/lib/pleroma/web/auth/ldap_authenticator.ex b/lib/pleroma/web/auth/ldap_authenticator.ex index 7f2cd3d69..18a4e81ee 100644 --- a/lib/pleroma/web/auth/ldap_authenticator.ex +++ b/lib/pleroma/web/auth/ldap_authenticator.ex @@ -40,34 +40,39 @@ defmodule Pleroma.Web.Auth.LDAPAuthenticator do host = Keyword.get(ldap, :host, "localhost") port = Keyword.get(ldap, :port, 389) ssl = Keyword.get(ldap, :ssl, false) - sslopts = Keyword.get(ldap, :sslopts, []) - tlsopts = Keyword.get(ldap, :tlsopts, []) + tls = Keyword.get(ldap, :tls, false) cacertfile = Keyword.get(ldap, :cacertfile) || CAStore.file_path() - options = - [{:port, port}, {:ssl, ssl}, {:timeout, @connection_timeout}] ++ - if sslopts != [], do: [{:sslopts, sslopts}], else: [] + default_secure_opts = [ + verify: :verify_peer, + cacerts: decode_certfile(cacertfile), + customize_hostname_check: [ + fqdn_fun: fn _ -> to_charlist(host) end + ] + ] - cacerts = decode_certfile(cacertfile) + sslopts = Keyword.merge(default_secure_opts, Keyword.get(ldap, :sslopts, [])) + tlsopts = Keyword.merge(default_secure_opts, Keyword.get(ldap, :tlsopts, [])) + + # :sslopts can only be included in :eldap.open/2 when {ssl: true} + # or the connection will fail + options = + if ssl do + [{:port, port}, {:ssl, ssl}, {:sslopts, sslopts}, {:timeout, @connection_timeout}] + else + [{:port, port}, {:ssl, ssl}, {:timeout, @connection_timeout}] + end case :eldap.open([to_charlist(host)], options) do {:ok, connection} -> - try do - if Keyword.get(ldap, :tls, false) do + cond do + ssl -> :application.ensure_all_started(:ssl) + tls -> case :eldap.start_tls( connection, - Keyword.merge( - [ - verify: :verify_peer, - cacerts: cacerts, - customize_hostname_check: [ - fqdn_fun: fn _ -> to_charlist(host) end - ] - ], - tlsopts - ), + tlsopts, @connection_timeout ) do :ok -> @@ -75,14 +80,15 @@ defmodule Pleroma.Web.Auth.LDAPAuthenticator do error -> Logger.error("Could not start TLS: #{inspect(error)}") + :eldap.close(connection) end - end - bind_user(connection, ldap, name, password) - after - :eldap.close(connection) + true -> + :ok end + bind_user(connection, ldap, name, password) + {:error, error} -> Logger.error("Could not open LDAP connection: #{inspect(error)}") {:error, {:ldap_connection_error, error}} From 91d1d7260b7084f59ae42e7c4b46c7fb963fda96 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sun, 15 Sep 2024 23:18:17 -0400 Subject: [PATCH 088/117] Retain the try do so an LDAP failure can fall back to local database. This fixes tests but the automatic fallback may not be well documented behavior. --- lib/pleroma/web/auth/ldap_authenticator.ex | 42 ++++++++++++---------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/lib/pleroma/web/auth/ldap_authenticator.ex b/lib/pleroma/web/auth/ldap_authenticator.ex index 18a4e81ee..ad5bc9863 100644 --- a/lib/pleroma/web/auth/ldap_authenticator.ex +++ b/lib/pleroma/web/auth/ldap_authenticator.ex @@ -65,30 +65,34 @@ defmodule Pleroma.Web.Auth.LDAPAuthenticator do case :eldap.open([to_charlist(host)], options) do {:ok, connection} -> - cond do - ssl -> - :application.ensure_all_started(:ssl) + try do + cond do + ssl -> + :application.ensure_all_started(:ssl) - tls -> - case :eldap.start_tls( - connection, - tlsopts, - @connection_timeout - ) do - :ok -> - :ok + tls -> + case :eldap.start_tls( + connection, + tlsopts, + @connection_timeout + ) do + :ok -> + :ok - error -> - Logger.error("Could not start TLS: #{inspect(error)}") - :eldap.close(connection) - end + error -> + Logger.error("Could not start TLS: #{inspect(error)}") + :eldap.close(connection) + end - true -> - :ok + true -> + :ok + end + + bind_user(connection, ldap, name, password) + after + :eldap.close(connection) end - bind_user(connection, ldap, name, password) - {:error, error} -> Logger.error("Could not open LDAP connection: #{inspect(error)}") {:error, {:ldap_connection_error, error}} From e59706c201bd71525c0a15008c3cb5dcdfb73289 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 16 Sep 2024 11:39:19 -0400 Subject: [PATCH 089/117] Reapply "Custom mix task to retry failed tests once in CI pipeline" This reverts commit b281ad06de2de331450a5e319e3ba497071d4197. --- .gitlab-ci.yml | 2 +- changelog.d/fix-test-failures.skip | 0 lib/mix/tasks/pleroma/test_runner.ex | 25 +++++++++++++++++++++++++ 3 files changed, 26 insertions(+), 1 deletion(-) delete mode 100644 changelog.d/fix-test-failures.skip create mode 100644 lib/mix/tasks/pleroma/test_runner.ex diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1e04dae76..76d1a4210 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -134,7 +134,7 @@ unit-testing-1.13.4-otp-25: script: &testing_script - mix ecto.create - mix ecto.migrate - - mix test --cover --preload-modules + - mix pleroma.test_runner --cover --preload-modules coverage: '/^Line total: ([^ ]*%)$/' artifacts: reports: diff --git a/changelog.d/fix-test-failures.skip b/changelog.d/fix-test-failures.skip deleted file mode 100644 index e69de29bb..000000000 diff --git a/lib/mix/tasks/pleroma/test_runner.ex b/lib/mix/tasks/pleroma/test_runner.ex new file mode 100644 index 000000000..69fefb001 --- /dev/null +++ b/lib/mix/tasks/pleroma/test_runner.ex @@ -0,0 +1,25 @@ +defmodule Mix.Tasks.Pleroma.TestRunner do + @shortdoc "Retries tests once if they fail" + + use Mix.Task + + def run(args \\ []) do + case System.cmd("mix", ["test"] ++ args, into: IO.stream(:stdio, :line)) do + {_, 0} -> + :ok + + _ -> + retry(args) + end + end + + def retry(args) do + case System.cmd("mix", ["test", "--failed"] ++ args, into: IO.stream(:stdio, :line)) do + {_, 0} -> + :ok + + _ -> + exit(1) + end + end +end From 9264b21907f5c6890694d6d611ade9b13433463a Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 16 Sep 2024 00:26:57 -0400 Subject: [PATCH 090/117] Pleroma.LDAP This adds a GenServer which will keep an LDAP connection open and auto reconnect on failure with a 5 second wait between retries. Another benefit is this prevents parsing the Root CAs for every login attempt as we only need to do it once per connection. --- lib/pleroma/application.ex | 1 + lib/pleroma/ldap.ex | 233 +++++++++++++++++++++ lib/pleroma/web/auth/ldap_authenticator.ex | 147 +------------ 3 files changed, 236 insertions(+), 145 deletions(-) create mode 100644 lib/pleroma/ldap.ex diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index cb15dc1e9..3f199c002 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -94,6 +94,7 @@ defmodule Pleroma.Application do children = [ Pleroma.PromEx, + Pleroma.LDAP, Pleroma.Repo, Config.TransferTask, Pleroma.Emoji, diff --git a/lib/pleroma/ldap.ex b/lib/pleroma/ldap.ex new file mode 100644 index 000000000..868932501 --- /dev/null +++ b/lib/pleroma/ldap.ex @@ -0,0 +1,233 @@ +defmodule Pleroma.LDAP do + use GenServer + + require Logger + + alias Pleroma.Config + alias Pleroma.User + + import Pleroma.Web.Auth.Helpers, only: [fetch_user: 1] + + @connection_timeout 10_000 + @search_timeout 10_000 + + def start_link(_) do + GenServer.start_link(__MODULE__, [], name: __MODULE__) + end + + @impl true + def init(state) do + case {Config.get(Pleroma.Web.Auth.Authenticator), Config.get([:ldap, :enabled])} do + {Pleroma.Web.Auth.LDAPAuthenticator, true} -> + {:ok, state, {:continue, :connect}} + + {Pleroma.Web.Auth.LDAPAuthenticator, false} -> + Logger.error( + "LDAP Authenticator enabled but :pleroma, :ldap is not enabled. Auth will not work." + ) + + {:ok, state} + + {_, true} -> + Logger.warning( + ":pleroma, :ldap is enabled but Pleroma.Web.Authenticator is not set to the LDAPAuthenticator. LDAP will not be used." + ) + + {:ok, state} + end + end + + @impl true + def handle_continue(:connect, _state), do: do_handle_connect() + + @impl true + def handle_info(:connect, _state), do: do_handle_connect() + + def handle_info({:bind_after_reconnect, name, password, from}, state) do + result = bind_user(state[:connection], name, password) + + GenServer.reply(from, result) + + {:noreply, state} + end + + defp do_handle_connect() do + state = + case connect() do + {:ok, connection} -> + :eldap.controlling_process(connection, self()) + [connection: connection] + + _ -> + Logger.error("Failed to connect to LDAP. Retrying in 5000ms") + Process.send_after(self(), :connect, 5_000) + [] + end + + {:noreply, state} + end + + @impl true + def handle_call({:bind_user, name, password}, from, state) do + case bind_user(state[:connection], name, password) do + :needs_reconnect -> + Process.send(self(), {:bind_after_reconnect, name, password, from}, []) + {:noreply, state, {:continue, :connect}} + + result -> + {:reply, result, state, :hibernate} + end + end + + @impl true + def terminate(_, state) do + :eldap.close(state[:connection]) + + :ok + end + + defp connect() do + ldap = Config.get(:ldap, []) + host = Keyword.get(ldap, :host, "localhost") + port = Keyword.get(ldap, :port, 389) + ssl = Keyword.get(ldap, :ssl, false) + tls = Keyword.get(ldap, :tls, false) + cacertfile = Keyword.get(ldap, :cacertfile) || CAStore.file_path() + + default_secure_opts = [ + verify: :verify_peer, + cacerts: decode_certfile(cacertfile), + customize_hostname_check: [ + fqdn_fun: fn _ -> to_charlist(host) end + ] + ] + + sslopts = Keyword.merge(default_secure_opts, Keyword.get(ldap, :sslopts, [])) + tlsopts = Keyword.merge(default_secure_opts, Keyword.get(ldap, :tlsopts, [])) + + default_options = [{:port, port}, {:ssl, ssl}, {:timeout, @connection_timeout}] + + # :sslopts can only be included in :eldap.open/2 when {ssl: true} + # or the connection will fail + options = + if ssl do + default_options ++ [{:sslopts, sslopts}] + else + default_options + end + + case :eldap.open([to_charlist(host)], options) do + {:ok, connection} -> + try do + cond do + ssl -> + :application.ensure_all_started(:ssl) + {:ok, connection} + + tls -> + case :eldap.start_tls( + connection, + tlsopts, + @connection_timeout + ) do + :ok -> + {:ok, connection} + + error -> + Logger.error("Could not start TLS: #{inspect(error)}") + :eldap.close(connection) + end + + true -> + {:ok, :connection} + end + after + :ok + end + + {:error, error} -> + Logger.error("Could not open LDAP connection: #{inspect(error)}") + {:error, {:ldap_connection_error, error}} + end + end + + defp bind_user(connection, name, password) do + uid = Config.get([:ldap, :uid], "cn") + base = Config.get([:ldap, :base]) + + case :eldap.simple_bind(connection, "#{uid}=#{name},#{base}", password) do + :ok -> + case fetch_user(name) do + %User{} = user -> + user + + _ -> + register_user(connection, base, uid, name) + end + + # eldap does not inform us of socket closure + # until it is used + {:error, {:gen_tcp_error, :closed}} -> + :eldap.close(connection) + :needs_reconnect + + error -> + Logger.error("Could not bind LDAP user #{name}: #{inspect(error)}") + {:error, {:ldap_bind_error, error}} + end + end + + defp register_user(connection, base, uid, name) do + case :eldap.search(connection, [ + {:base, to_charlist(base)}, + {:filter, :eldap.equalityMatch(to_charlist(uid), to_charlist(name))}, + {:scope, :eldap.wholeSubtree()}, + {:timeout, @search_timeout} + ]) do + # The :eldap_search_result record structure changed in OTP 24.3 and added a controls field + # https://github.com/erlang/otp/pull/5538 + {:ok, {:eldap_search_result, [{:eldap_entry, _object, attributes}], _referrals}} -> + try_register(name, attributes) + + {:ok, {:eldap_search_result, [{:eldap_entry, _object, attributes}], _referrals, _controls}} -> + try_register(name, attributes) + + error -> + Logger.error("Couldn't register user because LDAP search failed: #{inspect(error)}") + {:error, {:ldap_search_error, error}} + end + end + + defp try_register(name, attributes) do + params = %{ + name: name, + nickname: name, + password: nil + } + + params = + case List.keyfind(attributes, ~c"mail", 0) do + {_, [mail]} -> Map.put_new(params, :email, :erlang.list_to_binary(mail)) + _ -> params + end + + changeset = User.register_changeset_ldap(%User{}, params) + + case User.register(changeset) do + {:ok, user} -> user + error -> error + end + end + + defp decode_certfile(file) do + with {:ok, data} <- File.read(file) do + data + |> :public_key.pem_decode() + |> Enum.map(fn {_, b, _} -> b end) + else + _ -> + Logger.error("Unable to read certfile: #{file}") + [] + end + end +end diff --git a/lib/pleroma/web/auth/ldap_authenticator.ex b/lib/pleroma/web/auth/ldap_authenticator.ex index ad5bc9863..c420c8bc3 100644 --- a/lib/pleroma/web/auth/ldap_authenticator.ex +++ b/lib/pleroma/web/auth/ldap_authenticator.ex @@ -5,16 +5,11 @@ defmodule Pleroma.Web.Auth.LDAPAuthenticator do alias Pleroma.User - require Logger - - import Pleroma.Web.Auth.Helpers, only: [fetch_credentials: 1, fetch_user: 1] + import Pleroma.Web.Auth.Helpers, only: [fetch_credentials: 1] @behaviour Pleroma.Web.Auth.Authenticator @base Pleroma.Web.Auth.PleromaAuthenticator - @connection_timeout 10_000 - @search_timeout 10_000 - defdelegate get_registration(conn), to: @base defdelegate create_from_registration(conn, registration), to: @base defdelegate handle_error(conn, error), to: @base @@ -24,7 +19,7 @@ defmodule Pleroma.Web.Auth.LDAPAuthenticator do def get_user(%Plug.Conn{} = conn) do with {:ldap, true} <- {:ldap, Pleroma.Config.get([:ldap, :enabled])}, {:ok, {name, password}} <- fetch_credentials(conn), - %User{} = user <- ldap_user(name, password) do + %User{} = user <- GenServer.call(Pleroma.LDAP, {:bind_user, name, password}) do {:ok, user} else {:ldap, _} -> @@ -34,142 +29,4 @@ defmodule Pleroma.Web.Auth.LDAPAuthenticator do error end end - - defp ldap_user(name, password) do - ldap = Pleroma.Config.get(:ldap, []) - host = Keyword.get(ldap, :host, "localhost") - port = Keyword.get(ldap, :port, 389) - ssl = Keyword.get(ldap, :ssl, false) - tls = Keyword.get(ldap, :tls, false) - cacertfile = Keyword.get(ldap, :cacertfile) || CAStore.file_path() - - default_secure_opts = [ - verify: :verify_peer, - cacerts: decode_certfile(cacertfile), - customize_hostname_check: [ - fqdn_fun: fn _ -> to_charlist(host) end - ] - ] - - sslopts = Keyword.merge(default_secure_opts, Keyword.get(ldap, :sslopts, [])) - tlsopts = Keyword.merge(default_secure_opts, Keyword.get(ldap, :tlsopts, [])) - - # :sslopts can only be included in :eldap.open/2 when {ssl: true} - # or the connection will fail - options = - if ssl do - [{:port, port}, {:ssl, ssl}, {:sslopts, sslopts}, {:timeout, @connection_timeout}] - else - [{:port, port}, {:ssl, ssl}, {:timeout, @connection_timeout}] - end - - case :eldap.open([to_charlist(host)], options) do - {:ok, connection} -> - try do - cond do - ssl -> - :application.ensure_all_started(:ssl) - - tls -> - case :eldap.start_tls( - connection, - tlsopts, - @connection_timeout - ) do - :ok -> - :ok - - error -> - Logger.error("Could not start TLS: #{inspect(error)}") - :eldap.close(connection) - end - - true -> - :ok - end - - bind_user(connection, ldap, name, password) - after - :eldap.close(connection) - end - - {:error, error} -> - Logger.error("Could not open LDAP connection: #{inspect(error)}") - {:error, {:ldap_connection_error, error}} - end - end - - defp bind_user(connection, ldap, name, password) do - uid = Keyword.get(ldap, :uid, "cn") - base = Keyword.get(ldap, :base) - - case :eldap.simple_bind(connection, "#{uid}=#{name},#{base}", password) do - :ok -> - case fetch_user(name) do - %User{} = user -> - user - - _ -> - register_user(connection, base, uid, name) - end - - error -> - Logger.error("Could not bind LDAP user #{name}: #{inspect(error)}") - {:error, {:ldap_bind_error, error}} - end - end - - defp register_user(connection, base, uid, name) do - case :eldap.search(connection, [ - {:base, to_charlist(base)}, - {:filter, :eldap.equalityMatch(to_charlist(uid), to_charlist(name))}, - {:scope, :eldap.wholeSubtree()}, - {:timeout, @search_timeout} - ]) do - # The :eldap_search_result record structure changed in OTP 24.3 and added a controls field - # https://github.com/erlang/otp/pull/5538 - {:ok, {:eldap_search_result, [{:eldap_entry, _object, attributes}], _referrals}} -> - try_register(name, attributes) - - {:ok, {:eldap_search_result, [{:eldap_entry, _object, attributes}], _referrals, _controls}} -> - try_register(name, attributes) - - error -> - Logger.error("Couldn't register user because LDAP search failed: #{inspect(error)}") - {:error, {:ldap_search_error, error}} - end - end - - defp try_register(name, attributes) do - params = %{ - name: name, - nickname: name, - password: nil - } - - params = - case List.keyfind(attributes, ~c"mail", 0) do - {_, [mail]} -> Map.put_new(params, :email, :erlang.list_to_binary(mail)) - _ -> params - end - - changeset = User.register_changeset_ldap(%User{}, params) - - case User.register(changeset) do - {:ok, user} -> user - error -> error - end - end - - defp decode_certfile(file) do - with {:ok, data} <- File.read(file) do - data - |> :public_key.pem_decode() - |> Enum.map(fn {_, b, _} -> b end) - else - _ -> - Logger.error("Unable to read certfile: #{file}") - [] - end - end end From ead287d623e83b8d9ffaa327b9edf96e046bfacd Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 16 Sep 2024 13:14:19 -0400 Subject: [PATCH 091/117] Credo --- lib/pleroma/ldap.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/ldap.ex b/lib/pleroma/ldap.ex index 868932501..11544f0d9 100644 --- a/lib/pleroma/ldap.ex +++ b/lib/pleroma/ldap.ex @@ -51,7 +51,7 @@ defmodule Pleroma.LDAP do {:noreply, state} end - defp do_handle_connect() do + defp do_handle_connect do state = case connect() do {:ok, connection} -> @@ -86,7 +86,7 @@ defmodule Pleroma.LDAP do :ok end - defp connect() do + defp connect do ldap = Config.get(:ldap, []) host = Keyword.get(ldap, :host, "localhost") port = Keyword.get(ldap, :port, 389) From 7c04098dde0681f7ad299782bc09eaa9bc3a6bad Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 16 Sep 2024 16:15:53 -0400 Subject: [PATCH 092/117] Catchall for when LDAP is not enabled --- lib/pleroma/ldap.ex | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/pleroma/ldap.ex b/lib/pleroma/ldap.ex index 11544f0d9..ac819613e 100644 --- a/lib/pleroma/ldap.ex +++ b/lib/pleroma/ldap.ex @@ -34,6 +34,9 @@ defmodule Pleroma.LDAP do ) {:ok, state} + + _ -> + {:ok, state} end end From 44b836c94c1059551fbc7564770001311d2d1e6a Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 16 Sep 2024 16:24:27 -0400 Subject: [PATCH 093/117] Fix tests We do not need to mock and verify connections are closed as the new Pleroma.LDAP GenServer will handle managing the connection lifetime --- .../web/o_auth/ldap_authorization_test.exs | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/test/pleroma/web/o_auth/ldap_authorization_test.exs b/test/pleroma/web/o_auth/ldap_authorization_test.exs index 07ce2eed8..35b947fd0 100644 --- a/test/pleroma/web/o_auth/ldap_authorization_test.exs +++ b/test/pleroma/web/o_auth/ldap_authorization_test.exs @@ -28,11 +28,7 @@ defmodule Pleroma.Web.OAuth.LDAPAuthorizationTest do {:eldap, [], [ open: fn [^host], [{:port, ^port}, {:ssl, false} | _] -> {:ok, self()} end, - simple_bind: fn _connection, _dn, ^password -> :ok end, - close: fn _connection -> - send(self(), :close_connection) - :ok - end + simple_bind: fn _connection, _dn, ^password -> :ok end ]} ] do conn = @@ -50,7 +46,6 @@ defmodule Pleroma.Web.OAuth.LDAPAuthorizationTest do token = Repo.get_by(Token, token: token) assert token.user_id == user.id - assert_received :close_connection end end @@ -72,10 +67,6 @@ defmodule Pleroma.Web.OAuth.LDAPAuthorizationTest do wholeSubtree: fn -> :ok end, search: fn _connection, _options -> {:ok, {:eldap_search_result, [{:eldap_entry, ~c"", []}], []}} - end, - close: fn _connection -> - send(self(), :close_connection) - :ok end ]} ] do @@ -94,7 +85,6 @@ defmodule Pleroma.Web.OAuth.LDAPAuthorizationTest do token = Repo.get_by(Token, token: token) |> Repo.preload(:user) assert token.user.nickname == user.nickname - assert_received :close_connection end end @@ -111,11 +101,7 @@ defmodule Pleroma.Web.OAuth.LDAPAuthorizationTest do {:eldap, [], [ open: fn [^host], [{:port, ^port}, {:ssl, false} | _] -> {:ok, self()} end, - simple_bind: fn _connection, _dn, ^password -> {:error, :invalidCredentials} end, - close: fn _connection -> - send(self(), :close_connection) - :ok - end + simple_bind: fn _connection, _dn, ^password -> {:error, :invalidCredentials} end ]} ] do conn = @@ -129,7 +115,6 @@ defmodule Pleroma.Web.OAuth.LDAPAuthorizationTest do }) assert %{"error" => "Invalid credentials"} = json_response(conn, 400) - assert_received :close_connection end end end From d82abf925ddbe8b98ba8191713115db50c38a0c0 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 16 Sep 2024 16:25:44 -0400 Subject: [PATCH 094/117] Ensure :cacertfile is configurable in ConfigDB --- config/description.exs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/config/description.exs b/config/description.exs index 15faecb38..ade47b7e0 100644 --- a/config/description.exs +++ b/config/description.exs @@ -2297,6 +2297,12 @@ config :pleroma, :config_description, [ description: "LDAP attribute name to authenticate the user, e.g. when \"cn\", the filter will be \"cn=username,base\"", suggestions: ["cn"] + }, + %{ + key: :cacertfile, + label: "CACertfile", + type: :string, + description: "Path to CA certificate file" } ] }, From 65a7b387c35b4913b6109692a84bae80af8b9a96 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 16 Sep 2024 16:28:37 -0400 Subject: [PATCH 095/117] Require a reboot if LDAP configuration changes --- lib/pleroma/config/transfer_task.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/config/transfer_task.ex b/lib/pleroma/config/transfer_task.ex index ffc95f144..140dd7711 100644 --- a/lib/pleroma/config/transfer_task.ex +++ b/lib/pleroma/config/transfer_task.ex @@ -22,7 +22,8 @@ defmodule Pleroma.Config.TransferTask do {:pleroma, :markup}, {:pleroma, :streamer}, {:pleroma, :pools}, - {:pleroma, :connections_pool} + {:pleroma, :connections_pool}, + {:pleroma, :ldap} ] defp reboot_time_subkeys, From 123093a1868b25f101dfc4b02895c22a0daf5733 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 17 Sep 2024 13:07:26 -0400 Subject: [PATCH 096/117] Ensure :ssl is started before we attempt to make the LDAP connection --- lib/pleroma/ldap.ex | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/ldap.ex b/lib/pleroma/ldap.ex index ac819613e..042a4daa2 100644 --- a/lib/pleroma/ldap.ex +++ b/lib/pleroma/ldap.ex @@ -97,6 +97,8 @@ defmodule Pleroma.LDAP do tls = Keyword.get(ldap, :tls, false) cacertfile = Keyword.get(ldap, :cacertfile) || CAStore.file_path() + if ssl, do: Application.ensure_all_started(:ssl) + default_secure_opts = [ verify: :verify_peer, cacerts: decode_certfile(cacertfile), @@ -123,10 +125,6 @@ defmodule Pleroma.LDAP do {:ok, connection} -> try do cond do - ssl -> - :application.ensure_all_started(:ssl) - {:ok, connection} - tls -> case :eldap.start_tls( connection, From d0ee899ab94788e37e6ac3c43342017d1b27903a Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 17 Sep 2024 13:14:09 -0400 Subject: [PATCH 097/117] Only close connection if it is not nil --- lib/pleroma/ldap.ex | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/ldap.ex b/lib/pleroma/ldap.ex index 042a4daa2..3df20fe09 100644 --- a/lib/pleroma/ldap.ex +++ b/lib/pleroma/ldap.ex @@ -84,7 +84,11 @@ defmodule Pleroma.LDAP do @impl true def terminate(_, state) do - :eldap.close(state[:connection]) + connection = Keyword.get(state, :connection) + + if not is_nil(connection) do + :eldap.close(connection) + end :ok end From 164ffbcab822eda4c28f912082b6a7a3ec64a7e5 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 17 Sep 2024 13:17:40 -0400 Subject: [PATCH 098/117] Fix return value when not doing STARTTLS --- lib/pleroma/ldap.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/ldap.ex b/lib/pleroma/ldap.ex index 3df20fe09..6be2188f4 100644 --- a/lib/pleroma/ldap.ex +++ b/lib/pleroma/ldap.ex @@ -144,7 +144,7 @@ defmodule Pleroma.LDAP do end true -> - {:ok, :connection} + {:ok, connection} end after :ok From a1972d57e30f41e5173d61c0d0936685738c560d Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 17 Sep 2024 13:19:54 -0400 Subject: [PATCH 099/117] Link the eldap connection process Ensure if LDAP GenServer crashes it gets cleaned up, and we should crash and restart if somehow the eldap connection process crashes unexpectedly as we can't seem to receive any DOWN messages from it, etc. --- lib/pleroma/ldap.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/pleroma/ldap.ex b/lib/pleroma/ldap.ex index 6be2188f4..0723cd094 100644 --- a/lib/pleroma/ldap.ex +++ b/lib/pleroma/ldap.ex @@ -59,6 +59,7 @@ defmodule Pleroma.LDAP do case connect() do {:ok, connection} -> :eldap.controlling_process(connection, self()) + Process.link(connection) [connection: connection] _ -> From 14a9663f1abe49b8f4f4f719fa2f4db3a5dd81b7 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 17 Sep 2024 13:28:42 -0400 Subject: [PATCH 100/117] Remove cacertfile as child of SSL and TLS options We need to pass the cacerts (list of charlist encoded certs) not cacertfile, so our new cacertfile setting handles this for us. --- config/description.exs | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/config/description.exs b/config/description.exs index ade47b7e0..5062842f0 100644 --- a/config/description.exs +++ b/config/description.exs @@ -2241,14 +2241,8 @@ config :pleroma, :config_description, [ label: "SSL options", type: :keyword, description: "Additional SSL options", - suggestions: [cacertfile: "path/to/file/with/PEM/cacerts", verify: :verify_peer], + suggestions: [verify: :verify_peer], children: [ - %{ - key: :cacertfile, - type: :string, - description: "Path to file with PEM encoded cacerts", - suggestions: ["path/to/file/with/PEM/cacerts"] - }, %{ key: :verify, type: :atom, @@ -2268,14 +2262,8 @@ config :pleroma, :config_description, [ label: "TLS options", type: :keyword, description: "Additional TLS options", - suggestions: [cacertfile: "path/to/file/with/PEM/cacerts", verify: :verify_peer], + suggestions: [verify: :verify_peer], children: [ - %{ - key: :cacertfile, - type: :string, - description: "Path to file with PEM encoded cacerts", - suggestions: ["path/to/file/with/PEM/cacerts"] - }, %{ key: :verify, type: :atom, From 363b462c54c454e847072869db09f8f4d5da4426 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 17 Sep 2024 13:36:46 -0400 Subject: [PATCH 101/117] Make the email attribute configurable While here, fix the System.get_env usage to use the normal fallback value method and improve the UID label description --- config/config.exs | 11 ++++++----- config/description.exs | 9 ++++++++- lib/pleroma/ldap.ex | 4 +++- 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/config/config.exs b/config/config.exs index f53a083d0..47ddfac5a 100644 --- a/config/config.exs +++ b/config/config.exs @@ -612,16 +612,17 @@ config :pleroma, Pleroma.Formatter, config :pleroma, :ldap, enabled: System.get_env("LDAP_ENABLED") == "true", - host: System.get_env("LDAP_HOST") || "localhost", - port: String.to_integer(System.get_env("LDAP_PORT") || "389"), + host: System.get_env("LDAP_HOST", "localhost"), + port: String.to_integer(System.get_env("LDAP_PORT", "389")), ssl: System.get_env("LDAP_SSL") == "true", sslopts: [], tls: System.get_env("LDAP_TLS") == "true", tlsopts: [], - base: System.get_env("LDAP_BASE") || "dc=example,dc=com", - uid: System.get_env("LDAP_UID") || "cn", + base: System.get_env("LDAP_BASE", "dc=example,dc=com"), + uid: System.get_env("LDAP_UID", "cn"), # defaults to CAStore's Mozilla roots - cacertfile: nil + cacertfile: System.get_env("LDAP_CACERTFILE", nil), + mail: System.get_env("LDAP_MAIL", "mail") oauth_consumer_strategies = System.get_env("OAUTH_CONSUMER_STRATEGIES") diff --git a/config/description.exs b/config/description.exs index 5062842f0..e85ec0ff8 100644 --- a/config/description.exs +++ b/config/description.exs @@ -2280,7 +2280,7 @@ config :pleroma, :config_description, [ }, %{ key: :uid, - label: "UID", + label: "UID Attribute", type: :string, description: "LDAP attribute name to authenticate the user, e.g. when \"cn\", the filter will be \"cn=username,base\"", @@ -2291,6 +2291,13 @@ config :pleroma, :config_description, [ label: "CACertfile", type: :string, description: "Path to CA certificate file" + }, + %{ + key: :mail, + label: "Mail Attribute", + type: :string, + description: "LDAP attribute name to use as the email address when automatically registering the user on first login", + suggestions: ["mail"] } ] }, diff --git a/lib/pleroma/ldap.ex b/lib/pleroma/ldap.ex index 0723cd094..8e9c591b2 100644 --- a/lib/pleroma/ldap.ex +++ b/lib/pleroma/ldap.ex @@ -205,6 +205,8 @@ defmodule Pleroma.LDAP do end defp try_register(name, attributes) do + mail_attribute = Config.get([:ldap, :mail]) + params = %{ name: name, nickname: name, @@ -212,7 +214,7 @@ defmodule Pleroma.LDAP do } params = - case List.keyfind(attributes, ~c"mail", 0) do + case List.keyfind(attributes, to_charlist(mail_attribute), 0) do {_, [mail]} -> Map.put_new(params, :email, :erlang.list_to_binary(mail)) _ -> params end From 21bf229731f27426564140650397e51ba4bb4b93 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 17 Sep 2024 13:43:21 -0400 Subject: [PATCH 102/117] Reduce LDAP timeouts 10 seconds is way too long for any login attempt or search result. LDAP should always be fast. --- lib/pleroma/ldap.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/ldap.ex b/lib/pleroma/ldap.ex index 8e9c591b2..33c9cff29 100644 --- a/lib/pleroma/ldap.ex +++ b/lib/pleroma/ldap.ex @@ -8,8 +8,8 @@ defmodule Pleroma.LDAP do import Pleroma.Web.Auth.Helpers, only: [fetch_user: 1] - @connection_timeout 10_000 - @search_timeout 10_000 + @connection_timeout 2_000 + @search_timeout 2_000 def start_link(_) do GenServer.start_link(__MODULE__, [], name: __MODULE__) From 1d123832da6a2b8c67f34006b4ea05e0be86e366 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 17 Sep 2024 13:46:49 -0400 Subject: [PATCH 103/117] Formatting --- config/description.exs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config/description.exs b/config/description.exs index e85ec0ff8..47f4771eb 100644 --- a/config/description.exs +++ b/config/description.exs @@ -2296,7 +2296,8 @@ config :pleroma, :config_description, [ key: :mail, label: "Mail Attribute", type: :string, - description: "LDAP attribute name to use as the email address when automatically registering the user on first login", + description: + "LDAP attribute name to use as the email address when automatically registering the user on first login", suggestions: ["mail"] } ] From ea63533cf28be8218a27806b1f6430f0c1ca4a01 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 17 Sep 2024 13:46:56 -0400 Subject: [PATCH 104/117] Change :connection to :handle to match upstream nomenclature --- lib/pleroma/ldap.ex | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/lib/pleroma/ldap.ex b/lib/pleroma/ldap.ex index 33c9cff29..b357b371a 100644 --- a/lib/pleroma/ldap.ex +++ b/lib/pleroma/ldap.ex @@ -47,7 +47,7 @@ defmodule Pleroma.LDAP do def handle_info(:connect, _state), do: do_handle_connect() def handle_info({:bind_after_reconnect, name, password, from}, state) do - result = bind_user(state[:connection], name, password) + result = bind_user(state[:handle], name, password) GenServer.reply(from, result) @@ -57,10 +57,10 @@ defmodule Pleroma.LDAP do defp do_handle_connect do state = case connect() do - {:ok, connection} -> - :eldap.controlling_process(connection, self()) - Process.link(connection) - [connection: connection] + {:ok, handle} -> + :eldap.controlling_process(handle, self()) + Process.link(handle) + [handle: handle] _ -> Logger.error("Failed to connect to LDAP. Retrying in 5000ms") @@ -73,7 +73,7 @@ defmodule Pleroma.LDAP do @impl true def handle_call({:bind_user, name, password}, from, state) do - case bind_user(state[:connection], name, password) do + case bind_user(state[:handle], name, password) do :needs_reconnect -> Process.send(self(), {:bind_after_reconnect, name, password, from}, []) {:noreply, state, {:continue, :connect}} @@ -85,10 +85,10 @@ defmodule Pleroma.LDAP do @impl true def terminate(_, state) do - connection = Keyword.get(state, :connection) + handle = Keyword.get(state, :handle) - if not is_nil(connection) do - :eldap.close(connection) + if not is_nil(handle) do + :eldap.close(handle) end :ok @@ -127,25 +127,25 @@ defmodule Pleroma.LDAP do end case :eldap.open([to_charlist(host)], options) do - {:ok, connection} -> + {:ok, handle} -> try do cond do tls -> case :eldap.start_tls( - connection, + handle, tlsopts, @connection_timeout ) do :ok -> - {:ok, connection} + {:ok, handle} error -> Logger.error("Could not start TLS: #{inspect(error)}") - :eldap.close(connection) + :eldap.close(handle) end true -> - {:ok, connection} + {:ok, handle} end after :ok @@ -157,24 +157,24 @@ defmodule Pleroma.LDAP do end end - defp bind_user(connection, name, password) do + defp bind_user(handle, name, password) do uid = Config.get([:ldap, :uid], "cn") base = Config.get([:ldap, :base]) - case :eldap.simple_bind(connection, "#{uid}=#{name},#{base}", password) do + case :eldap.simple_bind(handle, "#{uid}=#{name},#{base}", password) do :ok -> case fetch_user(name) do %User{} = user -> user _ -> - register_user(connection, base, uid, name) + register_user(handle, base, uid, name) end # eldap does not inform us of socket closure # until it is used {:error, {:gen_tcp_error, :closed}} -> - :eldap.close(connection) + :eldap.close(handle) :needs_reconnect error -> @@ -183,8 +183,8 @@ defmodule Pleroma.LDAP do end end - defp register_user(connection, base, uid, name) do - case :eldap.search(connection, [ + defp register_user(handle, base, uid, name) do + case :eldap.search(handle, [ {:base, to_charlist(base)}, {:filter, :eldap.equalityMatch(to_charlist(uid), to_charlist(name))}, {:scope, :eldap.wholeSubtree()}, From 2b482e34ebf5aee49350d8198e6fade8820b3834 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 17 Sep 2024 13:54:57 -0400 Subject: [PATCH 105/117] Improve matching on bind errors --- lib/pleroma/ldap.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/ldap.ex b/lib/pleroma/ldap.ex index b357b371a..cd84dee02 100644 --- a/lib/pleroma/ldap.ex +++ b/lib/pleroma/ldap.ex @@ -177,9 +177,9 @@ defmodule Pleroma.LDAP do :eldap.close(handle) :needs_reconnect - error -> + {:error, error} = e -> Logger.error("Could not bind LDAP user #{name}: #{inspect(error)}") - {:error, {:ldap_bind_error, error}} + e end end From 35ddb1d2c8a53dcb54178522811242ef40a63211 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 17 Sep 2024 13:57:10 -0400 Subject: [PATCH 106/117] LDAP genserver changelog --- changelog.d/ldap-refactor.change | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/ldap-refactor.change diff --git a/changelog.d/ldap-refactor.change b/changelog.d/ldap-refactor.change new file mode 100644 index 000000000..1510eea6a --- /dev/null +++ b/changelog.d/ldap-refactor.change @@ -0,0 +1 @@ +LDAP authentication has been refactored to operate as a GenServer process which will maintain an active connection to the LDAP server. From 1de5208a9e90485be38ac0d00088f18c5b36390a Mon Sep 17 00:00:00 2001 From: Mint Date: Tue, 17 Sep 2024 21:48:37 +0300 Subject: [PATCH 107/117] Cheatsheet: add Mua mail adapter config --- docs/configuration/cheatsheet.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index 0b4e53b6f..ba41fe84d 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -742,6 +742,21 @@ config :pleroma, Pleroma.Emails.Mailer, auth: :always ``` +An example for Mua adapter: + +```elixir +config :pleroma, Pleroma.Emails.Mailer, + enabled: true, + adapter: Swoosh.Adapters.Mua, + relay: "mail.example.com", + port: 465, + auth: [ + username: "YOUR_USERNAME@domain.tld", + password: "YOUR_SMTP_PASSWORD" + ], + protocol: :ssl +``` + ### :email_notifications Email notifications settings. From 73204c1bca740dbca5c780891fc720ac728c11a6 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 18 Sep 2024 11:16:16 -0400 Subject: [PATCH 108/117] LDAP: fix compile warning Sometimes the compile will emit the following warning, so we'll just avoid it by making it call a function in the LDAP module which will never have this problem. warning: :GenServer.call/2 is undefined (module :GenServer is not available or is yet to be defined) --- changelog.d/ldap-warning.skip | 0 lib/pleroma/ldap.ex | 4 ++++ lib/pleroma/web/auth/ldap_authenticator.ex | 3 ++- 3 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 changelog.d/ldap-warning.skip diff --git a/changelog.d/ldap-warning.skip b/changelog.d/ldap-warning.skip new file mode 100644 index 000000000..e69de29bb diff --git a/lib/pleroma/ldap.ex b/lib/pleroma/ldap.ex index cd84dee02..46a2d0c17 100644 --- a/lib/pleroma/ldap.ex +++ b/lib/pleroma/ldap.ex @@ -94,6 +94,10 @@ defmodule Pleroma.LDAP do :ok end + def bind_user(name, password) do + GenServer.call(__MODULE__, {:bind_user, name, password}) + end + defp connect do ldap = Config.get(:ldap, []) host = Keyword.get(ldap, :host, "localhost") diff --git a/lib/pleroma/web/auth/ldap_authenticator.ex b/lib/pleroma/web/auth/ldap_authenticator.ex index c420c8bc3..7eb06183d 100644 --- a/lib/pleroma/web/auth/ldap_authenticator.ex +++ b/lib/pleroma/web/auth/ldap_authenticator.ex @@ -3,6 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.Auth.LDAPAuthenticator do + alias Pleroma.LDAP alias Pleroma.User import Pleroma.Web.Auth.Helpers, only: [fetch_credentials: 1] @@ -19,7 +20,7 @@ defmodule Pleroma.Web.Auth.LDAPAuthenticator do def get_user(%Plug.Conn{} = conn) do with {:ldap, true} <- {:ldap, Pleroma.Config.get([:ldap, :enabled])}, {:ok, {name, password}} <- fetch_credentials(conn), - %User{} = user <- GenServer.call(Pleroma.LDAP, {:bind_user, name, password}) do + %User{} = user <- LDAP.bind_user(name, password) do {:ok, user} else {:ldap, _} -> From ecd1b8393befe91175872af3db67a5c01f10eaf2 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 18 Sep 2024 12:07:52 -0400 Subject: [PATCH 109/117] Oban: update to 2.18.3 This release includes the fix which should prevent the scenario where Postgrex crashes can cause Oban to get into a state where it will stop processing jobs. --- changelog.d/oban-update.change | 1 + mix.lock | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/oban-update.change diff --git a/changelog.d/oban-update.change b/changelog.d/oban-update.change new file mode 100644 index 000000000..48a54ed2d --- /dev/null +++ b/changelog.d/oban-update.change @@ -0,0 +1 @@ +Oban updated to 2.18.3 diff --git a/mix.lock b/mix.lock index 2cf44862b..421f99ec0 100644 --- a/mix.lock +++ b/mix.lock @@ -92,7 +92,7 @@ "nimble_parsec": {:hex, :nimble_parsec, "0.6.0", "32111b3bf39137144abd7ba1cce0914533b2d16ef35e8abc5ec8be6122944263", [:mix], [], "hexpm", "27eac315a94909d4dc68bc07a4a83e06c8379237c5ea528a9acff4ca1c873c52"}, "nimble_pool": {:hex, :nimble_pool, "0.2.6", "91f2f4c357da4c4a0a548286c84a3a28004f68f05609b4534526871a22053cde", [:mix], [], "hexpm", "1c715055095d3f2705c4e236c18b618420a35490da94149ff8b580a2144f653f"}, "nodex": {:git, "https://git.pleroma.social/pleroma/nodex", "cb6730f943cfc6aad674c92161be23a8411f15d1", [ref: "cb6730f943cfc6aad674c92161be23a8411f15d1"]}, - "oban": {:hex, :oban, "2.18.2", "583e78965ee15263ac968e38c983bad169ae55eadaa8e1e39912562badff93ba", [:mix], [{:ecto_sql, "~> 3.10", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:ecto_sqlite3, "~> 0.9", [hex: :ecto_sqlite3, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.16", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "9dd25fd35883a91ed995e9fe516e479344d3a8623dfe2b8c3fc8e5be0228ec3a"}, + "oban": {:hex, :oban, "2.18.3", "1608c04f8856c108555c379f2f56bc0759149d35fa9d3b825cb8a6769f8ae926", [:mix], [{:ecto_sql, "~> 3.10", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:ecto_sqlite3, "~> 0.9", [hex: :ecto_sqlite3, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.16", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "36ca6ca84ef6518f9c2c759ea88efd438a3c81d667ba23b02b062a0aa785475e"}, "oban_live_dashboard": {:hex, :oban_live_dashboard, "0.1.1", "8aa4ceaf381c818f7d5c8185cc59942b8ac82ef0cf559881aacf8d3f8ac7bdd3", [:mix], [{:oban, "~> 2.15", [hex: :oban, repo: "hexpm", optional: false]}, {:phoenix_live_dashboard, "~> 0.7", [hex: :phoenix_live_dashboard, repo: "hexpm", optional: false]}], "hexpm", "16dc4ce9c9a95aa2e655e35ed4e675652994a8def61731a18af85e230e1caa63"}, "octo_fetch": {:hex, :octo_fetch, "0.4.0", "074b5ecbc08be10b05b27e9db08bc20a3060142769436242702931c418695b19", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~> 1.1", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm", "cf8be6f40cd519d7000bb4e84adcf661c32e59369ca2827c4e20042eda7a7fc6"}, "open_api_spex": {:hex, :open_api_spex, "3.18.2", "8c855e83bfe8bf81603d919d6e892541eafece3720f34d1700b58024dadde247", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:poison, "~> 3.0 or ~> 4.0 or ~> 5.0", [hex: :poison, repo: "hexpm", optional: true]}, {:ymlr, "~> 2.0 or ~> 3.0 or ~> 4.0", [hex: :ymlr, repo: "hexpm", optional: true]}], "hexpm", "aa3e6dcfc0ad6a02596b2172662da21c9dd848dac145ea9e603f54e3d81b8d2b"}, From f00545d85bd601734cdbbc28454f33541dbf530d Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 18 Sep 2024 13:14:17 -0400 Subject: [PATCH 110/117] Elixir 1.14 and Erlang/OTP 23 is now the minimum supported release --- .gitlab-ci.yml | 8 ++++---- changelog.d/elixir.change | 1 + docs/installation/debian_based_jp.md | 2 +- docs/installation/generic_dependencies.include | 4 ++-- mix.exs | 2 +- 5 files changed, 9 insertions(+), 8 deletions(-) create mode 100644 changelog.d/elixir.change diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 76d1a4210..39947c75e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,8 +1,8 @@ -image: git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.13.4-otp-25 +image: git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.14.5-otp-25 variables: &global_variables # Only used for the release - ELIXIR_VER: 1.13.4 + ELIXIR_VER: 1.14.5 POSTGRES_DB: pleroma_test POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres @@ -71,7 +71,7 @@ check-changelog: tags: - amd64 -build-1.13.4-otp-25: +build-1.14.5-otp-25: extends: - .build_changes_policy - .using-ci-base @@ -119,7 +119,7 @@ benchmark: - mix ecto.migrate - mix pleroma.load_testing -unit-testing-1.13.4-otp-25: +unit-testing-1.14.5-otp-25: extends: - .build_changes_policy - .using-ci-base diff --git a/changelog.d/elixir.change b/changelog.d/elixir.change new file mode 100644 index 000000000..779c01562 --- /dev/null +++ b/changelog.d/elixir.change @@ -0,0 +1 @@ +Elixir 1.14 and Erlang/OTP 23 is now the minimum supported release diff --git a/docs/installation/debian_based_jp.md b/docs/installation/debian_based_jp.md index 5a0823a63..0817934ff 100644 --- a/docs/installation/debian_based_jp.md +++ b/docs/installation/debian_based_jp.md @@ -14,7 +14,7 @@ Note: This article is potentially outdated because at this time we may not have - PostgreSQL 11.0以上 (Ubuntu16.04では9.5しか提供されていないので，[](https://www.postgresql.org/download/linux/ubuntu/)こちらから新しいバージョンを入手してください) - `postgresql-contrib` 11.0以上 (同上) -- Elixir 1.13 以上 ([Debianのリポジトリからインストールしないこと！！！ ここからインストールすること！](https://elixir-lang.org/install.html#unix-and-unix-like)。または [asdf](https://github.com/asdf-vm/asdf) をpleromaユーザーでインストールしてください) +- Elixir 1.14 以上 ([Debianのリポジトリからインストールしないこと！！！ ここからインストールすること！](https://elixir-lang.org/install.html#unix-and-unix-like)。または [asdf](https://github.com/asdf-vm/asdf) をpleromaユーザーでインストールしてください) - `erlang-dev` - `erlang-nox` - `git` diff --git a/docs/installation/generic_dependencies.include b/docs/installation/generic_dependencies.include index bdb7f94d3..9f07f62c6 100644 --- a/docs/installation/generic_dependencies.include +++ b/docs/installation/generic_dependencies.include @@ -1,8 +1,8 @@ ## Required dependencies * PostgreSQL >=11.0 -* Elixir >=1.13.0 <1.17 -* Erlang OTP >=22.2.0 (supported: <27) +* Elixir >=1.14.0 <1.17 +* Erlang OTP >=23.0.0 (supported: <27) * git * file / libmagic * gcc or clang diff --git a/mix.exs b/mix.exs index ceae5c26d..89ec5e831 100644 --- a/mix.exs +++ b/mix.exs @@ -5,7 +5,7 @@ defmodule Pleroma.Mixfile do [ app: :pleroma, version: version("2.7.0"), - elixir: "~> 1.13", + elixir: "~> 1.14", elixirc_paths: elixirc_paths(Mix.env()), compilers: Mix.compilers(), elixirc_options: [warnings_as_errors: warnings_as_errors(), prune_code_paths: false], From 7e303600fb2914ab66fe54a8022ddc65ff93edf5 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 18 Sep 2024 17:15:55 +0000 Subject: [PATCH 111/117] Remove old elixir 1.12 build image generation script --- ci/elixir-1.12/Dockerfile | 8 -------- ci/elixir-1.12/build_and_push.sh | 1 - 2 files changed, 9 deletions(-) delete mode 100644 ci/elixir-1.12/Dockerfile delete mode 100755 ci/elixir-1.12/build_and_push.sh diff --git a/ci/elixir-1.12/Dockerfile b/ci/elixir-1.12/Dockerfile deleted file mode 100644 index a2b566873..000000000 --- a/ci/elixir-1.12/Dockerfile +++ /dev/null @@ -1,8 +0,0 @@ -FROM elixir:1.12.3 - -# Single RUN statement, otherwise intermediate images are created -# https://docs.docker.com/develop/develop-images/dockerfile_best-practices/#run -RUN apt-get update &&\ - apt-get install -y libmagic-dev cmake libimage-exiftool-perl ffmpeg &&\ - mix local.hex --force &&\ - mix local.rebar --force diff --git a/ci/elixir-1.12/build_and_push.sh b/ci/elixir-1.12/build_and_push.sh deleted file mode 100755 index 508262ed8..000000000 --- a/ci/elixir-1.12/build_and_push.sh +++ /dev/null @@ -1 +0,0 @@ -docker buildx build --platform linux/amd64,linux/arm64,linux/arm/v7 -t git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.12 --push . From 1bd28e7d592b429c5eee072db8d1f2ae77d76e29 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 18 Sep 2024 17:28:48 +0000 Subject: [PATCH 112/117] CI script to build and publish an image for Elixir 1.14 --- ci/{elixir-1.13.4-otp-25 => elixir-1.14.5-otp-25}/Dockerfile | 2 +- .../build_and_push.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename ci/{elixir-1.13.4-otp-25 => elixir-1.14.5-otp-25}/Dockerfile (91%) rename ci/{elixir-1.13.4-otp-25 => elixir-1.14.5-otp-25}/build_and_push.sh (52%) diff --git a/ci/elixir-1.13.4-otp-25/Dockerfile b/ci/elixir-1.14.5-otp-25/Dockerfile similarity index 91% rename from ci/elixir-1.13.4-otp-25/Dockerfile rename to ci/elixir-1.14.5-otp-25/Dockerfile index 25a1639e8..3a35c84c3 100644 --- a/ci/elixir-1.13.4-otp-25/Dockerfile +++ b/ci/elixir-1.14.5-otp-25/Dockerfile @@ -1,4 +1,4 @@ -FROM elixir:1.13.4-otp-25 +FROM elixir:1.14.5-otp-25 # Single RUN statement, otherwise intermediate images are created # https://docs.docker.com/develop/develop-images/dockerfile_best-practices/#run diff --git a/ci/elixir-1.13.4-otp-25/build_and_push.sh b/ci/elixir-1.14.5-otp-25/build_and_push.sh similarity index 52% rename from ci/elixir-1.13.4-otp-25/build_and_push.sh rename to ci/elixir-1.14.5-otp-25/build_and_push.sh index b8ca1d24d..912c47d0c 100755 --- a/ci/elixir-1.13.4-otp-25/build_and_push.sh +++ b/ci/elixir-1.14.5-otp-25/build_and_push.sh @@ -1 +1 @@ -docker buildx build --platform linux/amd64,linux/arm64 -t git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.13.4-otp-25 --push . +docker buildx build --platform linux/amd64,linux/arm64 -t git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.14.5-otp-25 --push . From 03e14e759db47633cce320285d93d8c1f3bde65c Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Fri, 24 Mar 2023 09:08:39 +0100 Subject: [PATCH 113/117] MRF: Add filtering against AP id --- changelog.d/mrf-id_filter.add | 1 + lib/pleroma/web/activity_pub/mrf.ex | 8 ++++++++ lib/pleroma/web/activity_pub/mrf/policy.ex | 3 ++- 3 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 changelog.d/mrf-id_filter.add diff --git a/changelog.d/mrf-id_filter.add b/changelog.d/mrf-id_filter.add new file mode 100644 index 000000000..f556f9bc4 --- /dev/null +++ b/changelog.d/mrf-id_filter.add @@ -0,0 +1 @@ +Add `id_filter` to MRF to filter URLs and their domain prior to fetching \ No newline at end of file diff --git a/lib/pleroma/web/activity_pub/mrf.ex b/lib/pleroma/web/activity_pub/mrf.ex index bc418d908..51ab476b7 100644 --- a/lib/pleroma/web/activity_pub/mrf.ex +++ b/lib/pleroma/web/activity_pub/mrf.ex @@ -108,6 +108,14 @@ defmodule Pleroma.Web.ActivityPub.MRF do def filter(%{} = object), do: get_policies() |> filter(object) + def id_filter(policies, id) when is_binary(id) do + policies + |> Enum.filter(&function_exported?(&1, :id_filter, 1)) + |> Enum.all?(& &1.id_filter(id)) + end + + def id_filter(id) when is_binary(id), do: get_policies() |> id_filter(id) + @impl true def pipeline_filter(%{} = message, meta) do object = meta[:object_data] diff --git a/lib/pleroma/web/activity_pub/mrf/policy.ex b/lib/pleroma/web/activity_pub/mrf/policy.ex index 54ca4b735..08bcac08a 100644 --- a/lib/pleroma/web/activity_pub/mrf/policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/policy.ex @@ -4,6 +4,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.Policy do @callback filter(Pleroma.Activity.t()) :: {:ok | :reject, Pleroma.Activity.t()} + @callback id_filter(String.t()) :: boolean() @callback describe() :: {:ok | :error, map()} @callback config_description() :: %{ optional(:children) => [map()], @@ -13,5 +14,5 @@ defmodule Pleroma.Web.ActivityPub.MRF.Policy do description: String.t() } @callback history_awareness() :: :auto | :manual - @optional_callbacks config_description: 0, history_awareness: 0 + @optional_callbacks config_description: 0, history_awareness: 0, id_filter: 1 end From 3dd6f6585985a085c1c2f2243501323864dcac2d Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Fri, 24 Mar 2023 09:09:41 +0100 Subject: [PATCH 114/117] Object.Fetcher: Hook to MRF.id_filter --- lib/pleroma/object/fetcher.ex | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index 69a5f3268..c85a8b09f 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -145,6 +145,7 @@ defmodule Pleroma.Object.Fetcher do Logger.debug("Fetching object #{id} via AP") with {:scheme, true} <- {:scheme, String.starts_with?(id, "http")}, + {_, true} <- {:mrf, MRF.id_filter(id)}, {:ok, body} <- get_object(id), {:ok, data} <- safe_json_decode(body), :ok <- Containment.contain_origin_from_id(id, data) do @@ -160,6 +161,9 @@ defmodule Pleroma.Object.Fetcher do {:error, e} -> {:error, e} + {:mrf, false} -> + {:error, {:reject, "Filtered by id"}} + e -> {:error, e} end From 30063c5914d229753f3bacab98c38736f2a447e6 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Fri, 24 Mar 2023 09:09:58 +0100 Subject: [PATCH 115/117] MRF.DropPolicy: Add id_filter/1 --- lib/pleroma/web/activity_pub/mrf/drop_policy.ex | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/pleroma/web/activity_pub/mrf/drop_policy.ex b/lib/pleroma/web/activity_pub/mrf/drop_policy.ex index e4fcc9935..cf07db7f3 100644 --- a/lib/pleroma/web/activity_pub/mrf/drop_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/drop_policy.ex @@ -13,6 +13,12 @@ defmodule Pleroma.Web.ActivityPub.MRF.DropPolicy do {:reject, activity} end + @impl true + def id_filter(id) do + Logger.debug("REJECTING #{id}") + false + end + @impl true def describe, do: {:ok, %{}} end From 0fa13c55357ca83ae00b39626a0fa4be3a936640 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Fri, 24 Mar 2023 09:16:25 +0100 Subject: [PATCH 116/117] MRF.SimplePolicy: Add id_filter/1 --- lib/pleroma/web/activity_pub/mrf/simple_policy.ex | 12 ++++++++++++ .../web/activity_pub/mrf/simple_policy_test.exs | 15 +++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex index ae7f18bfe..a97e8db7b 100644 --- a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex @@ -191,6 +191,18 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do |> MRF.instance_list_from_tuples() end + @impl true + def id_filter(id) do + host_info = URI.parse(id) + + with {:ok, _} <- check_accept(host_info, %{}), + {:ok, _} <- check_reject(host_info, %{}) do + true + else + _ -> false + end + end + @impl true def filter(%{"type" => "Delete", "actor" => actor} = activity) do %{host: actor_host} = URI.parse(actor) diff --git a/test/pleroma/web/activity_pub/mrf/simple_policy_test.exs b/test/pleroma/web/activity_pub/mrf/simple_policy_test.exs index 1a51b7d30..f49a7b8ff 100644 --- a/test/pleroma/web/activity_pub/mrf/simple_policy_test.exs +++ b/test/pleroma/web/activity_pub/mrf/simple_policy_test.exs @@ -252,6 +252,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do remote_message = build_remote_message() assert SimplePolicy.filter(remote_message) == {:ok, remote_message} + assert SimplePolicy.id_filter(remote_message["actor"]) end test "activity has a matching host" do @@ -260,6 +261,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do remote_message = build_remote_message() assert {:reject, _} = SimplePolicy.filter(remote_message) + refute SimplePolicy.id_filter(remote_message["actor"]) end test "activity matches with wildcard domain" do @@ -268,6 +270,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do remote_message = build_remote_message() assert {:reject, _} = SimplePolicy.filter(remote_message) + refute SimplePolicy.id_filter(remote_message["actor"]) end test "actor has a matching host" do @@ -276,6 +279,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do remote_user = build_remote_user() assert {:reject, _} = SimplePolicy.filter(remote_user) + refute SimplePolicy.id_filter(remote_user["id"]) end test "reject Announce when object would be rejected" do @@ -288,6 +292,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do } assert {:reject, _} = SimplePolicy.filter(announce) + # Note: Non-Applicable for id_filter/1 end test "reject by URI object" do @@ -300,6 +305,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do } assert {:reject, _} = SimplePolicy.filter(announce) + # Note: Non-Applicable for id_filter/1 end end @@ -370,6 +376,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do assert SimplePolicy.filter(local_message) == {:ok, local_message} assert SimplePolicy.filter(remote_message) == {:ok, remote_message} + assert SimplePolicy.id_filter(local_message["actor"]) + assert SimplePolicy.id_filter(remote_message["actor"]) end test "is not empty but activity doesn't have a matching host" do @@ -380,6 +388,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do assert SimplePolicy.filter(local_message) == {:ok, local_message} assert {:reject, _} = SimplePolicy.filter(remote_message) + assert SimplePolicy.id_filter(local_message["actor"]) + refute SimplePolicy.id_filter(remote_message["actor"]) end test "activity has a matching host" do @@ -390,6 +400,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do assert SimplePolicy.filter(local_message) == {:ok, local_message} assert SimplePolicy.filter(remote_message) == {:ok, remote_message} + assert SimplePolicy.id_filter(local_message["actor"]) + assert SimplePolicy.id_filter(remote_message["actor"]) end test "activity matches with wildcard domain" do @@ -400,6 +412,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do assert SimplePolicy.filter(local_message) == {:ok, local_message} assert SimplePolicy.filter(remote_message) == {:ok, remote_message} + assert SimplePolicy.id_filter(local_message["actor"]) + assert SimplePolicy.id_filter(remote_message["actor"]) end test "actor has a matching host" do @@ -408,6 +422,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do remote_user = build_remote_user() assert SimplePolicy.filter(remote_user) == {:ok, remote_user} + assert SimplePolicy.id_filter(remote_user["id"]) end end From a1e3fb506b309a529f0ce8ef231d853e7866be21 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Sat, 21 Sep 2024 15:39:02 +0200 Subject: [PATCH 117/117] Dockerfile: Elixir 1.14 --- Dockerfile | 7 ++++--- changelog.d/elixir-1.14-docker.skip | 0 2 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 changelog.d/elixir-1.14-docker.skip diff --git a/Dockerfile b/Dockerfile index 72461305c..fff58154e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,8 @@ +# https://hub.docker.com/r/hexpm/elixir/tags ARG ELIXIR_IMG=hexpm/elixir -ARG ELIXIR_VER=1.13.4 -ARG ERLANG_VER=24.3.4.15 -ARG ALPINE_VER=3.17.5 +ARG ELIXIR_VER=1.14.5 +ARG ERLANG_VER=25.3.2.14 +ARG ALPINE_VER=3.17.9 FROM ${ELIXIR_IMG}:${ELIXIR_VER}-erlang-${ERLANG_VER}-alpine-${ALPINE_VER} as build diff --git a/changelog.d/elixir-1.14-docker.skip b/changelog.d/elixir-1.14-docker.skip new file mode 100644 index 000000000..e69de29bb