From 32994bb9c37f822ec97a5f07aab94bb3e94d9b1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Thu, 3 Nov 2022 00:13:09 +0100 Subject: [PATCH 001/128] Language detection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- config/description.exs | 22 +++++++++ lib/pleroma/application_requirements.ex | 22 ++++++++- lib/pleroma/language/language_detector.ex | 34 ++++++++++++++ .../language/language_detector/fasttext.ex | 47 +++++++++++++++++++ .../language/language_detector/provider.ex | 11 +++++ lib/pleroma/web/common_api/activity_draft.ex | 15 +++--- 6 files changed, 144 insertions(+), 7 deletions(-) create mode 100644 lib/pleroma/language/language_detector.ex create mode 100644 lib/pleroma/language/language_detector/fasttext.ex create mode 100644 lib/pleroma/language/language_detector/provider.ex diff --git a/config/description.exs b/config/description.exs index 6c13bde31..f317c4e34 100644 --- a/config/description.exs +++ b/config/description.exs @@ -3523,5 +3523,27 @@ config :pleroma, :config_description, [ suggestion: [100_000] } ] + }, + %{ + group: :pleroma, + key: Pleroma.Language.LanguageDetector, + type: :group, + description: "Language detection providers", + children: [ + %{ + key: :provider, + type: :module, + suggestions: [ + Pleroma.Language.LanguageDetector.Fasttext + ] + }, + %{ + group: {:subgroup, Pleroma.Language.LanguageDetector.Fasttext}, + key: :model, + label: "fastText language detection model", + type: :string, + suggestions: ["/usr/share/fasttext/lid.176.bin"] + } + ] } ] diff --git a/lib/pleroma/application_requirements.ex b/lib/pleroma/application_requirements.ex index 819245481..ff22f835b 100644 --- a/lib/pleroma/application_requirements.ex +++ b/lib/pleroma/application_requirements.ex @@ -188,7 +188,27 @@ defmodule Pleroma.ApplicationRequirements do false end - if Enum.all?([preview_proxy_commands_status | filter_commands_statuses], & &1) do + language_detector_commands_status = + if Pleroma.Language.LanguageDetector.missing_dependencies() == [] do + true + else + Logger.error( + "The following dependencies required by the currently enabled " <> + "language detection provider are not installed: " <> + inspect(Pleroma.Language.LanguageDetector.missing_dependencies()) + ) + + false + end + + if Enum.all?( + [ + preview_proxy_commands_status, + language_detector_commands_status + | filter_commands_statuses + ], + & &1 + ) do :ok else {:error, diff --git a/lib/pleroma/language/language_detector.ex b/lib/pleroma/language/language_detector.ex new file mode 100644 index 000000000..3901a8b90 --- /dev/null +++ b/lib/pleroma/language/language_detector.ex @@ -0,0 +1,34 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Language.LanguageDetector do + @words_threshold 4 + + def missing_dependencies do + provider = get_provider() + + if provider do + provider.missing_dependencies() + else + nil + end + end + + def detect(text) do + provider = get_provider() + + {:ok, text} = text |> FastSanitize.strip_tags() + word_count = text |> String.split(~r/\s+/) |> Enum.count() + + if word_count < @words_threshold or !provider or !provider.configured? do + nil + else + provider.detect(text) + end + end + + defp get_provider() do + Pleroma.Config.get([__MODULE__, :provider]) + end +end diff --git a/lib/pleroma/language/language_detector/fasttext.ex b/lib/pleroma/language/language_detector/fasttext.ex new file mode 100644 index 000000000..d479d2125 --- /dev/null +++ b/lib/pleroma/language/language_detector/fasttext.ex @@ -0,0 +1,47 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Language.LanguageDetector.Fasttext do + import Pleroma.Web.Utils.Guards, only: [not_empty_string: 1] + + alias Pleroma.Language.LanguageDetector.Provider + + @behaviour Provider + + @impl Provider + def missing_dependencies do + if Pleroma.Utils.command_available?("fasttext") do + [] + else + ["fasttext"] + end + end + + @impl Provider + def configured?, do: not_empty_string(get_model()) + + @impl Provider + def detect(text) do + text_path = Path.join(System.tmp_dir!(), "fasttext-#{Ecto.UUID.generate()}") + + File.write(text_path, text) + + detected_language = + case System.cmd("fasttext", ["predict", get_model(), text_path]) do + {"__label__" <> language, _} -> + language |> String.trim() + + _ -> + nil + end + + File.rm(text_path) + + detected_language + end + + defp get_model do + Pleroma.Config.get([__MODULE__, :model]) + end +end diff --git a/lib/pleroma/language/language_detector/provider.ex b/lib/pleroma/language/language_detector/provider.ex new file mode 100644 index 000000000..08e7c8eef --- /dev/null +++ b/lib/pleroma/language/language_detector/provider.ex @@ -0,0 +1,11 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Language.LanguageDetector.Provider do + @callback missing_dependencies() :: [String.t()] + + @callback configured?() :: boolean() + + @callback detect(text :: String.t()) :: String.t() | nil +end diff --git a/lib/pleroma/web/common_api/activity_draft.ex b/lib/pleroma/web/common_api/activity_draft.ex index 4b7d28f5c..7728c6bcb 100644 --- a/lib/pleroma/web/common_api/activity_draft.ex +++ b/lib/pleroma/web/common_api/activity_draft.ex @@ -5,6 +5,7 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do alias Pleroma.Activity alias Pleroma.Conversation.Participation + alias Pleroma.Language.LanguageDetector alias Pleroma.Object alias Pleroma.Web.ActivityPub.Builder alias Pleroma.Web.ActivityPub.Visibility @@ -241,13 +242,15 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do end defp language(draft) do - language = draft.params[:language] + language = + with language <- draft.params[:language], + true <- good_locale_code?(language) do + language + else + _ -> LanguageDetector.detect(draft.full_payload) + end - if good_locale_code?(language) do - %__MODULE__{draft | language: language} - else - draft - end + %__MODULE__{draft | language: language} end defp object(draft) do From 9932aeffc5b1469c9e42799d31f4599fc3db993b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Thu, 3 Nov 2022 22:43:20 +0100 Subject: [PATCH 002/128] Add test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/language/language_detector.ex | 4 +-- .../language/language_detector_test.ex | 31 +++++++++++++++++++ test/support/language_detector_mock.ex | 18 +++++++++++ 3 files changed, 51 insertions(+), 2 deletions(-) create mode 100644 test/pleroma/language/language_detector_test.ex create mode 100644 test/support/language_detector_mock.ex diff --git a/lib/pleroma/language/language_detector.ex b/lib/pleroma/language/language_detector.ex index 3901a8b90..b19eb4571 100644 --- a/lib/pleroma/language/language_detector.ex +++ b/lib/pleroma/language/language_detector.ex @@ -11,7 +11,7 @@ defmodule Pleroma.Language.LanguageDetector do if provider do provider.missing_dependencies() else - nil + [] end end @@ -28,7 +28,7 @@ defmodule Pleroma.Language.LanguageDetector do end end - defp get_provider() do + defp get_provider do Pleroma.Config.get([__MODULE__, :provider]) end end diff --git a/test/pleroma/language/language_detector_test.ex b/test/pleroma/language/language_detector_test.ex new file mode 100644 index 000000000..4d9af33bf --- /dev/null +++ b/test/pleroma/language/language_detector_test.ex @@ -0,0 +1,31 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Language.LanguageDetectorTest do + use Pleroma.Web.ConnCase + + alias Pleroma.Language.LanguageDetector + + setup do: clear_config([Pleroma.Language.LanguageDetector, :provider], LanguageDetectorMock) + + test "it detects text language" do + detected_language = LanguageDetector.detect("Je viens d'atterrir en Tchéquie.") + + assert detected_language == "fr" + end + + test "it returns nil if text is not long enough" do + detected_language = LanguageDetector.detect("it returns nil") + + assert detected_language == nil + end + + test "it returns nil if no provider specified" do + clear_config([Pleroma.Language.LanguageDetector, :provider], nil) + + detected_language = LanguageDetector.detect("this should also return nil") + + assert detected_language == nil + end +end diff --git a/test/support/language_detector_mock.ex b/test/support/language_detector_mock.ex new file mode 100644 index 000000000..2a85dcd63 --- /dev/null +++ b/test/support/language_detector_mock.ex @@ -0,0 +1,18 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule LanguageDetectorMock do + alias Pleroma.Language.LanguageDetector.Provider + + @behaviour Provider + + @impl Provider + def missing_dependencies, do: [] + + @impl Provider + def configured?, do: true + + @impl Provider + def detect(text), do: "fr" +end From 80dbbd5501a0665656aadc3b76f3db7d1da9becb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Thu, 25 Apr 2024 23:11:12 +0200 Subject: [PATCH 003/128] Detect language for incoming posts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- .../web/activity_pub/object_validators/common_fixes.ex | 10 +++++++++- test/support/language_detector_mock.ex | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex index a9dc4a312..52afc827e 100644 --- a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex +++ b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex @@ -4,6 +4,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes do alias Pleroma.EctoType.ActivityPub.ObjectValidators + alias Pleroma.Language.LanguageDetector alias Pleroma.Maps alias Pleroma.Object alias Pleroma.Object.Containment @@ -145,7 +146,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes do language = [ get_language_from_context(object), - get_language_from_content_map(object) + get_language_from_content_map(object), + get_language_from_content(object) ] |> Enum.find(&good_locale_code?(&1)) @@ -180,6 +182,12 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes do defp get_language_from_content_map(_), do: nil + defp get_language_from_content(%{"summary" => summary, "content" => content}) do + LanguageDetector.detect("#{summary} #{content}") + end + + defp get_language_from_content(_), do: nil + def maybe_add_content_map(%{"language" => language, "content" => content} = object) when not_empty_string(language) do Map.put(object, "contentMap", Map.put(%{}, language, content)) diff --git a/test/support/language_detector_mock.ex b/test/support/language_detector_mock.ex index 2a85dcd63..3e6a258ae 100644 --- a/test/support/language_detector_mock.ex +++ b/test/support/language_detector_mock.ex @@ -14,5 +14,5 @@ defmodule LanguageDetectorMock do def configured?, do: true @impl Provider - def detect(text), do: "fr" + def detect(_text), do: "fr" end From 17d885fed87ede236488e80552b9ee9557001e19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Sat, 5 Nov 2022 20:16:32 +0100 Subject: [PATCH 004/128] Fix fasttext for multiline posts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/language/language_detector/fasttext.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/language/language_detector/fasttext.ex b/lib/pleroma/language/language_detector/fasttext.ex index d479d2125..0f621a000 100644 --- a/lib/pleroma/language/language_detector/fasttext.ex +++ b/lib/pleroma/language/language_detector/fasttext.ex @@ -25,7 +25,7 @@ defmodule Pleroma.Language.LanguageDetector.Fasttext do def detect(text) do text_path = Path.join(System.tmp_dir!(), "fasttext-#{Ecto.UUID.generate()}") - File.write(text_path, text) + File.write(text_path, text |> String.replace(~r/\s+/, " ")) detected_language = case System.cmd("fasttext", ["predict", get_model(), text_path]) do From 8bec926bebe855e0968f5b71368876cbf2439333 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 8 Nov 2022 17:06:16 -0600 Subject: [PATCH 005/128] LanguageDetector: strip non-language text to (hopefully) improve accuracy --- lib/pleroma/language/language_detector.ex | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/language/language_detector.ex b/lib/pleroma/language/language_detector.ex index b19eb4571..0be69d220 100644 --- a/lib/pleroma/language/language_detector.ex +++ b/lib/pleroma/language/language_detector.ex @@ -15,10 +15,18 @@ defmodule Pleroma.Language.LanguageDetector do end end + # Strip tags from text, etc. + defp prepare_text(text) do + text + |> Floki.parse_fragment!() + |> Floki.filter_out(".h-card, .mention, .hashtag, .u-url, .quote-inline, .recipients-inline, code, pre") + |> Floki.text() + end + def detect(text) do provider = get_provider() - {:ok, text} = text |> FastSanitize.strip_tags() + text = prepare_text(text) word_count = text |> String.split(~r/\s+/) |> Enum.count() if word_count < @words_threshold or !provider or !provider.configured? do From 91f42781d36d92b791c84b5c59ea9df7090997a1 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 8 Nov 2022 17:23:41 -0600 Subject: [PATCH 006/128] ActivityDraft: detect language from content_html so it can strip links --- lib/pleroma/web/common_api/activity_draft.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/common_api/activity_draft.ex b/lib/pleroma/web/common_api/activity_draft.ex index 7728c6bcb..2495978b4 100644 --- a/lib/pleroma/web/common_api/activity_draft.ex +++ b/lib/pleroma/web/common_api/activity_draft.ex @@ -247,7 +247,7 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do true <- good_locale_code?(language) do language else - _ -> LanguageDetector.detect(draft.full_payload) + _ -> LanguageDetector.detect(draft.content_html <> " " <> draft.summary) end %__MODULE__{draft | language: language} From df0d84833d6bd5a82bade27486eaee09ca690d1c Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 8 Nov 2022 17:47:17 -0600 Subject: [PATCH 007/128] mix format --- lib/pleroma/language/language_detector.ex | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/language/language_detector.ex b/lib/pleroma/language/language_detector.ex index 0be69d220..42d200a28 100644 --- a/lib/pleroma/language/language_detector.ex +++ b/lib/pleroma/language/language_detector.ex @@ -19,7 +19,9 @@ defmodule Pleroma.Language.LanguageDetector do defp prepare_text(text) do text |> Floki.parse_fragment!() - |> Floki.filter_out(".h-card, .mention, .hashtag, .u-url, .quote-inline, .recipients-inline, code, pre") + |> Floki.filter_out( + ".h-card, .mention, .hashtag, .u-url, .quote-inline, .recipients-inline, code, pre" + ) |> Floki.text() end From fa24e0ff2229c645850d903c74c893c59f7537ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Thu, 25 Apr 2024 23:37:22 +0200 Subject: [PATCH 008/128] changelog MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- changelog.d/language-detection.add | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/language-detection.add diff --git a/changelog.d/language-detection.add b/changelog.d/language-detection.add new file mode 100644 index 000000000..6d1a7f705 --- /dev/null +++ b/changelog.d/language-detection.add @@ -0,0 +1 @@ +Implement language detection with fastText \ No newline at end of file From 557a7d736a873bf57c3e3e271669c0815fd6fe28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Sun, 30 Oct 2022 18:47:41 +0100 Subject: [PATCH 009/128] WIP Translation backends support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/application.ex | 3 +- .../api_spec/operations/status_operation.ex | 60 +++++++++++++++++++ .../controllers/status_controller.ex | 29 ++++++++- .../web/mastodon_api/views/status_view.ex | 8 +++ lib/pleroma/web/router.ex | 1 + 5 files changed, 99 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index de668052f..921384be9 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -182,7 +182,8 @@ defmodule Pleroma.Application do expiration: chat_message_id_idempotency_key_expiration(), limit: 500_000 ), - build_cachex("rel_me", limit: 2500) + build_cachex("rel_me", limit: 2500), + build_cachex("translations", default_ttl: :timer.hours(24), limit: 5_000) ] end diff --git a/lib/pleroma/web/api_spec/operations/status_operation.ex b/lib/pleroma/web/api_spec/operations/status_operation.ex index ef4e34044..961a2f402 100644 --- a/lib/pleroma/web/api_spec/operations/status_operation.ex +++ b/lib/pleroma/web/api_spec/operations/status_operation.ex @@ -409,6 +409,38 @@ defmodule Pleroma.Web.ApiSpec.StatusOperation do } end + def translate_operation do + %Operation{ + tags: ["Retrieve status information"], + summary: "Translate status", + description: "Translate status with an external API", + operationId: "StatusController.translate", + security: [%{"oAuth" => ["read:statuses"]}], + parameters: [id_param()], + requestBody: + request_body( + "Parameters", + %Schema{ + type: :object, + properties: %{ + target_language: %Schema{ + type: :string, + nullable: true, + description: "Translation target language." + } + } + }, + required: false + ), + responses: %{ + 200 => Operation.response("Translation", "application/json", translation()) + 400 => Operation.response("Error", "application/json", ApiError) + 404 => Operation.response("Error", "application/json", ApiError) + 503 => Operation.response("Error", "application/json", ApiError) + } + } + end + def favourites_operation do %Operation{ tags: ["Timelines"], @@ -793,4 +825,32 @@ defmodule Pleroma.Web.ApiSpec.StatusOperation do } } end + + defp translation do + %Schema{ + title: "StatusTranslation", + description: "Represents status translation with related information.", + type: :object, + required: [:content, :detected_source_language, :provider], + properties: %{ + content: %Schema{ + type: :string, + description: "Translated status content" + }, + detected_source_language: %Schema{ + type: :string, + description: "Detected source language" + }, + provider: %Schema{ + type: :string, + description: "Translation provider service name" + } + }, + example: %{ + "content" => "Software für die nächste Generation der sozialen Medien.", + "detected_source_language" => "en", + "provider" => "Deepl" + } + } + end end diff --git a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex index 5aa7bddf0..4e1651596 100644 --- a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex @@ -15,6 +15,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do alias Pleroma.Object alias Pleroma.Repo alias Pleroma.ScheduledActivity + alias Pleroma.Translation alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Visibility @@ -44,6 +45,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do ] ) + plug(OAuthScopesPlug, %{scopes: ["read:statuses"]} when action == :translate) + plug( OAuthScopesPlug, %{scopes: ["write:statuses"]} @@ -85,7 +88,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do %{scopes: ["write:bookmarks"]} when action in [:bookmark, :unbookmark] ) - @rate_limited_status_actions ~w(reblog unreblog favourite unfavourite create delete)a + @rate_limited_status_actions ~w(reblog unreblog favourite unfavourite create delete translate)a plug( RateLimiter, @@ -554,6 +557,30 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do end end + @doc "POST /api/v1/statuses/:id/translate" + def translate(%{body_params: params, assigns: %{user: user}} = conn, %{id: status_id}) do + with %Activity{object: object} <- Activity.get_by_id_with_object(status_id), + {:language, language} when is_binary(language) <- + {:language, Map.get(params, :target_language) || user.language}, + {:ok, result} <- + Translation.translate( + object.data["content"], + object.data["language"], + language + ) do + render(conn, "translation.json", result) + else + {:language, nil} -> + render_error(conn, :bad_request, "Language not specified") + + {:error, :not_found} -> + render_error(conn, :not_found, "Translation service not configured") + + {:error, error} when error in [:unexpected_response, :quota_exceeded, :too_many_requests] -> + render_error(conn, :service_unavailable, "Translation service not available") + end + end + @doc "GET /api/v1/favourites" def favourites( %{assigns: %{user: %User{} = user}, private: %{open_api_spex: %{params: params}}} = conn, diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index bae3fd1f8..026429d56 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -656,6 +656,14 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do } end + def render("translation.json", %{ + content: content, + detected_source_language: detected_source_language, + provider: provider + }) do + %{content: content, detected_source_language: detected_source_language, provider: provider} + end + def get_reply_to(activity, %{replied_to_activities: replied_to_activities}) do object = Object.normalize(activity, fetch: false) diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 8ba845364..5c949fc95 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -718,6 +718,7 @@ defmodule Pleroma.Web.Router do post("/statuses/:id/unbookmark", StatusController, :unbookmark) post("/statuses/:id/mute", StatusController, :mute_conversation) post("/statuses/:id/unmute", StatusController, :unmute_conversation) + post("/statuses/:id/translate", StatusController, :translate) post("/push/subscription", SubscriptionController, :create) get("/push/subscription", SubscriptionController, :show) From 90f91168f7ed9af6a4141fafa11417a6419a0c83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Sun, 30 Oct 2022 18:52:26 +0100 Subject: [PATCH 010/128] Expose translation service availability MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- config/description.exs | 13 ++++ lib/pleroma/translation.ex | 54 +++++++++++++ lib/pleroma/translation/deepl.ex | 75 +++++++++++++++++++ lib/pleroma/translation/libretranslate.ex | 66 ++++++++++++++++ lib/pleroma/translation/service.ex | 20 +++++ .../api_spec/operations/status_operation.ex | 6 +- .../web/mastodon_api/views/instance_view.ex | 3 +- 7 files changed, 233 insertions(+), 4 deletions(-) create mode 100644 lib/pleroma/translation.ex create mode 100644 lib/pleroma/translation/deepl.ex create mode 100644 lib/pleroma/translation/libretranslate.ex create mode 100644 lib/pleroma/translation/service.ex diff --git a/config/description.exs b/config/description.exs index 6c13bde31..0efea0882 100644 --- a/config/description.exs +++ b/config/description.exs @@ -3523,5 +3523,18 @@ config :pleroma, :config_description, [ suggestion: [100_000] } ] + }, + %{ + group: :pleroma, + key: Pleroma.Translation, + type: :group, + description: "Translation providers", + children: [ + %{ + key: Pleroma.Translation, + type: :service, + suggestions: [Pleroma.Translation.DeepL, Pleroma.Translation.LibreTranslate] + } + ] } ] diff --git a/lib/pleroma/translation.ex b/lib/pleroma/translation.ex new file mode 100644 index 000000000..112f12cab --- /dev/null +++ b/lib/pleroma/translation.ex @@ -0,0 +1,54 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Translation do + @cache_ttl 86_400_000 + @cachex Pleroma.Config.get([:cachex, :provider], Cachex) + + def configured? do + service = get_service() + + !!service and service.configured? + end + + def translate(text, source_language, target_language) do + cache_key = get_cache_key(text, source_language, target_language) + + case @cachex.get(:translations_cache, cache_key) do + {:ok, nil} -> + service = get_service() + + result = + if !service or !service.configured? do + {:error, :not_found} + else + service.translate(text, source_language, target_language) + end + + store_result(result, cache_key) + + result + + {:ok, result} -> + {:ok, result} + + {:error, error} -> + {:error, error} + end + end + + defp get_service, do: Pleroma.Config.get([__MODULE__, :service]) + + defp get_cache_key(text, source_language, target_language) do + "#{source_language}/#{target_language}/#{content_hash(text)}" + end + + defp store_result({:ok, result}, cache_key) do + @cachex.put(:translations_cache, cache_key, result, ttl: @cache_ttl) + end + + defp store_result(_, _), do: nil + + defp content_hash(text), do: :crypto.hash(:sha256, text) |> Base.encode64() +end diff --git a/lib/pleroma/translation/deepl.ex b/lib/pleroma/translation/deepl.ex new file mode 100644 index 000000000..76fff4693 --- /dev/null +++ b/lib/pleroma/translation/deepl.ex @@ -0,0 +1,75 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Translation.DeepL do + import Pleroma.Web.Utils.Guards, only: [not_empty_string: 1] + + alias Pleroma.Translation.Service + + @behaviour Service + + @impl Service + def configured? do + not_empty_string(get_plan()) and not_empty_string(get_api_key()) + end + + @impl Service + def translate(content, source_language, target_language) do + endpoint = endpoint_url() + + case Pleroma.HTTP.post( + endpoint <> + "?" <> + URI.encode_query(%{ + text: content, + source_lang: source_language |> String.upcase(), + target_lang: target_language, + tag_handling: "html" + }), + "", + [ + {"Content-Type", "application/x-www-form-urlencoded"}, + {"Authorization", "DeepL-Auth-Key #{get_api_key()}"} + ] + ) do + {:ok, %{status: 429}} -> + {:error, :too_many_requests} + + {:ok, %{status: 456}} -> + {:error, :quota_exceeded} + + {:ok, %{status: 200} = res} -> + %{ + "translations" => [ + %{"text" => content, "detected_source_language" => detected_source_language} + ] + } = Jason.decode!(res.body) + + {:ok, + %{ + content: content, + detected_source_language: detected_source_language, + provider: "DeepL" + }} + + _ -> + {:error, :internal_server_error} + end + end + + defp endpoint_url do + case get_plan() do + "free" -> "https://api-free.deepl.com/v2/translate" + _ -> "https://api.deepl.com/v2/translate" + end + end + + defp get_plan do + Pleroma.Config.get([__MODULE__, :plan]) + end + + defp get_api_key do + Pleroma.Config.get([__MODULE__, :api_key]) + end +end diff --git a/lib/pleroma/translation/libretranslate.ex b/lib/pleroma/translation/libretranslate.ex new file mode 100644 index 000000000..049053d43 --- /dev/null +++ b/lib/pleroma/translation/libretranslate.ex @@ -0,0 +1,66 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Translation.LibreTranslate do + import Pleroma.Web.Utils.Guards, only: [not_empty_string: 1] + + alias Pleroma.Translation.Service + + @behaviour Service + + @impl Service + def configured?, do: not_empty_string(get_base_url()) + + @impl Service + def translate(content, source_language, target_language) do + endpoint = endpoint_url() + + case Pleroma.HTTP.post( + endpoint, + Jason.encode!(%{ + q: content, + source: source_language |> String.upcase(), + target: target_language, + format: "html", + api_key: get_api_key() + }), + [ + {"Content-Type", "application/json"} + ] + ) do + {:ok, %{status: 429}} -> + {:error, :too_many_requests} + + {:ok, %{status: 403}} -> + {:error, :quota_exceeded} + + {:ok, %{status: 200} = res} -> + %{ + "translatedText" => content + } = Jason.decode!(res.body) + + {:ok, + %{ + content: content, + detected_source_language: source_language, + provider: "LibreTranslate" + }} + + _ -> + {:error, :internal_server_error} + end + end + + defp endpoint_url do + get_base_url() <> "/translate" + end + + defp get_base_url do + Pleroma.Config.get([__MODULE__, :base_url]) + end + + defp get_api_key do + Pleroma.Config.get([__MODULE__, :api_key], "") + end +end diff --git a/lib/pleroma/translation/service.ex b/lib/pleroma/translation/service.ex new file mode 100644 index 000000000..55e995e92 --- /dev/null +++ b/lib/pleroma/translation/service.ex @@ -0,0 +1,20 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Translation.Service do + @callback configured?() :: boolean() + + @callback translate( + content :: String.t(), + source_language :: String.t(), + target_language :: String.t() + ) :: + {:ok, + %{ + content: String.t(), + detected_source_language: String.t(), + provider: String.t() + }} + | {:error, atom()} +end diff --git a/lib/pleroma/web/api_spec/operations/status_operation.ex b/lib/pleroma/web/api_spec/operations/status_operation.ex index 961a2f402..00529bc47 100644 --- a/lib/pleroma/web/api_spec/operations/status_operation.ex +++ b/lib/pleroma/web/api_spec/operations/status_operation.ex @@ -433,9 +433,9 @@ defmodule Pleroma.Web.ApiSpec.StatusOperation do required: false ), responses: %{ - 200 => Operation.response("Translation", "application/json", translation()) - 400 => Operation.response("Error", "application/json", ApiError) - 404 => Operation.response("Error", "application/json", ApiError) + 200 => Operation.response("Translation", "application/json", translation()), + 400 => Operation.response("Error", "application/json", ApiError), + 404 => Operation.response("Error", "application/json", ApiError), 503 => Operation.response("Error", "application/json", ApiError) } } diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index e4c6c81e1..f48e80fd6 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -202,7 +202,8 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do }, vapid: %{ public_key: Keyword.get(Pleroma.Web.Push.vapid_config(), :public_key) - } + }, + translation: %{enabled: Pleroma.Translation.configured?} }) end From 90f590788cffd154f9d2b40e5e644ad533883195 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Sun, 30 Oct 2022 21:06:31 +0100 Subject: [PATCH 011/128] Add tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/translation.ex | 3 +- .../web/mastodon_api/views/instance_view.ex | 7 +++- test/pleroma/translation_test.ex | 29 ++++++++++++++ .../controllers/status_controller_test.exs | 40 +++++++++++++++++++ test/support/translation_mock.ex | 22 ++++++++++ 5 files changed, 97 insertions(+), 4 deletions(-) create mode 100644 test/pleroma/translation_test.ex create mode 100644 test/support/translation_mock.ex diff --git a/lib/pleroma/translation.ex b/lib/pleroma/translation.ex index 112f12cab..7efec62a6 100644 --- a/lib/pleroma/translation.ex +++ b/lib/pleroma/translation.ex @@ -3,7 +3,6 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Translation do - @cache_ttl 86_400_000 @cachex Pleroma.Config.get([:cachex, :provider], Cachex) def configured? do @@ -45,7 +44,7 @@ defmodule Pleroma.Translation do end defp store_result({:ok, result}, cache_key) do - @cachex.put(:translations_cache, cache_key, result, ttl: @cache_ttl) + @cachex.put(:translations_cache, cache_key, result) end defp store_result(_, _), do: nil diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index f48e80fd6..63edd4b30 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -129,7 +129,10 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do "profile_directory" end, "pleroma:get:main/ostatus", - "pleroma:group_actors" + "pleroma:group_actors", + if Pleroma.Translation.configured?() do + "translation" + end ] |> Enum.filter(& &1) end @@ -203,7 +206,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do vapid: %{ public_key: Keyword.get(Pleroma.Web.Push.vapid_config(), :public_key) }, - translation: %{enabled: Pleroma.Translation.configured?} + translation: %{enabled: Pleroma.Translation.configured?()} }) end diff --git a/test/pleroma/translation_test.ex b/test/pleroma/translation_test.ex new file mode 100644 index 000000000..2ae7856ee --- /dev/null +++ b/test/pleroma/translation_test.ex @@ -0,0 +1,29 @@ +defmodule Pleroma.TranslationTest do + use Pleroma.Web.ConnCase + + alias Pleroma.Translation + # use Oban.Testing, repo: Pleroma.Repo + + setup do: clear_config([Pleroma.Translation, :service], TranslationMock) + + test "it translates text" do + assert {:ok, + %{ + content: "txet emos", + detected_source_language: _, + provider: _ + }} = Translation.translate("some text", "en", "uk") + end + + test "it stores translation result in cache" do + Translation.translate("some text", "en", "uk") + + assert {:ok, result} = + Cachex.get( + :translations_cache, + "en/uk/#{:crypto.hash(:sha256, "some text") |> Base.encode64()}" + ) + + assert result.content == "txet emos" + end +end diff --git a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs index f95f15ec3..fd2f3e11c 100644 --- a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs @@ -2550,4 +2550,44 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do |> json_response_and_validate_schema(:not_found) end end + + describe "translating statuses" do + setup do: clear_config([Pleroma.Translation, :service], TranslationMock) + + test "it translates a status to user language" do + user = insert(:user, language: "fr") + %{conn: conn, user: user} = oauth_access(["read:statuses"], user: user) + another_user = insert(:user) + + {:ok, activity} = + CommonAPI.post(another_user, %{ + status: "Cześć!", + visibility: "public", + language: "pl" + }) + + response = + conn + |> post("/api/v1/statuses/#{activity.id}/translate") + |> json_response_and_validate_schema(200) + + assert response == %{"content" => "!ćśezC", "detected_source_language" => "pl", "provider" => "TranslationMock"} + end + + test "it returns an error if no target language provided" do + %{conn: conn, user: user} = oauth_access(["read:statuses"]) + another_user = insert(:user) + + {:ok, activity} = + CommonAPI.post(another_user, %{ + status: "Cześć!", + language: "pl" + }) + + response = + conn + |> post("/api/v1/statuses/#{activity.id}/translate") + |> json_response_and_validate_schema(400) + end + end end diff --git a/test/support/translation_mock.ex b/test/support/translation_mock.ex new file mode 100644 index 000000000..8da2116e8 --- /dev/null +++ b/test/support/translation_mock.ex @@ -0,0 +1,22 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule TranslationMock do + alias Pleroma.Translation.Service + + @behaviour Service + + @impl Service + def configured?, do: true + + @impl Service + def translate(content, source_language, _target_language) do + {:ok, + %{ + content: content |> String.reverse(), + detected_source_language: source_language, + provider: "TranslationMock" + }} + end +end From aa429f6e6a059e58af2550d87f8272dea92acc9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Sun, 30 Oct 2022 21:57:05 +0100 Subject: [PATCH 012/128] Do not translate non-public statuses MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- .../controllers/status_controller.ex | 5 +++++ .../controllers/status_controller_test.exs | 22 ++++++++++++++++++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex index 4e1651596..239e15005 100644 --- a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex @@ -560,6 +560,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do @doc "POST /api/v1/statuses/:id/translate" def translate(%{body_params: params, assigns: %{user: user}} = conn, %{id: status_id}) do with %Activity{object: object} <- Activity.get_by_id_with_object(status_id), + {:visibility, visibility} when visibility in ["public", "unlisted"] <- + {:visibility, Visibility.get_visibility(object)}, {:language, language} when is_binary(language) <- {:language, Map.get(params, :target_language) || user.language}, {:ok, result} <- @@ -573,6 +575,9 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do {:language, nil} -> render_error(conn, :bad_request, "Language not specified") + {:visibility, _} -> + render_error(conn, :not_found, "Record not found") + {:error, :not_found} -> render_error(conn, :not_found, "Translation service not configured") diff --git a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs index fd2f3e11c..f81864b6c 100644 --- a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs @@ -2571,7 +2571,11 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do |> post("/api/v1/statuses/#{activity.id}/translate") |> json_response_and_validate_schema(200) - assert response == %{"content" => "!ćśezC", "detected_source_language" => "pl", "provider" => "TranslationMock"} + assert response == %{ + "content" => "!ćśezC", + "detected_source_language" => "pl", + "provider" => "TranslationMock" + } end test "it returns an error if no target language provided" do @@ -2589,5 +2593,21 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do |> post("/api/v1/statuses/#{activity.id}/translate") |> json_response_and_validate_schema(400) end + + test "it doesn't translate non-public statuses" do + %{conn: conn, user: user} = oauth_access(["read:statuses"]) + + {:ok, activity} = + CommonAPI.post(user, %{ + status: "Cześć!", + visibility: "private", + language: "pl" + }) + + response = + conn + |> post("/api/v1/statuses/#{activity.id}/translate") + |> json_response_and_validate_schema(404) + end end end From 066ec8fe955b9ff1e3cf15a76a8f2c4968015213 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Sun, 30 Oct 2022 22:57:20 +0100 Subject: [PATCH 013/128] Update description.exs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- config/description.exs | 34 +++++++++++++++++++++-- lib/pleroma/translation/deepl.ex | 4 +-- lib/pleroma/translation/libretranslate.ex | 2 +- 3 files changed, 34 insertions(+), 6 deletions(-) diff --git a/config/description.exs b/config/description.exs index 0efea0882..bdf2fc2f3 100644 --- a/config/description.exs +++ b/config/description.exs @@ -3531,9 +3531,37 @@ config :pleroma, :config_description, [ description: "Translation providers", children: [ %{ - key: Pleroma.Translation, - type: :service, - suggestions: [Pleroma.Translation.DeepL, Pleroma.Translation.LibreTranslate] + key: :service, + type: :module, + suggestions: [Pleroma.Translation.Deepl, Pleroma.Translation.Libretranslate] + }, + %{ + group: {:subgroup, Pleroma.Translation.Deepl}, + key: :plan, + label: "DeepL plan", + type: {:dropdown, :atom}, + suggestions: [:free, :pro] + }, + %{ + group: {:subgroup, Pleroma.Translation.Deepl}, + key: :api_key, + label: "DeepL API Key", + type: :string, + suggestions: ["YOUR_API_KEY"] + }, + %{ + group: {:subgroup, Pleroma.Translation.Libretranslate}, + key: :base_url, + label: "LibreTranslate plan", + type: :string, + suggestions: ["https://libretranslate.com"] + }, + %{ + group: {:subgroup, Pleroma.Translation.Libretranslate}, + key: :api_key, + label: "LibreTranslate API Key", + type: :string, + suggestions: ["YOUR_API_KEY"] } ] } diff --git a/lib/pleroma/translation/deepl.ex b/lib/pleroma/translation/deepl.ex index 76fff4693..944dab8ec 100644 --- a/lib/pleroma/translation/deepl.ex +++ b/lib/pleroma/translation/deepl.ex @@ -2,7 +2,7 @@ # Copyright © 2017-2022 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.Translation.DeepL do +defmodule Pleroma.Translation.Deepl do import Pleroma.Web.Utils.Guards, only: [not_empty_string: 1] alias Pleroma.Translation.Service @@ -60,7 +60,7 @@ defmodule Pleroma.Translation.DeepL do defp endpoint_url do case get_plan() do - "free" -> "https://api-free.deepl.com/v2/translate" + :free -> "https://api-free.deepl.com/v2/translate" _ -> "https://api.deepl.com/v2/translate" end end diff --git a/lib/pleroma/translation/libretranslate.ex b/lib/pleroma/translation/libretranslate.ex index 049053d43..9c9b4b9b5 100644 --- a/lib/pleroma/translation/libretranslate.ex +++ b/lib/pleroma/translation/libretranslate.ex @@ -2,7 +2,7 @@ # Copyright © 2017-2022 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.Translation.LibreTranslate do +defmodule Pleroma.Translation.Libretranslate do import Pleroma.Web.Utils.Guards, only: [not_empty_string: 1] alias Pleroma.Translation.Service From 2b739faa7edc69781eab85da4f122bad05d0576d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Mon, 31 Oct 2022 21:58:10 +0100 Subject: [PATCH 014/128] Rename MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- config/description.exs | 17 ++++++++++------- lib/pleroma/{ => language}/translation.ex | 4 ++-- lib/pleroma/{ => language}/translation/deepl.ex | 10 +++++----- .../translation/libretranslate.ex | 10 +++++----- .../translation/provider.ex} | 2 +- .../controllers/status_controller.ex | 2 +- .../web/mastodon_api/views/instance_view.ex | 4 ++-- test/pleroma/{ => language}/translation_test.ex | 6 +++--- .../controllers/status_controller_test.exs | 2 +- test/support/translation_mock.ex | 8 ++++---- 10 files changed, 34 insertions(+), 31 deletions(-) rename lib/pleroma/{ => language}/translation.ex (92%) rename lib/pleroma/{ => language}/translation/deepl.ex (92%) rename lib/pleroma/{ => language}/translation/libretranslate.ex (90%) rename lib/pleroma/{translation/service.ex => language/translation/provider.ex} (92%) rename test/pleroma/{ => language}/translation_test.ex (79%) diff --git a/config/description.exs b/config/description.exs index bdf2fc2f3..cc09e2991 100644 --- a/config/description.exs +++ b/config/description.exs @@ -3526,38 +3526,41 @@ config :pleroma, :config_description, [ }, %{ group: :pleroma, - key: Pleroma.Translation, + key: Pleroma.Language.Translation, type: :group, description: "Translation providers", children: [ %{ - key: :service, + key: :provider, type: :module, - suggestions: [Pleroma.Translation.Deepl, Pleroma.Translation.Libretranslate] + suggestions: [ + Pleroma.Language.Translation.Deepl, + Pleroma.Language.Translation.Libretranslate + ] }, %{ - group: {:subgroup, Pleroma.Translation.Deepl}, + group: {:subgroup, Pleroma.Language.Translation.Deepl}, key: :plan, label: "DeepL plan", type: {:dropdown, :atom}, suggestions: [:free, :pro] }, %{ - group: {:subgroup, Pleroma.Translation.Deepl}, + group: {:subgroup, Pleroma.Language.Translation.Deepl}, key: :api_key, label: "DeepL API Key", type: :string, suggestions: ["YOUR_API_KEY"] }, %{ - group: {:subgroup, Pleroma.Translation.Libretranslate}, + group: {:subgroup, Pleroma.Language.Translation.Libretranslate}, key: :base_url, label: "LibreTranslate plan", type: :string, suggestions: ["https://libretranslate.com"] }, %{ - group: {:subgroup, Pleroma.Translation.Libretranslate}, + group: {:subgroup, Pleroma.Language.Translation.Libretranslate}, key: :api_key, label: "LibreTranslate API Key", type: :string, diff --git a/lib/pleroma/translation.ex b/lib/pleroma/language/translation.ex similarity index 92% rename from lib/pleroma/translation.ex rename to lib/pleroma/language/translation.ex index 7efec62a6..c9cd9d2dd 100644 --- a/lib/pleroma/translation.ex +++ b/lib/pleroma/language/translation.ex @@ -2,7 +2,7 @@ # Copyright © 2017-2022 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.Translation do +defmodule Pleroma.Language.Translation do @cachex Pleroma.Config.get([:cachex, :provider], Cachex) def configured? do @@ -37,7 +37,7 @@ defmodule Pleroma.Translation do end end - defp get_service, do: Pleroma.Config.get([__MODULE__, :service]) + defp get_service, do: Pleroma.Config.get([__MODULE__, :provider]) defp get_cache_key(text, source_language, target_language) do "#{source_language}/#{target_language}/#{content_hash(text)}" diff --git a/lib/pleroma/translation/deepl.ex b/lib/pleroma/language/translation/deepl.ex similarity index 92% rename from lib/pleroma/translation/deepl.ex rename to lib/pleroma/language/translation/deepl.ex index 944dab8ec..81048378c 100644 --- a/lib/pleroma/translation/deepl.ex +++ b/lib/pleroma/language/translation/deepl.ex @@ -2,19 +2,19 @@ # Copyright © 2017-2022 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.Translation.Deepl do +defmodule Pleroma.Language.Translation.Deepl do import Pleroma.Web.Utils.Guards, only: [not_empty_string: 1] - alias Pleroma.Translation.Service + alias Pleroma.Language.Translation.Provider - @behaviour Service + @behaviour Provider - @impl Service + @impl Provider def configured? do not_empty_string(get_plan()) and not_empty_string(get_api_key()) end - @impl Service + @impl Provider def translate(content, source_language, target_language) do endpoint = endpoint_url() diff --git a/lib/pleroma/translation/libretranslate.ex b/lib/pleroma/language/translation/libretranslate.ex similarity index 90% rename from lib/pleroma/translation/libretranslate.ex rename to lib/pleroma/language/translation/libretranslate.ex index 9c9b4b9b5..0c1fe17a0 100644 --- a/lib/pleroma/translation/libretranslate.ex +++ b/lib/pleroma/language/translation/libretranslate.ex @@ -2,17 +2,17 @@ # Copyright © 2017-2022 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.Translation.Libretranslate do +defmodule Pleroma.Language.Translation.Libretranslate do import Pleroma.Web.Utils.Guards, only: [not_empty_string: 1] - alias Pleroma.Translation.Service + alias Pleroma.Language.Translation.Provider - @behaviour Service + @behaviour Provider - @impl Service + @impl Provider def configured?, do: not_empty_string(get_base_url()) - @impl Service + @impl Provider def translate(content, source_language, target_language) do endpoint = endpoint_url() diff --git a/lib/pleroma/translation/service.ex b/lib/pleroma/language/translation/provider.ex similarity index 92% rename from lib/pleroma/translation/service.ex rename to lib/pleroma/language/translation/provider.ex index 55e995e92..a88461a47 100644 --- a/lib/pleroma/translation/service.ex +++ b/lib/pleroma/language/translation/provider.ex @@ -2,7 +2,7 @@ # Copyright © 2017-2022 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.Translation.Service do +defmodule Pleroma.Language.Translation.Provider do @callback configured?() :: boolean() @callback translate( diff --git a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex index 239e15005..733f33e13 100644 --- a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex @@ -12,10 +12,10 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do alias Pleroma.Activity alias Pleroma.Bookmark + alias Pleroma.Language.Translation alias Pleroma.Object alias Pleroma.Repo alias Pleroma.ScheduledActivity - alias Pleroma.Translation alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Visibility diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index 63edd4b30..f98cf801f 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -130,7 +130,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do end, "pleroma:get:main/ostatus", "pleroma:group_actors", - if Pleroma.Translation.configured?() do + if Pleroma.Language.Translation.configured?() do "translation" end ] @@ -206,7 +206,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do vapid: %{ public_key: Keyword.get(Pleroma.Web.Push.vapid_config(), :public_key) }, - translation: %{enabled: Pleroma.Translation.configured?()} + translation: %{enabled: Pleroma.Language.Translation.configured?()} }) end diff --git a/test/pleroma/translation_test.ex b/test/pleroma/language/translation_test.ex similarity index 79% rename from test/pleroma/translation_test.ex rename to test/pleroma/language/translation_test.ex index 2ae7856ee..ecab3d20f 100644 --- a/test/pleroma/translation_test.ex +++ b/test/pleroma/language/translation_test.ex @@ -1,10 +1,10 @@ -defmodule Pleroma.TranslationTest do +defmodule Pleroma.Language.TranslationTest do use Pleroma.Web.ConnCase - alias Pleroma.Translation + alias Pleroma.Language.Translation # use Oban.Testing, repo: Pleroma.Repo - setup do: clear_config([Pleroma.Translation, :service], TranslationMock) + setup do: clear_config([Pleroma.Language.Translation, :provider], TranslationMock) test "it translates text" do assert {:ok, diff --git a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs index f81864b6c..f05f4191d 100644 --- a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs @@ -2552,7 +2552,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do end describe "translating statuses" do - setup do: clear_config([Pleroma.Translation, :service], TranslationMock) + setup do: clear_config([Pleroma.Language.Translation, :provider], TranslationMock) test "it translates a status to user language" do user = insert(:user, language: "fr") diff --git a/test/support/translation_mock.ex b/test/support/translation_mock.ex index 8da2116e8..7e618c263 100644 --- a/test/support/translation_mock.ex +++ b/test/support/translation_mock.ex @@ -3,14 +3,14 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule TranslationMock do - alias Pleroma.Translation.Service + alias Pleroma.Language.Translation.Provider - @behaviour Service + @behaviour Provider - @impl Service + @impl Provider def configured?, do: true - @impl Service + @impl Provider def translate(content, source_language, _target_language) do {:ok, %{ From fedae008c8b4a017b56a76c9a3b18bc031e520c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Sat, 5 Nov 2022 21:41:58 +0100 Subject: [PATCH 015/128] Deepl: use :base_url MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- config/description.exs | 10 +++---- lib/pleroma/language/translation/deepl.ex | 15 +++-------- .../language/translation/deepl_test.ex | 27 +++++++++++++++++++ 3 files changed, 36 insertions(+), 16 deletions(-) create mode 100644 test/pleroma/language/translation/deepl_test.ex diff --git a/config/description.exs b/config/description.exs index cc09e2991..b1730bab3 100644 --- a/config/description.exs +++ b/config/description.exs @@ -3540,10 +3540,10 @@ config :pleroma, :config_description, [ }, %{ group: {:subgroup, Pleroma.Language.Translation.Deepl}, - key: :plan, - label: "DeepL plan", - type: {:dropdown, :atom}, - suggestions: [:free, :pro] + key: :base_url, + label: "DeepL base URL", + type: :string, + suggestions: ["https://api-free.deepl.com", "https://api.deepl.com"] }, %{ group: {:subgroup, Pleroma.Language.Translation.Deepl}, @@ -3555,7 +3555,7 @@ config :pleroma, :config_description, [ %{ group: {:subgroup, Pleroma.Language.Translation.Libretranslate}, key: :base_url, - label: "LibreTranslate plan", + label: "LibreTranslate instance URL", type: :string, suggestions: ["https://libretranslate.com"] }, diff --git a/lib/pleroma/language/translation/deepl.ex b/lib/pleroma/language/translation/deepl.ex index 81048378c..5a3474090 100644 --- a/lib/pleroma/language/translation/deepl.ex +++ b/lib/pleroma/language/translation/deepl.ex @@ -11,12 +11,12 @@ defmodule Pleroma.Language.Translation.Deepl do @impl Provider def configured? do - not_empty_string(get_plan()) and not_empty_string(get_api_key()) + is_atom(get_base_url()) and not_empty_string(get_api_key()) end @impl Provider def translate(content, source_language, target_language) do - endpoint = endpoint_url() + endpoint = get_base_url() case Pleroma.HTTP.post( endpoint <> @@ -58,15 +58,8 @@ defmodule Pleroma.Language.Translation.Deepl do end end - defp endpoint_url do - case get_plan() do - :free -> "https://api-free.deepl.com/v2/translate" - _ -> "https://api.deepl.com/v2/translate" - end - end - - defp get_plan do - Pleroma.Config.get([__MODULE__, :plan]) + defp get_base_url do + Pleroma.Config.get([__MODULE__, :base_url]) end defp get_api_key do diff --git a/test/pleroma/language/translation/deepl_test.ex b/test/pleroma/language/translation/deepl_test.ex new file mode 100644 index 000000000..0c29b84a4 --- /dev/null +++ b/test/pleroma/language/translation/deepl_test.ex @@ -0,0 +1,27 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Language.Translation.DeeplTest do + use Pleroma.Web.ConnCase + + alias Pleroma.Language.Translation.Deepl + + test "it translates text" do + Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) + clear_config([Pleroma.Language.Translation.Deepl, :base_url], "https://api-free.deepl.com") + clear_config([Pleroma.Language.Translation.Deepl, :api_key], "API_KEY") + + {:ok, res} = + Deepl.translate( + "USUNĄĆ ŚLEDZIKA!Wklej to na swojego śledzika. Jeżeli uzbieramy 70% użytkowników nk...to usuną śledzika!!!", + "pl", + "en" + ) + + assert %{ + detected_source_language: "PL", + provider: "DeepL" + } = res + end +end From f0eb8e0b0c637104acaf95d8387a0d8f807e964f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Thu, 25 Apr 2024 23:50:11 +0200 Subject: [PATCH 016/128] Add tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- test/fixtures/tesla_mock/deepl-translation.json | 1 + test/support/http_request_mock.ex | 9 +++++++++ 2 files changed, 10 insertions(+) create mode 100644 test/fixtures/tesla_mock/deepl-translation.json diff --git a/test/fixtures/tesla_mock/deepl-translation.json b/test/fixtures/tesla_mock/deepl-translation.json new file mode 100644 index 000000000..fef7bb215 --- /dev/null +++ b/test/fixtures/tesla_mock/deepl-translation.json @@ -0,0 +1 @@ +{"translations":[{"detected_source_language":"PL","text":"REMOVE THE FOLLOWER!Paste this on your follower. If we get 70% of nk users...they will remove the follower!!!"}]} \ No newline at end of file diff --git a/test/support/http_request_mock.ex b/test/support/http_request_mock.ex index f4b6f1f9f..c5f29e0b8 100644 --- a/test/support/http_request_mock.ex +++ b/test/support/http_request_mock.ex @@ -1569,6 +1569,15 @@ defmodule HttpRequestMock do }} end + def post("https://api-free.deepl.com/v2/translate" <> _, _, _, _) do + {:ok, + %Tesla.Env{ + status: 200, + body: File.read!("test/fixtures/tesla_mock/deepl-translation.json"), + headers: [{"content-type", "application/json"}] + }} + end + def post(url, query, body, headers) do {:error, "Mock response not implemented for POST #{inspect(url)}, #{query}, #{inspect(body)}, #{inspect(headers)}"} From 28f8bb00d8d6770732a7985aa03d326e601d3694 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Tue, 8 Nov 2022 23:09:42 +0100 Subject: [PATCH 017/128] Add supported languages list to /api/v2/instance MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/language/translation.ex | 38 +++++++++++--- lib/pleroma/language/translation/deepl.ex | 49 ++++++++++++++++--- .../language/translation/libretranslate.ex | 32 ++++++++---- lib/pleroma/language/translation/provider.ex | 5 ++ .../web/mastodon_api/views/instance_view.ex | 24 ++++++++- .../tesla_mock/deepl-languages-list.json | 1 + .../language/translation/deepl_test.ex | 10 ++++ test/support/http_request_mock.ex | 9 ++++ test/support/translation_mock.ex | 12 ++++- 9 files changed, 155 insertions(+), 25 deletions(-) create mode 100644 test/fixtures/tesla_mock/deepl-languages-list.json diff --git a/lib/pleroma/language/translation.ex b/lib/pleroma/language/translation.ex index c9cd9d2dd..05ab898f3 100644 --- a/lib/pleroma/language/translation.ex +++ b/lib/pleroma/language/translation.ex @@ -6,9 +6,9 @@ defmodule Pleroma.Language.Translation do @cachex Pleroma.Config.get([:cachex, :provider], Cachex) def configured? do - service = get_service() + provider = get_provider() - !!service and service.configured? + !!provider and provider.configured? end def translate(text, source_language, target_language) do @@ -16,13 +16,13 @@ defmodule Pleroma.Language.Translation do case @cachex.get(:translations_cache, cache_key) do {:ok, nil} -> - service = get_service() + provider = get_provider() result = - if !service or !service.configured? do + if !configured?() do {:error, :not_found} else - service.translate(text, source_language, target_language) + provider.translate(text, source_language, target_language) end store_result(result, cache_key) @@ -37,7 +37,33 @@ defmodule Pleroma.Language.Translation do end end - defp get_service, do: Pleroma.Config.get([__MODULE__, :provider]) + def supported_languages(type) when type in [:source, :target] do + provider = get_provider() + + cache_key = "#{type}_languages/#{provider.name()}" + + case @cachex.get(:translations_cache, cache_key) do + {:ok, nil} -> + result = + if !configured?() do + {:error, :not_found} + else + provider.supported_languages(type) + end + + store_result(result, cache_key) + + result + + {:ok, result} -> + {:ok, result} + + {:error, error} -> + {:error, error} + end + end + + defp get_provider, do: Pleroma.Config.get([__MODULE__, :provider]) defp get_cache_key(text, source_language, target_language) do "#{source_language}/#{target_language}/#{content_hash(text)}" diff --git a/lib/pleroma/language/translation/deepl.ex b/lib/pleroma/language/translation/deepl.ex index 5a3474090..8ce1209cc 100644 --- a/lib/pleroma/language/translation/deepl.ex +++ b/lib/pleroma/language/translation/deepl.ex @@ -9,14 +9,17 @@ defmodule Pleroma.Language.Translation.Deepl do @behaviour Provider + @name "DeepL" + @impl Provider - def configured? do - is_atom(get_base_url()) and not_empty_string(get_api_key()) - end + def configured?, do: not_empty_string(base_url()) and not_empty_string(api_key()) @impl Provider def translate(content, source_language, target_language) do - endpoint = get_base_url() + endpoint = + base_url() + |> URI.merge("/v2/translate") + |> URI.to_string() case Pleroma.HTTP.post( endpoint <> @@ -30,7 +33,7 @@ defmodule Pleroma.Language.Translation.Deepl do "", [ {"Content-Type", "application/x-www-form-urlencoded"}, - {"Authorization", "DeepL-Auth-Key #{get_api_key()}"} + {"Authorization", "DeepL-Auth-Key #{api_key()}"} ] ) do {:ok, %{status: 429}} -> @@ -50,7 +53,7 @@ defmodule Pleroma.Language.Translation.Deepl do %{ content: content, detected_source_language: detected_source_language, - provider: "DeepL" + provider: @name }} _ -> @@ -58,11 +61,41 @@ defmodule Pleroma.Language.Translation.Deepl do end end - defp get_base_url do + @impl Provider + def supported_languages(type) when type in [:source, :target] do + endpoint = + base_url() + |> URI.merge("/v2/languages") + |> URI.to_string() + + case Pleroma.HTTP.post( + endpoint <> "?" <> URI.encode_query(%{type: type}), + "", + [ + {"Content-Type", "application/x-www-form-urlencoded"}, + {"Authorization", "DeepL-Auth-Key #{api_key()}"} + ] + ) do + {:ok, %{status: 200} = res} -> + languages = + Jason.decode!(res.body) + |> Enum.map(fn %{"language" => language} -> language |> String.downcase() end) + + {:ok, languages} + + _ -> + {:error, :internal_server_error} + end + end + + @impl Provider + def name, do: @name + + defp base_url do Pleroma.Config.get([__MODULE__, :base_url]) end - defp get_api_key do + defp api_key do Pleroma.Config.get([__MODULE__, :api_key]) end end diff --git a/lib/pleroma/language/translation/libretranslate.ex b/lib/pleroma/language/translation/libretranslate.ex index 0c1fe17a0..92bde8772 100644 --- a/lib/pleroma/language/translation/libretranslate.ex +++ b/lib/pleroma/language/translation/libretranslate.ex @@ -9,21 +9,21 @@ defmodule Pleroma.Language.Translation.Libretranslate do @behaviour Provider + @name "LibreTranslate" + @impl Provider - def configured?, do: not_empty_string(get_base_url()) + def configured?, do: not_empty_string(base_url()) and not_empty_string(api_key()) @impl Provider def translate(content, source_language, target_language) do - endpoint = endpoint_url() - case Pleroma.HTTP.post( - endpoint, + base_url() <> "/translate", Jason.encode!(%{ q: content, source: source_language |> String.upcase(), target: target_language, format: "html", - api_key: get_api_key() + api_key: api_key() }), [ {"Content-Type", "application/json"} @@ -52,15 +52,29 @@ defmodule Pleroma.Language.Translation.Libretranslate do end end - defp endpoint_url do - get_base_url() <> "/translate" + @impl Provider + def supported_languages(_) do + case Pleroma.HTTP.get(base_url() <> "/languages") do + {:ok, %{status: 200} = res} -> + languages = + Jason.decode!(res.body) + |> Enum.map(fn %{"code" => code} -> code end) + + {:ok, languages} + + _ -> + {:error, :internal_server_error} + end end - defp get_base_url do + @impl Provider + def name, do: @name + + defp base_url do Pleroma.Config.get([__MODULE__, :base_url]) end - defp get_api_key do + defp api_key do Pleroma.Config.get([__MODULE__, :api_key], "") end end diff --git a/lib/pleroma/language/translation/provider.ex b/lib/pleroma/language/translation/provider.ex index a88461a47..a8b151fd7 100644 --- a/lib/pleroma/language/translation/provider.ex +++ b/lib/pleroma/language/translation/provider.ex @@ -17,4 +17,9 @@ defmodule Pleroma.Language.Translation.Provider do provider: String.t() }} | {:error, atom()} + + @callback supported_languages(type :: :string | :target) :: + {:ok, [String.t()]} | {:error, atom()} + + @callback name() :: String.t() end diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index f98cf801f..8c2462c80 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -206,10 +206,32 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do vapid: %{ public_key: Keyword.get(Pleroma.Web.Push.vapid_config(), :public_key) }, - translation: %{enabled: Pleroma.Language.Translation.configured?()} + translation: translation_config() }) end + defp translation_config do + enabled = Pleroma.Language.Translation.configured?() + + source_languages = + case Pleroma.Language.Translation.supported_languages(:source) do + {:ok, languages} -> languages + _ -> nil + end + + target_languages = + case Pleroma.Language.Translation.supported_languages(:target) do + {:ok, languages} -> languages + _ -> nil + end + + %{ + enabled: enabled, + source_languages: source_languages, + target_languages: target_languages + } + end + defp pleroma_configuration(instance) do %{ metadata: %{ diff --git a/test/fixtures/tesla_mock/deepl-languages-list.json b/test/fixtures/tesla_mock/deepl-languages-list.json new file mode 100644 index 000000000..03d47d2ec --- /dev/null +++ b/test/fixtures/tesla_mock/deepl-languages-list.json @@ -0,0 +1 @@ +[{"language":"BG","name":"Bulgarian","supports_formality":false},{"language":"CS","name":"Czech","supports_formality":false},{"language":"DA","name":"Danish","supports_formality":false},{"language":"DE","name":"German","supports_formality":true},{"language":"EL","name":"Greek","supports_formality":false},{"language":"EN-GB","name":"English (British)","supports_formality":false},{"language":"EN-US","name":"English (American)","supports_formality":false},{"language":"ES","name":"Spanish","supports_formality":true},{"language":"ET","name":"Estonian","supports_formality":false},{"language":"FI","name":"Finnish","supports_formality":false},{"language":"FR","name":"French","supports_formality":true},{"language":"HU","name":"Hungarian","supports_formality":false},{"language":"ID","name":"Indonesian","supports_formality":false},{"language":"IT","name":"Italian","supports_formality":true},{"language":"JA","name":"Japanese","supports_formality":false},{"language":"LT","name":"Lithuanian","supports_formality":false},{"language":"LV","name":"Latvian","supports_formality":false},{"language":"NL","name":"Dutch","supports_formality":true},{"language":"PL","name":"Polish","supports_formality":true},{"language":"PT-BR","name":"Portuguese (Brazilian)","supports_formality":true},{"language":"PT-PT","name":"Portuguese (European)","supports_formality":true},{"language":"RO","name":"Romanian","supports_formality":false},{"language":"RU","name":"Russian","supports_formality":true},{"language":"SK","name":"Slovak","supports_formality":false},{"language":"SL","name":"Slovenian","supports_formality":false},{"language":"SV","name":"Swedish","supports_formality":false},{"language":"TR","name":"Turkish","supports_formality":false},{"language":"UK","name":"Ukrainian","supports_formality":false},{"language":"ZH","name":"Chinese (simplified)","supports_formality":false}] \ No newline at end of file diff --git a/test/pleroma/language/translation/deepl_test.ex b/test/pleroma/language/translation/deepl_test.ex index 0c29b84a4..3a7265622 100644 --- a/test/pleroma/language/translation/deepl_test.ex +++ b/test/pleroma/language/translation/deepl_test.ex @@ -24,4 +24,14 @@ defmodule Pleroma.Language.Translation.DeeplTest do provider: "DeepL" } = res end + + test "it returns languages list" do + Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) + clear_config([Pleroma.Language.Translation.Deepl, :base_url], "https://api-free.deepl.com") + clear_config([Pleroma.Language.Translation.Deepl, :api_key], "API_KEY") + + assert {:ok, [language | _languages]} = Deepl.supported_languages(:target) + + assert is_binary(language) + end end diff --git a/test/support/http_request_mock.ex b/test/support/http_request_mock.ex index c5f29e0b8..771336b6f 100644 --- a/test/support/http_request_mock.ex +++ b/test/support/http_request_mock.ex @@ -1578,6 +1578,15 @@ defmodule HttpRequestMock do }} end + def post("https://api-free.deepl.com/v2/languages" <> _, _, _, _) do + {:ok, + %Tesla.Env{ + status: 200, + body: File.read!("test/fixtures/tesla_mock/deepl-languages-list.json"), + headers: [{"content-type", "application/json"}] + }} + end + def post(url, query, body, headers) do {:error, "Mock response not implemented for POST #{inspect(url)}, #{query}, #{inspect(body)}, #{inspect(headers)}"} diff --git a/test/support/translation_mock.ex b/test/support/translation_mock.ex index 7e618c263..2047d6426 100644 --- a/test/support/translation_mock.ex +++ b/test/support/translation_mock.ex @@ -7,6 +7,8 @@ defmodule TranslationMock do @behaviour Provider + @name "TranslationMock" + @impl Provider def configured?, do: true @@ -16,7 +18,15 @@ defmodule TranslationMock do %{ content: content |> String.reverse(), detected_source_language: source_language, - provider: "TranslationMock" + provider: @name }} end + + @impl Provider + def supported_languages(_) do + ["en", "pl"] + end + + @impl Provider + def name, do: @name end From 4696487f1f34f76735a24df628c8c15f3ba5ecfa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Thu, 10 Nov 2022 22:15:49 +0100 Subject: [PATCH 018/128] Fix instance view MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/web/mastodon_api/views/instance_view.ex | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index 8c2462c80..f81d33f8d 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -214,14 +214,18 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do enabled = Pleroma.Language.Translation.configured?() source_languages = - case Pleroma.Language.Translation.supported_languages(:source) do - {:ok, languages} -> languages + with true <- enabled, + {:ok, languages} <- Pleroma.Language.Translation.supported_languages(:source) do + languages + else _ -> nil end target_languages = - case Pleroma.Language.Translation.supported_languages(:target) do - {:ok, languages} -> languages + with true <- enabled, + {:ok, languages} <- Pleroma.Language.Translation.supported_languages(:target) do + languages + else _ -> nil end From 7fca35f4fd9ca396761f236119936978b63120a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Sat, 10 Dec 2022 21:21:38 +0100 Subject: [PATCH 019/128] InstanceView: Move supported languages to pleroma.metadata MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- .../web/mastodon_api/views/instance_view.ex | 56 +++++++++---------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index f81d33f8d..b6b99c477 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -206,36 +206,10 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do vapid: %{ public_key: Keyword.get(Pleroma.Web.Push.vapid_config(), :public_key) }, - translation: translation_config() + translation: %{enabled: Pleroma.Language.Translation.configured?()} }) end - defp translation_config do - enabled = Pleroma.Language.Translation.configured?() - - source_languages = - with true <- enabled, - {:ok, languages} <- Pleroma.Language.Translation.supported_languages(:source) do - languages - else - _ -> nil - end - - target_languages = - with true <- enabled, - {:ok, languages} <- Pleroma.Language.Translation.supported_languages(:target) do - languages - else - _ -> nil - end - - %{ - enabled: enabled, - source_languages: source_languages, - target_languages: target_languages - } - end - defp pleroma_configuration(instance) do %{ metadata: %{ @@ -245,7 +219,8 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do fields_limits: fields_limits(), post_formats: Config.get([:instance, :allowed_post_formats]), birthday_required: Config.get([:instance, :birthday_required]), - birthday_min_age: Config.get([:instance, :birthday_min_age]) + birthday_min_age: Config.get([:instance, :birthday_min_age]), + translation: supported_languages() }, stats: %{mau: Pleroma.User.active_user_count()}, vapid_public_key: Keyword.get(Pleroma.Web.Push.vapid_config(), :public_key) @@ -271,4 +246,29 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do }) }) end + + defp supported_languages do + enabled = Pleroma.Language.Translation.configured?() + + source_languages = + with true <- enabled, + {:ok, languages} <- Pleroma.Language.Translation.supported_languages(:source) do + languages + else + _ -> nil + end + + target_languages = + with true <- enabled, + {:ok, languages} <- Pleroma.Language.Translation.supported_languages(:target) do + languages + else + _ -> nil + end + + %{ + source_languages: source_languages, + target_languages: target_languages + } + end end From 010c23e729e2d643938e6a8a55cd57ee2b5b3d88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Wed, 14 Dec 2022 18:21:43 +0100 Subject: [PATCH 020/128] Include unspecified variants in target languages list for DeepL MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/language/translation/deepl.ex | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lib/pleroma/language/translation/deepl.ex b/lib/pleroma/language/translation/deepl.ex index 8ce1209cc..0b56b35d7 100644 --- a/lib/pleroma/language/translation/deepl.ex +++ b/lib/pleroma/language/translation/deepl.ex @@ -80,6 +80,15 @@ defmodule Pleroma.Language.Translation.Deepl do languages = Jason.decode!(res.body) |> Enum.map(fn %{"language" => language} -> language |> String.downcase() end) + |> Enum.map(fn language -> + if String.contains?(language, "-") do + [language, language |> String.split("-") |> Enum.at(0)] + else + language + end + end) + |> List.flatten() + |> Enum.uniq() {:ok, languages} From f954f98fb7a1dd2503f5929de7589d43df9d5c06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Sat, 13 May 2023 13:22:04 +0200 Subject: [PATCH 021/128] Implement /api/v1/instance/translation_languages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/language/translation.ex | 26 +++++++++++++++++++ lib/pleroma/language/translation/deepl.ex | 11 ++++++++ .../language/translation/libretranslate.ex | 11 ++++++++ lib/pleroma/language/translation/provider.ex | 2 ++ .../api_spec/operations/instance_operation.ex | 23 ++++++++++++++++ .../controllers/instance_controller.ex | 5 ++++ .../web/mastodon_api/views/instance_view.ex | 9 +++++++ lib/pleroma/web/router.ex | 1 + .../controllers/instance_controller_test.exs | 9 +++++++ test/support/translation_mock.ex | 11 +++++++- 10 files changed, 107 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/language/translation.ex b/lib/pleroma/language/translation.ex index 05ab898f3..e4916389d 100644 --- a/lib/pleroma/language/translation.ex +++ b/lib/pleroma/language/translation.ex @@ -63,6 +63,32 @@ defmodule Pleroma.Language.Translation do end end + def languages_matrix do + provider = get_provider() + + cache_key = "languages_matrix/#{provider.name()}" + + case @cachex.get(:translations_cache, cache_key) do + {:ok, nil} -> + result = + if !configured?() do + {:error, :not_found} + else + provider.languages_matrix() + end + + store_result(result, cache_key) + + result + + {:ok, result} -> + {:ok, result} + + {:error, error} -> + {:error, error} + end + end + defp get_provider, do: Pleroma.Config.get([__MODULE__, :provider]) defp get_cache_key(text, source_language, target_language) do diff --git a/lib/pleroma/language/translation/deepl.ex b/lib/pleroma/language/translation/deepl.ex index 0b56b35d7..4f668fbba 100644 --- a/lib/pleroma/language/translation/deepl.ex +++ b/lib/pleroma/language/translation/deepl.ex @@ -97,6 +97,17 @@ defmodule Pleroma.Language.Translation.Deepl do end end + @impl Provider + def languages_matrix do + with {:ok, source_languages} <- supported_languages(:source), + {:ok, target_languages} <- supported_languages(:target) do + {:ok, + Map.new(source_languages, fn language -> {language, target_languages -- [language]} end)} + else + {:error, error} -> {:error, error} + end + end + @impl Provider def name, do: @name diff --git a/lib/pleroma/language/translation/libretranslate.ex b/lib/pleroma/language/translation/libretranslate.ex index 92bde8772..b793b166e 100644 --- a/lib/pleroma/language/translation/libretranslate.ex +++ b/lib/pleroma/language/translation/libretranslate.ex @@ -67,6 +67,17 @@ defmodule Pleroma.Language.Translation.Libretranslate do end end + @impl Provider + def languages_matrix do + with {:ok, source_languages} <- supported_languages(:source), + {:ok, target_languages} <- supported_languages(:target) do + {:ok, + Map.new(source_languages, fn language -> {language, target_languages -- [language]} end)} + else + {:error, error} -> {:error, error} + end + end + @impl Provider def name, do: @name diff --git a/lib/pleroma/language/translation/provider.ex b/lib/pleroma/language/translation/provider.ex index a8b151fd7..f12cba2cd 100644 --- a/lib/pleroma/language/translation/provider.ex +++ b/lib/pleroma/language/translation/provider.ex @@ -21,5 +21,7 @@ defmodule Pleroma.Language.Translation.Provider do @callback supported_languages(type :: :string | :target) :: {:ok, [String.t()]} | {:error, atom()} + @callback languages_matrix() :: {:ok, Map.t()} | {:error, atom()} + @callback name() :: String.t() end diff --git a/lib/pleroma/web/api_spec/operations/instance_operation.ex b/lib/pleroma/web/api_spec/operations/instance_operation.ex index 708b74b12..30f4c2a97 100644 --- a/lib/pleroma/web/api_spec/operations/instance_operation.ex +++ b/lib/pleroma/web/api_spec/operations/instance_operation.ex @@ -46,6 +46,29 @@ defmodule Pleroma.Web.ApiSpec.InstanceOperation do } end + def translation_languages_operation do + %Operation{ + tags: ["Instance misc"], + summary: "Retrieve supported languages matrix", + operationId: "InstanceController.translation_languages", + responses: %{ + 200 => + Operation.response( + "Translation languages matrix", + "application/json", + %Schema{ + type: :object, + additionalProperties: %Schema{ + type: :array, + items: %Schema{type: :string}, + description: "Supported target languages for a source language" + } + } + ) + } + } + end + defp instance do %Schema{ type: :object, diff --git a/lib/pleroma/web/mastodon_api/controllers/instance_controller.ex b/lib/pleroma/web/mastodon_api/controllers/instance_controller.ex index 3e664903a..bce64cf6e 100644 --- a/lib/pleroma/web/mastodon_api/controllers/instance_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/instance_controller.ex @@ -24,5 +24,10 @@ defmodule Pleroma.Web.MastodonAPI.InstanceController do @doc "GET /api/v1/instance/peers" def peers(conn, _params) do json(conn, Pleroma.Stats.get_peers()) +end + + @doc "GET /api/v1/instance/translation_languages" + def translation_languages(conn, _params) do + render(conn, "translation_languages.json") end end diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index b6b99c477..096ed00ad 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -84,6 +84,15 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do } end + def render("translation_languages.json", _) do + with true <- Pleroma.Language.Translation.configured?(), + {:ok, languages} <- Pleroma.Language.Translation.languages_matrix() do + languages + else + _ -> %{} + end + end + def features do [ "pleroma_api", diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 5c949fc95..fcf6edfd7 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -760,6 +760,7 @@ defmodule Pleroma.Web.Router do get("/instance", InstanceController, :show) get("/instance/peers", InstanceController, :peers) + get("/instance/translation_languages", InstanceController, :translation_languages) get("/statuses", StatusController, :index) get("/statuses/:id", StatusController, :show) diff --git a/test/pleroma/web/mastodon_api/controllers/instance_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/instance_controller_test.exs index 2243b0d4a..8ffcff9f3 100644 --- a/test/pleroma/web/mastodon_api/controllers/instance_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/instance_controller_test.exs @@ -113,4 +113,13 @@ defmodule Pleroma.Web.MastodonAPI.InstanceControllerTest do assert get(conn, "/api/v2/instance") |> json_response_and_validate_schema(200) end + + test "translation languages matrix", %{conn: conn} do + clear_config([Pleroma.Language.Translation, :provider], TranslationMock) + + assert %{"en" => ["pl"], "pl" => ["en"]} = + conn + |> get("/api/v1/instance/translation_languages") + |> json_response_and_validate_schema(200) + end end diff --git a/test/support/translation_mock.ex b/test/support/translation_mock.ex index 2047d6426..95da738d1 100644 --- a/test/support/translation_mock.ex +++ b/test/support/translation_mock.ex @@ -24,7 +24,16 @@ defmodule TranslationMock do @impl Provider def supported_languages(_) do - ["en", "pl"] + {:ok, ["en", "pl"]} + end + + @impl Provider + def languages_matrix do + {:ok, + %{ + "en" => ["pl"], + "pl" => ["en"] + }} end @impl Provider From b53abd9d79eff1d7a650954a9585a145105b6e25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Fri, 26 Apr 2024 00:00:30 +0200 Subject: [PATCH 022/128] changelog MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- changelog.d/translate-posts.add | 1 + .../controllers/instance_controller.ex | 2 +- .../controllers/status_controller.ex | 8 +++++++- .../web/mastodon_api/views/instance_view.ex | 18 +++++++++--------- .../controllers/status_controller_test.exs | 18 ++++++++---------- 5 files changed, 26 insertions(+), 21 deletions(-) create mode 100644 changelog.d/translate-posts.add diff --git a/changelog.d/translate-posts.add b/changelog.d/translate-posts.add new file mode 100644 index 000000000..e7a9317a1 --- /dev/null +++ b/changelog.d/translate-posts.add @@ -0,0 +1 @@ +Support translation providers (DeepL, LibreTranslate) \ No newline at end of file diff --git a/lib/pleroma/web/mastodon_api/controllers/instance_controller.ex b/lib/pleroma/web/mastodon_api/controllers/instance_controller.ex index bce64cf6e..ed015d574 100644 --- a/lib/pleroma/web/mastodon_api/controllers/instance_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/instance_controller.ex @@ -24,7 +24,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceController do @doc "GET /api/v1/instance/peers" def peers(conn, _params) do json(conn, Pleroma.Stats.get_peers()) -end + end @doc "GET /api/v1/instance/translation_languages" def translation_languages(conn, _params) do diff --git a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex index 733f33e13..ad1e78c30 100644 --- a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex @@ -558,7 +558,13 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do end @doc "POST /api/v1/statuses/:id/translate" - def translate(%{body_params: params, assigns: %{user: user}} = conn, %{id: status_id}) do + def translate( + %{ + assigns: %{user: user}, + private: %{open_api_spex: %{body_params: params, params: %{id: status_id}}} + } = conn, + _ + ) do with %Activity{object: object} <- Activity.get_by_id_with_object(status_id), {:visibility, visibility} when visibility in ["public", "unlisted"] <- {:visibility, Visibility.get_visibility(object)}, diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index 096ed00ad..e093b1a2f 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -75,15 +75,6 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do }) end - defp common_information(instance) do - %{ - title: Keyword.get(instance, :name), - version: "#{@mastodon_api_level} (compatible; #{Pleroma.Application.named_version()})", - languages: Keyword.get(instance, :languages, ["en"]), - rules: [] - } - end - def render("translation_languages.json", _) do with true <- Pleroma.Language.Translation.configured?(), {:ok, languages} <- Pleroma.Language.Translation.languages_matrix() do @@ -93,6 +84,15 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do end end + defp common_information(instance) do + %{ + title: Keyword.get(instance, :name), + version: "#{@mastodon_api_level} (compatible; #{Pleroma.Application.named_version()})", + languages: Keyword.get(instance, :languages, ["en"]), + rules: [] + } + end + def features do [ "pleroma_api", diff --git a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs index f05f4191d..2a64cac5f 100644 --- a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs @@ -2556,7 +2556,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do test "it translates a status to user language" do user = insert(:user, language: "fr") - %{conn: conn, user: user} = oauth_access(["read:statuses"], user: user) + %{conn: conn} = oauth_access(["read:statuses"], user: user) another_user = insert(:user) {:ok, activity} = @@ -2579,7 +2579,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do end test "it returns an error if no target language provided" do - %{conn: conn, user: user} = oauth_access(["read:statuses"]) + %{conn: conn} = oauth_access(["read:statuses"]) another_user = insert(:user) {:ok, activity} = @@ -2588,10 +2588,9 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do language: "pl" }) - response = - conn - |> post("/api/v1/statuses/#{activity.id}/translate") - |> json_response_and_validate_schema(400) + assert conn + |> post("/api/v1/statuses/#{activity.id}/translate") + |> json_response_and_validate_schema(400) end test "it doesn't translate non-public statuses" do @@ -2604,10 +2603,9 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do language: "pl" }) - response = - conn - |> post("/api/v1/statuses/#{activity.id}/translate") - |> json_response_and_validate_schema(404) + assert conn + |> post("/api/v1/statuses/#{activity.id}/translate") + |> json_response_and_validate_schema(404) end end end From b430093caba7cf8cc8c4b9b9f1885146cd49a10f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Fri, 2 Aug 2024 09:41:48 +0200 Subject: [PATCH 023/128] Translation: Rename target language param MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/web/api_spec/operations/status_operation.ex | 2 +- lib/pleroma/web/mastodon_api/controllers/status_controller.ex | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/api_spec/operations/status_operation.ex b/lib/pleroma/web/api_spec/operations/status_operation.ex index d87d59ef6..b2dc606e1 100644 --- a/lib/pleroma/web/api_spec/operations/status_operation.ex +++ b/lib/pleroma/web/api_spec/operations/status_operation.ex @@ -435,7 +435,7 @@ defmodule Pleroma.Web.ApiSpec.StatusOperation do %Schema{ type: :object, properties: %{ - target_language: %Schema{ + lang: %Schema{ type: :string, nullable: true, description: "Translation target language." diff --git a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex index 9d40e0c30..cec35d7e2 100644 --- a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex @@ -563,7 +563,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do {:visibility, visibility} when visibility in ["public", "unlisted"] <- {:visibility, Visibility.get_visibility(object)}, {:language, language} when is_binary(language) <- - {:language, Map.get(params, :target_language) || user.language}, + {:language, Map.get(params, :lang) || user.language}, {:ok, result} <- Translation.translate( object.data["content"], From 9d9bc74e9187d423aea6745d6b6f8e1b38bf24a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Tue, 27 Aug 2024 23:30:47 +0200 Subject: [PATCH 024/128] Expose language detection in features MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/web/mastodon_api/views/instance_view.ex | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index 99fc6d0c3..8cd862e2a 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -145,7 +145,10 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do end, "pleroma:get:main/ostatus", "pleroma:group_actors", - "pleroma:bookmark_folders" + "pleroma:bookmark_folders", + if Config.get([Pleroma.Language.LanguageDetector, :provider]) do + "pleroma:language_detection" + end ] |> Enum.filter(& &1) end From e35e84228db9dc29906647c1d30dd90749f6cc2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Sun, 1 Sep 2024 11:26:01 +0200 Subject: [PATCH 025/128] Change scrobble external link param name to use snake case MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- changelog.d/scrobbles.change | 1 + docs/development/API/pleroma_api.md | 1 + .../operations/pleroma_scrobble_operation.ex | 12 ++++--- lib/pleroma/web/common_api/activity_draft.ex | 3 +- .../controllers/scrobble_controller.ex | 4 +++ .../web/pleroma_api/views/scrobble_view.ex | 6 ++-- .../controllers/scrobble_controller_test.exs | 33 ++++++++++++++++--- 7 files changed, 48 insertions(+), 12 deletions(-) create mode 100644 changelog.d/scrobbles.change diff --git a/changelog.d/scrobbles.change b/changelog.d/scrobbles.change new file mode 100644 index 000000000..ed1777b2d --- /dev/null +++ b/changelog.d/scrobbles.change @@ -0,0 +1 @@ +Change scrobble external link param name to use snake case \ No newline at end of file diff --git a/docs/development/API/pleroma_api.md b/docs/development/API/pleroma_api.md index 000d7d27d..b17f61cbb 100644 --- a/docs/development/API/pleroma_api.md +++ b/docs/development/API/pleroma_api.md @@ -671,6 +671,7 @@ Audio scrobbling in Pleroma is **deprecated**. "artist": "Some Artist", "album": "Some Album", "length": 180000, + "external_link": "https://www.last.fm/music/Some+Artist/_/Some+Title", "created_at": "2019-09-28T12:40:45.000Z" } ] diff --git a/lib/pleroma/web/api_spec/operations/pleroma_scrobble_operation.ex b/lib/pleroma/web/api_spec/operations/pleroma_scrobble_operation.ex index f595583b6..6f77584a8 100644 --- a/lib/pleroma/web/api_spec/operations/pleroma_scrobble_operation.ex +++ b/lib/pleroma/web/api_spec/operations/pleroma_scrobble_operation.ex @@ -59,11 +59,15 @@ defmodule Pleroma.Web.ApiSpec.PleromaScrobbleOperation do album: %Schema{type: :string, description: "The album of the media playing"}, artist: %Schema{type: :string, description: "The artist of the media playing"}, length: %Schema{type: :integer, description: "The length of the media playing"}, - externalLink: %Schema{type: :string, description: "A URL referencing the media playing"}, + external_link: %Schema{type: :string, description: "A URL referencing the media playing"}, visibility: %Schema{ allOf: [VisibilityScope], default: "public", description: "Scrobble visibility" + }, + externalLink: %Schema{ + type: :string, + description: "Deprecated, use `external_link` instead" } }, example: %{ @@ -71,7 +75,7 @@ defmodule Pleroma.Web.ApiSpec.PleromaScrobbleOperation do "artist" => "Some Artist", "album" => "Some Album", "length" => 180_000, - "externalLink" => "https://www.last.fm/music/Some+Artist/_/Some+Title" + "external_link" => "https://www.last.fm/music/Some+Artist/_/Some+Title" } } end @@ -85,7 +89,7 @@ defmodule Pleroma.Web.ApiSpec.PleromaScrobbleOperation do title: %Schema{type: :string, description: "The title of the media playing"}, album: %Schema{type: :string, description: "The album of the media playing"}, artist: %Schema{type: :string, description: "The artist of the media playing"}, - externalLink: %Schema{type: :string, description: "A URL referencing the media playing"}, + external_link: %Schema{type: :string, description: "A URL referencing the media playing"}, length: %Schema{ type: :integer, description: "The length of the media playing", @@ -100,7 +104,7 @@ defmodule Pleroma.Web.ApiSpec.PleromaScrobbleOperation do "artist" => "Some Artist", "album" => "Some Album", "length" => 180_000, - "externalLink" => "https://www.last.fm/music/Some+Artist/_/Some+Title", + "external_link" => "https://www.last.fm/music/Some+Artist/_/Some+Title", "created_at" => "2019-09-28T12:40:45.000Z" } } diff --git a/lib/pleroma/web/common_api/activity_draft.ex b/lib/pleroma/web/common_api/activity_draft.ex index 8aa1e258d..0268d3f48 100644 --- a/lib/pleroma/web/common_api/activity_draft.ex +++ b/lib/pleroma/web/common_api/activity_draft.ex @@ -85,7 +85,8 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do defp listen_object(draft) do object = draft.params - |> Map.take([:album, :artist, :title, :length, :externalLink]) + |> Map.take([:album, :artist, :title, :length]) + |> Map.put(:externalLink, Map.get(draft.params, :external_link)) |> Map.new(fn {key, value} -> {to_string(key), value} end) |> Map.put("type", "Audio") |> Map.put("to", draft.to) diff --git a/lib/pleroma/web/pleroma_api/controllers/scrobble_controller.ex b/lib/pleroma/web/pleroma_api/controllers/scrobble_controller.ex index bf6dc500c..5f5f7643f 100644 --- a/lib/pleroma/web/pleroma_api/controllers/scrobble_controller.ex +++ b/lib/pleroma/web/pleroma_api/controllers/scrobble_controller.ex @@ -24,6 +24,10 @@ defmodule Pleroma.Web.PleromaAPI.ScrobbleController do defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.PleromaScrobbleOperation def create(%{assigns: %{user: user}, body_params: params} = conn, _) do + params = + params + |> Map.put_new(:external_link, Map.get(params, :externalLink)) + with {:ok, activity} <- CommonAPI.listen(user, params) do render(conn, "show.json", activity: activity, for: user) else diff --git a/lib/pleroma/web/pleroma_api/views/scrobble_view.ex b/lib/pleroma/web/pleroma_api/views/scrobble_view.ex index edf0a2390..51828ad97 100644 --- a/lib/pleroma/web/pleroma_api/views/scrobble_view.ex +++ b/lib/pleroma/web/pleroma_api/views/scrobble_view.ex @@ -27,8 +27,10 @@ defmodule Pleroma.Web.PleromaAPI.ScrobbleView do title: object.data["title"] |> HTML.strip_tags(), artist: object.data["artist"] |> HTML.strip_tags(), album: object.data["album"] |> HTML.strip_tags(), - externalLink: object.data["externalLink"], - length: object.data["length"] + external_link: object.data["externalLink"], + length: object.data["length"], + # DEPRECATED + externalLink: object.data["externalLink"] } end diff --git a/test/pleroma/web/pleroma_api/controllers/scrobble_controller_test.exs b/test/pleroma/web/pleroma_api/controllers/scrobble_controller_test.exs index be94a02ad..bcc25b83e 100644 --- a/test/pleroma/web/pleroma_api/controllers/scrobble_controller_test.exs +++ b/test/pleroma/web/pleroma_api/controllers/scrobble_controller_test.exs @@ -19,10 +19,33 @@ defmodule Pleroma.Web.PleromaAPI.ScrobbleControllerTest do "artist" => "lain", "album" => "lain radio", "length" => "180000", - "externalLink" => "https://www.last.fm/music/lain/lain+radio/lain+radio+episode+1" + "external_link" => "https://www.last.fm/music/lain/lain+radio/lain+radio+episode+1" }) - assert %{"title" => "lain radio episode 1"} = json_response_and_validate_schema(conn, 200) + assert %{ + "title" => "lain radio episode 1", + "external_link" => "https://www.last.fm/music/lain/lain+radio/lain+radio+episode+1" + } = json_response_and_validate_schema(conn, 200) + end + + test "external_link fallback" do + %{conn: conn} = oauth_access(["write"]) + + conn = + conn + |> put_req_header("content-type", "application/json") + |> post("/api/v1/pleroma/scrobble", %{ + "title" => "lain radio episode 2", + "artist" => "lain", + "album" => "lain radio", + "length" => "180000", + "externalLink" => "https://www.last.fm/music/lain/lain+radio/lain+radio+episode+2" + }) + + assert %{ + "title" => "lain radio episode 2", + "external_link" => "https://www.last.fm/music/lain/lain+radio/lain+radio+episode+2" + } = json_response_and_validate_schema(conn, 200) end end @@ -35,7 +58,7 @@ defmodule Pleroma.Web.PleromaAPI.ScrobbleControllerTest do title: "lain radio episode 1", artist: "lain", album: "lain radio", - externalLink: "https://www.last.fm/music/lain/lain+radio/lain+radio+episode+1" + external_link: "https://www.last.fm/music/lain/lain+radio/lain+radio+episode+1" }) {:ok, _activity} = @@ -43,7 +66,7 @@ defmodule Pleroma.Web.PleromaAPI.ScrobbleControllerTest do title: "lain radio episode 2", artist: "lain", album: "lain radio", - externalLink: "https://www.last.fm/music/lain/lain+radio/lain+radio+episode+2" + external_link: "https://www.last.fm/music/lain/lain+radio/lain+radio+episode+2" }) {:ok, _activity} = @@ -51,7 +74,7 @@ defmodule Pleroma.Web.PleromaAPI.ScrobbleControllerTest do title: "lain radio episode 3", artist: "lain", album: "lain radio", - externalLink: "https://www.last.fm/music/lain/lain+radio/lain+radio+episode+3" + external_link: "https://www.last.fm/music/lain/lain+radio/lain+radio+episode+3" }) conn = get(conn, "/api/v1/pleroma/accounts/#{user.id}/scrobbles") From b2a716fc913f9777236dd771726068d4ac811e26 Mon Sep 17 00:00:00 2001 From: Phantasm Date: Tue, 10 Sep 2024 21:26:44 +0200 Subject: [PATCH 026/128] openbsd rc: replace deprecated flags, renamed to fit other service files --- installation/openbsd/rc.d/{pleromad => pleroma} | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) rename installation/openbsd/rc.d/{pleromad => pleroma} (61%) diff --git a/installation/openbsd/rc.d/pleromad b/installation/openbsd/rc.d/pleroma similarity index 61% rename from installation/openbsd/rc.d/pleromad rename to installation/openbsd/rc.d/pleroma index 19ac4bb51..9b54d5967 100755 --- a/installation/openbsd/rc.d/pleromad +++ b/installation/openbsd/rc.d/pleroma @@ -4,16 +4,18 @@ # # Simple installation instructions: # 1. Install Pleroma per wiki instructions -# 2. Place this pleromad file in /etc/rc.d +# 2. Place this pleroma file in /etc/rc.d # 3. Enable and start Pleroma -# # doas rcctl enable pleromad -# # doas rcctl start pleromad +# # doas rcctl enable pleroma +# # doas rcctl start pleroma # daemon="/usr/local/bin/elixir" -daemon_flags="--detached -S /usr/local/bin/mix phx.server" +daemon_flags="--erl \"-detached\" -S /usr/local/bin/mix phx.server" daemon_user="_pleroma" +env="MIX_ENV=prod" + . /etc/rc.d/rc.subr rc_reload=NO @@ -24,7 +26,7 @@ rc_check() { } rc_start() { - ${rcexec} "cd pleroma; ${daemon} ${daemon_flags}" + rc_exec "cd pleroma; export ${env}; ${daemon} ${daemon_flags}" } rc_stop() { From 9b71f57e372b5131b85ddceb6caf1e70a5e0de17 Mon Sep 17 00:00:00 2001 From: Phantasm Date: Tue, 10 Sep 2024 21:40:34 +0200 Subject: [PATCH 027/128] docs openbsd: add missing vips and libmagic depends to required software --- docs/installation/openbsd_en.md | 4 ++-- docs/installation/openbsd_fi.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/installation/openbsd_en.md b/docs/installation/openbsd_en.md index 78bbf399f..4c2f33f42 100644 --- a/docs/installation/openbsd_en.md +++ b/docs/installation/openbsd_en.md @@ -1,6 +1,6 @@ # Installing on OpenBSD -This guide describes the installation and configuration of pleroma (and the required software to run it) on a single OpenBSD 6.6 server. +This guide describes the installation and configuration of pleroma (and the required software to run it) on a single OpenBSD 7.5 server. For any additional information regarding commands and configuration files mentioned here, check the man pages [online](https://man.openbsd.org/) or directly on your server with the man command. @@ -12,7 +12,7 @@ For any additional information regarding commands and configuration files mentio To install them, run the following command (with doas or as root): ``` -pkg_add elixir gmake git postgresql-server postgresql-contrib cmake ffmpeg ImageMagick libvips +pkg_add elixir gmake git postgresql-server postgresql-contrib cmake libmagic libvips ``` Pleroma requires a reverse proxy, OpenBSD has relayd in base (and is used in this guide) and packages/ports are available for nginx (www/nginx) and apache (www/apache-httpd). Independently of the reverse proxy, [acme-client(1)](https://man.openbsd.org/acme-client) can be used to get a certificate from Let's Encrypt. diff --git a/docs/installation/openbsd_fi.md b/docs/installation/openbsd_fi.md index d7c94d8a0..858e64020 100644 --- a/docs/installation/openbsd_fi.md +++ b/docs/installation/openbsd_fi.md @@ -4,7 +4,7 @@ Note: This article is potentially outdated because at this time we may not have Tarvitset: * Oman domainin -* OpenBSD 6.3 -serverin +* OpenBSD 7.5 -serverin * Auttavan ymmärryksen unix-järjestelmistä Komennot, joiden edessä on '#', tulee ajaa käyttäjänä `root`. Tämä on @@ -18,7 +18,7 @@ Matrix-kanava #pleroma:libera.chat ovat hyviä paikkoja löytää apua Asenna tarvittava ohjelmisto: -`# pkg_add git elixir gmake postgresql-server-10.3 postgresql-contrib-10.3 cmake ffmpeg ImageMagick libvips` +`# pkg_add git elixir gmake postgresql-server postgresql-contrib cmake libmagic libvips` #### Optional software From cf0296bfdc8bb6ba935ad9b5362734329fc29fce Mon Sep 17 00:00:00 2001 From: Phantasm Date: Thu, 12 Sep 2024 21:55:29 +0200 Subject: [PATCH 028/128] docs openbsd: Add differences between otp and src, improved formatting and wording httpd/relayd and acme-client parts are untouched --- docs/installation/openbsd_en.md | 161 +++++++++++++++++++++++--------- 1 file changed, 116 insertions(+), 45 deletions(-) diff --git a/docs/installation/openbsd_en.md b/docs/installation/openbsd_en.md index 4c2f33f42..e47f40a87 100644 --- a/docs/installation/openbsd_en.md +++ b/docs/installation/openbsd_en.md @@ -1,25 +1,28 @@ # Installing on OpenBSD -This guide describes the installation and configuration of pleroma (and the required software to run it) on a single OpenBSD 7.5 server. +{! backend/installation/otp_vs_from_source_source.include !} + +This guide describes the installation and configuration of Pleroma (and the required software to run it) on a single OpenBSD 7.5 server. For any additional information regarding commands and configuration files mentioned here, check the man pages [online](https://man.openbsd.org/) or directly on your server with the man command. {! backend/installation/generic_dependencies.include !} +## Installation + ### Preparing the system #### Required software -To install them, run the following command (with doas or as root): +To install required packages, run the following command: ``` -pkg_add elixir gmake git postgresql-server postgresql-contrib cmake libmagic libvips +# pkg_add elixir gmake git postgresql-server postgresql-contrib cmake libmagic libvips ``` Pleroma requires a reverse proxy, OpenBSD has relayd in base (and is used in this guide) and packages/ports are available for nginx (www/nginx) and apache (www/apache-httpd). Independently of the reverse proxy, [acme-client(1)](https://man.openbsd.org/acme-client) can be used to get a certificate from Let's Encrypt. #### Optional software -Per [`docs/installation/optional/media_graphics_packages.md`](../installation/optional/media_graphics_packages.md): * ImageMagick * ffmpeg * exiftool @@ -27,41 +30,97 @@ Per [`docs/installation/optional/media_graphics_packages.md`](../installation/op To install the above: ``` -pkg_add ImageMagick ffmpeg p5-Image-ExifTool +# pkg_add ImageMagick ffmpeg p5-Image-ExifTool ``` -#### Creating the pleroma user -Pleroma will be run by a dedicated user, \_pleroma. Before creating it, insert the following lines in login.conf: +For more information read [`docs/installation/optional/media_graphics_packages.md`](../installation/optional/media_graphics_packages.md): + +### PostgreSQL + +Switch to the \_postgresql user and initialize PostgreSQL: + +``` +# su _postgresql +$ initdb -D /var/postgresql/data -U postgres +``` + +Running PostgreSQL in a different directory than `/var/postgresql/data` requires changing the `daemon_flags` variable in the `/etc/rc.d/postgresql` script. + +Enable and start the postgresql service: + +``` +# rcctl enable postgresql +# rcctl start postgresql +``` + +To check that PostgreSQL started properly and didn't fail right after starting, you can run `ps aux | grep postgres`, there should be multiple lines of output. Or alternatively run `# rcctl check postgresql` which should return `postgresql(ok)`. + +### Configuring Pleroma + +Pleroma will be run by a dedicated \_pleroma user. Before creating it, insert the following lines in /etc/login.conf: + ``` pleroma:\ :datasize-max=1536M:\ :datasize-cur=1536M:\ - :openfiles-max=4096 + :openfiles-max=4096:\ + :setenv=LC_ALL=en_US.UTF-8 ``` -This creates a "pleroma" login class and sets higher values than default for datasize and openfiles (see [login.conf(5)](https://man.openbsd.org/login.conf)), this is required to avoid having pleroma crash some time after starting. -Create the \_pleroma user, assign it the pleroma login class and create its home directory (/home/\_pleroma/): `useradd -m -L pleroma _pleroma` +This creates a "pleroma" login class and sets higher values than default for datasize and openfiles (see [login.conf(5)](https://man.openbsd.org/login.conf)), this is required to avoid having Pleroma crash some time after starting. -#### Clone pleroma's directory -Enter a shell as the \_pleroma user. As root, run `su _pleroma -;cd`. Then clone the repository with `git clone -b stable https://git.pleroma.social/pleroma/pleroma.git`. Pleroma is now installed in /home/\_pleroma/pleroma/, it will be configured and started at the end of this guide. - -#### PostgreSQL -Start a shell as the \_postgresql user (as root run `su _postgresql -` then run the `initdb` command to initialize postgresql: -You will need to specify pgdata directory to the default (/var/postgresql/data) with the `-D ` and set the user to postgres with the `-U ` flag. This can be done as follows: +Create the \_pleroma user, assign it the pleroma login class and create its home directory (/home/\_pleroma/): ``` -initdb -D /var/postgresql/data -U postgres +# useradd -m -L pleroma _pleroma +# echo 'export VIX_COMPILATION_MODE=PLATFORM_PROVIDED_LIBVIPS' >> /home/_pleroma/.profile ``` -If you are not using the default directory, you will have to update the `datadir` variable in the /etc/rc.d/postgresql script. -When this is done, enable postgresql so that it starts on boot and start it. As root, run: +Switch to the _pleroma user: + ``` -rcctl enable postgresql -rcctl start postgresql +# su _pleroma ``` -To check that it started properly and didn't fail right after starting, you can run `ps aux | grep postgres`, there should be multiple lines of output. + +Change to the home directory (/home/\_pleroma) and clone the Pleroma repository: + +``` +$ cd +$ git clone -b stable https://git.pleroma.social/pleroma/pleroma.git +$ cd pleroma +``` + +Pleroma is now installed in /home/\_pleroma/pleroma/. To configure it run: + +``` +$ mix deps.get +$ MIX_ENV=prod mix pleroma.instance gen # You will be asked a few questions here. +$ cp config/generated_config.exs config/prod.secret.exs +``` + +Note: Answer yes when asked to install Hex and rebar3. This step might take some time as Pleroma gets compiled first. + +Create the Pleroma database: + +``` +# psql -U postgres -f /home/_pleroma/pleroma/config/setup_db.psql +``` + +Switch back to the \_pleroma user and apply database migrations: + +``` +# su _pleroma +$ cd /home/_pleroma/pleroma +$ MIX_ENV=prod mix ecto.migrate +``` + +Note: You will need to run this step again when updating your instance to a newer version with `git pull` or `git checkout tags/NEW_VERSION`. + +As \_pleroma in /home/\_pleroma/pleroma, you can now run `MIX_ENV=prod mix phx.server` to start your instance. +In another SSH session or a tmux window, check that it is working properly by running `ftp -MVo - http://127.0.0.1:4000/api/v1/instance`, you should get json output. Double-check that the *uri* value near the bottom is your instance's domain name and the instance *title* is correct. #### httpd + httpd will have three functions: * redirect requests trying to reach the instance over http to the https URL @@ -69,6 +128,7 @@ httpd will have three functions: * get Let's Encrypt certificates, with acme-client Insert the following config in httpd.conf: + ``` # $OpenBSD: httpd.conf,v 1.17 2017/04/16 08:50:49 ajacoutot Exp $ @@ -95,18 +155,22 @@ server "default" { types { } ``` + Do not forget to change ** to your server's address(es). If httpd should only listen on one protocol family, comment one of the two first *listen* options. Create the /var/www/htdocs/local/ folder and write the content of your robots.txt in /var/www/htdocs/local/robots.txt. Check the configuration with `httpd -n`, if it is OK enable and start httpd (as root): + ``` -rcctl enable httpd -rcctl start httpd +# rcctl enable httpd +# rcctl start httpd ``` #### acme-client + acme-client is used to get SSL/TLS certificates from Let's Encrypt. Insert the following configuration in /etc/acme-client.conf: + ``` # # $OpenBSD: acme-client.conf,v 1.4 2017/03/22 11:14:14 benno Exp $ @@ -126,19 +190,24 @@ domain { challengedir "/var/www/acme/" } ``` + Replace ** by the domain name you'll use for your instance. As root, run `acme-client -n` to check the config, then `acme-client -ADv ` to create account and domain keys, and request a certificate for the first time. Make acme-client run everyday by adding it in /etc/daily.local. As root, run the following command: `echo "acme-client " >> /etc/daily.local`. Relayd will look for certificates and keys based on the address it listens on (see next part), the easiest way to make them available to relayd is to create a link, as root run: + ``` ln -s /etc/ssl/.fullchain.pem /etc/ssl/.crt ln -s /etc/ssl/private/.key /etc/ssl/private/.key ``` + This will have to be done for each IPv4 and IPv6 address relayd listens on. #### relayd + relayd will be used as the reverse proxy sitting in front of pleroma. Insert the following configuration in /etc/relayd.conf: + ``` # $OpenBSD: relayd.conf,v 1.4 2018/03/23 09:55:06 claudio Exp $ @@ -188,8 +257,10 @@ relay wwwtls { forward to port 80 check http "/robots.txt" code 200 } ``` + Again, change ** to your server's address(es) and comment one of the two *listen* options if needed. Also change *wss://CHANGEME.tld* to *wss://*. Check the configuration with `relayd -n`, if it is OK enable and start relayd (as root): + ``` rcctl enable relayd rcctl start relayd @@ -225,36 +296,36 @@ pass in quick on $if inet6 proto icmp6 to ($if) icmp6-type { echoreq unreach par pass in quick on $if proto tcp to ($if) port { http https } # relayd/httpd pass in quick on $if proto tcp from $authorized_ssh_clients to ($if) port ssh ``` + Replace ** by your server's network interface name (which you can get with ifconfig). Consider replacing the content of the authorized\_ssh\_clients macro by, for example, your home IP address, to avoid SSH connection attempts from bots. Check pf's configuration by running `pfctl -nf /etc/pf.conf`, load it with `pfctl -f /etc/pf.conf` and enable pf at boot with `rcctl enable pf`. -#### Configure and start pleroma -Enter a shell as \_pleroma (as root `su _pleroma -`) and enter pleroma's installation directory (`cd ~/pleroma/`). +### Starting pleroma at boot -Then follow the main installation guide: +Copy the startup script and make sure it's executable: - * run `mix deps.get` - * run `MIX_ENV=prod mix pleroma.instance gen` and enter your instance's information when asked - * copy config/generated\_config.exs to config/prod.secret.exs. The default values should be sufficient but you should edit it and check that everything seems OK. - * exit your current shell back to a root one and run `psql -U postgres -f /home/_pleroma/pleroma/config/setup_db.psql` to setup the database. - * return to a \_pleroma shell into pleroma's installation directory (`su _pleroma -;cd ~/pleroma`) and run `MIX_ENV=prod mix ecto.migrate` - -As \_pleroma in /home/\_pleroma/pleroma, you can now run `LC_ALL=en_US.UTF-8 MIX_ENV=prod mix phx.server` to start your instance. -In another SSH session/tmux window, check that it is working properly by running `ftp -MVo - http://127.0.0.1:4000/api/v1/instance`, you should get json output. Double-check that *uri*'s value is your instance's domain name. - -##### Starting pleroma at boot -An rc script to automatically start pleroma at boot hasn't been written yet, it can be run in a tmux session (tmux is in base). - - -#### Create administrative user - -If your instance is up and running, you can create your first user with administrative rights with the following command as the \_pleroma user. ``` -LC_ALL=en_US.UTF-8 MIX_ENV=prod mix pleroma.user new --admin +# cp /home/_pleroma/pleroma/installation/openbsd/rc.d/pleroma /etc/rc.d/pleroma +# chmod +x /etc/rc.d/pleroma ``` -#### Further reading +Enable and start the pleroma service: + +``` +# rcctl enable pleroma +# rcctl start pleroma +``` + +### Create administrative user + +If your instance is up and running, you can create your first user with administrative rights with the following command as the \_pleroma user: + +``` +MIX_ENV=prod mix pleroma.user new --admin +``` + +### Further reading {! backend/installation/further_reading.include !} From 1fcf7333540bda5f2957a5eecbb3122621d7b8e8 Mon Sep 17 00:00:00 2001 From: Phantasm Date: Mon, 23 Sep 2024 23:36:18 +0200 Subject: [PATCH 029/128] docs openbsd: Add nginx guide, do not recommend httpd/relayd MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit OpenBSD's httpd does not support caching in any way and putting a caching layer between it and Pleroma is pointless when nginx works fine. I also ran into issues with relayd when accessing it from the Tor browser. Federation seems to be unaffected by this as is base Firefox and Chrome. --- docs/installation/openbsd_en.md | 167 ++++++++++++++++++++++++-------- 1 file changed, 128 insertions(+), 39 deletions(-) diff --git a/docs/installation/openbsd_en.md b/docs/installation/openbsd_en.md index e47f40a87..b732205c2 100644 --- a/docs/installation/openbsd_en.md +++ b/docs/installation/openbsd_en.md @@ -117,7 +117,133 @@ $ MIX_ENV=prod mix ecto.migrate Note: You will need to run this step again when updating your instance to a newer version with `git pull` or `git checkout tags/NEW_VERSION`. As \_pleroma in /home/\_pleroma/pleroma, you can now run `MIX_ENV=prod mix phx.server` to start your instance. -In another SSH session or a tmux window, check that it is working properly by running `ftp -MVo - http://127.0.0.1:4000/api/v1/instance`, you should get json output. Double-check that the *uri* value near the bottom is your instance's domain name and the instance *title* is correct. +In another SSH session or a tmux window, check that it is working properly by running `ftp -MVo - http://127.0.0.1:4000/api/v1/instance`, you should get json output. Double-check that the *uri* value near the bottom is your instance's domain name and the instance *title* are correct. + +### Configuring acme-client + +acme-client is used to get SSL/TLS certificates from Let's Encrypt. +Insert the following configuration in /etc/acme-client.conf and replace `example.tld` with your domain: + +``` +# +# $OpenBSD: acme-client.conf,v 1.5 2023/05/10 07:34:57 tb Exp $ +# + +authority letsencrypt { + api url "https://acme-v02.api.letsencrypt.org/directory" + account key "/etc/acme/letsencrypt-privkey.pem" +} + +domain example.tld { + # Adds alternative names to the certificate. Useful when serving media on another domain. Comma or space separated list. + # alternative names { } + + domain key "/etc/ssl/private/example.tld.key" + domain certificate "/etc/ssl/example.tld_cert-only.crt" + domain full chain certificate "/etc/ssl/example.tld.crt" + sign with letsencrypt +} +``` + +Check the configuration: + +``` +# acme-client -n +``` + +Add auto-renewal by adding acme-client to `/etc/weekly.local`, replace `example.tld` with your domain: + +``` +echo "acme-client example.tld >> /etc/weekly.local +``` + +### Configuring the Web server + +Pleroma supports two Web servers: + + * nginx (recommended for most users) + * OpenBSD's httpd and relayd (ONLY for advanced users, media proxy cache is NOT supported and will NOT work properly) + +#### nginx + +Since nginx is not installed by default, install it by running: + +``` +# pkg_add nginx +``` + +Add the following to `/etc/nginx/nginx.conf`, within the `server {}` block listening on port 80 and change `server_name`, as follows: + +``` +http { + ... + + server { + ... + server_name example.tld; # Replace with your domain + + location ~ /.well-known/acme-challenge { + root /var/www/acme; + } + } +} +``` + +Start the nginx service and acquire certificates: + +``` +# rcctl start nginx +# acme-client example.tld +``` + +OpenBSD's default nginx configuration does not contain an include directive, which is typically used for multiple sites. +Therefore, you will need to first create the required directory as follows: + +``` +# mkdir /etc/nginx/sites-available +# mkdir /etc/nginx/sites-enabled +``` + +Next add the `include` directive to `/etc/nginx/nginx.conf`, within the `http {}` block, as follows: + +``` +http { + ... + + server { + ... + } + + include /etc/nginx/sites-enabled/*; +} +``` + +As root, copy `/home/_pleroma/pleroma/installation/pleroma.nginx` to `/etc/nginx/sites-available/pleroma.nginx`. + +Edit default `/etc/nginx/sites-available/pleroma.nginx` settings and replace `example.tld` with your domain: + + * Change `ssl_trusted_certificate` to `/etc/ssl/example.tld_cert-only.crt` + * Change `ssl_certificate` to `/etc/ssl/example.tld.crt` + * Change `ssl_certificate_key` to `/etc/ssl/private/example.tld.key` + +Symlink the Pleroma configuration to the enabled sites: + +``` +# ln -s /etc/nginx/sites-available/pleroma.nginx /etc/nginx/sites-enabled +``` + +Check nginx configuration syntax by running: + +``` +# nginx -t +``` + +If the configuration is correct, you can now enable and reload the nginx service: + +``` +# rcctl enable nginx +# rcctl reload nginx +``` #### httpd @@ -166,43 +292,6 @@ Check the configuration with `httpd -n`, if it is OK enable and start httpd (as # rcctl start httpd ``` -#### acme-client - -acme-client is used to get SSL/TLS certificates from Let's Encrypt. -Insert the following configuration in /etc/acme-client.conf: - -``` -# -# $OpenBSD: acme-client.conf,v 1.4 2017/03/22 11:14:14 benno Exp $ -# - -authority letsencrypt- { - #agreement url "https://letsencrypt.org/documents/LE-SA-v1.2-November-15-2017.pdf" - api url "https://acme-v02.api.letsencrypt.org/directory" - account key "/etc/acme/letsencrypt-privkey-.pem" -} - -domain { - domain key "/etc/ssl/private/.key" - domain certificate "/etc/ssl/.crt" - domain full chain certificate "/etc/ssl/.fullchain.pem" - sign with letsencrypt- - challengedir "/var/www/acme/" -} -``` - -Replace ** by the domain name you'll use for your instance. As root, run `acme-client -n` to check the config, then `acme-client -ADv ` to create account and domain keys, and request a certificate for the first time. -Make acme-client run everyday by adding it in /etc/daily.local. As root, run the following command: `echo "acme-client " >> /etc/daily.local`. - -Relayd will look for certificates and keys based on the address it listens on (see next part), the easiest way to make them available to relayd is to create a link, as root run: - -``` -ln -s /etc/ssl/.fullchain.pem /etc/ssl/.crt -ln -s /etc/ssl/private/.key /etc/ssl/private/.key -``` - -This will have to be done for each IPv4 and IPv6 address relayd listens on. - #### relayd relayd will be used as the reverse proxy sitting in front of pleroma. @@ -322,7 +411,7 @@ Enable and start the pleroma service: If your instance is up and running, you can create your first user with administrative rights with the following command as the \_pleroma user: ``` -MIX_ENV=prod mix pleroma.user new --admin +$ MIX_ENV=prod mix pleroma.user new --admin ``` ### Further reading From 71c60aa9fe5a58be92b32c1af56cac6ade742264 Mon Sep 17 00:00:00 2001 From: Phantasm Date: Sat, 26 Oct 2024 20:38:43 +0200 Subject: [PATCH 030/128] docs openbsd: specifically install erlang 26 due to a TLSv1.3 bug OTP 25 and earlier versions have a broken TLSv1.3 minimum requirements check that breaks federation for TLSv1.3-only instances. --- docs/installation/openbsd_en.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/installation/openbsd_en.md b/docs/installation/openbsd_en.md index b732205c2..1e7a011fc 100644 --- a/docs/installation/openbsd_en.md +++ b/docs/installation/openbsd_en.md @@ -2,7 +2,7 @@ {! backend/installation/otp_vs_from_source_source.include !} -This guide describes the installation and configuration of Pleroma (and the required software to run it) on a single OpenBSD 7.5 server. +This guide describes the installation and configuration of Pleroma (and the required software to run it) on a single OpenBSD 7.6 server. For any additional information regarding commands and configuration files mentioned here, check the man pages [online](https://man.openbsd.org/) or directly on your server with the man command. @@ -16,7 +16,7 @@ For any additional information regarding commands and configuration files mentio To install required packages, run the following command: ``` -# pkg_add elixir gmake git postgresql-server postgresql-contrib cmake libmagic libvips +# pkg_add erlang%26 elixir gmake git postgresql-server postgresql-contrib cmake libmagic libvips ``` Pleroma requires a reverse proxy, OpenBSD has relayd in base (and is used in this guide) and packages/ports are available for nginx (www/nginx) and apache (www/apache-httpd). Independently of the reverse proxy, [acme-client(1)](https://man.openbsd.org/acme-client) can be used to get a certificate from Let's Encrypt. From 3dc2655f5954fbcd426a67f96cc40b16fedf52eb Mon Sep 17 00:00:00 2001 From: Phantasm Date: Mon, 11 Nov 2024 23:48:33 +0100 Subject: [PATCH 031/128] openbsd: update relayd and httpd configuration files * httpd: use proper server names * httpd: add example of a very basic static website along with Pleroma * httpd: let Pleroma serve robots.txt * relayd: add example of forwarding to a basic httpd website * relayd: remove appended response headers (most of them already served by Pleroma anyway) * relayd: add comments about hosting Pleroma on subdomains * relayd: reject request that don't belong to any forward * relayd: add example of hosting media uploads on subdomain * relayd: change forward timeout check to something sane that actually works --- installation/openbsd/httpd.conf | 17 +++++++--- installation/openbsd/relayd.conf | 56 +++++++++++++++++++++++--------- 2 files changed, 53 insertions(+), 20 deletions(-) diff --git a/installation/openbsd/httpd.conf b/installation/openbsd/httpd.conf index 82f4803fd..912a541af 100644 --- a/installation/openbsd/httpd.conf +++ b/installation/openbsd/httpd.conf @@ -4,8 +4,9 @@ # 1. Place file in /etc # 2. Replace with your public IP address # 3. If using IPv6, uncomment IPv6 lines and replace with your public IPv6 address -# 4. Check file using 'doas httpd -n' -# 5. Enable and start httpd: +# 4. Replace all occurences of example.tld with your instance's domain name. +# 5. Check file using 'doas httpd -n' +# 6. Enable and start httpd: # # doas rcctl enable httpd # # doas rcctl start httpd # @@ -13,7 +14,7 @@ ext_inet="" #ext_inet6="" -server "default" { +server "example.tld" { listen on $ext_inet port 80 # Comment to disable listening on IPv4 # listen on $ext_inet6 port 80 # Comment to disable listening on IPv6 listen on 127.0.0.1 port 80 # Do NOT comment this line @@ -26,10 +27,18 @@ server "default" { request strip 2 } - location "/robots.txt" { root "/htdocs/local/" } location "/*" { block return 302 "https://$HTTP_HOST$REQUEST_URI" } } +# Example of serving a basic static website besides Pleroma using the example configuration in relayd +#server "site.example.tld" { +# listen on 127.0.0.1 port 8080 +# +# location "/*" { +# root "/website" +# } +#} + types { include "/usr/share/misc/mime.types" } diff --git a/installation/openbsd/relayd.conf b/installation/openbsd/relayd.conf index 31c2c1129..816de6de7 100644 --- a/installation/openbsd/relayd.conf +++ b/installation/openbsd/relayd.conf @@ -4,8 +4,9 @@ # 1. Place in /etc # 2. Replace with your public IPv4 address # 3. If using IPv6i, uncomment IPv6 lines and replace with your public IPv6 address -# 4. Check file using 'doas relayd -n' -# 5. Reload/start relayd +# 4. Replace all occurrences of example.tld with your instance's domain +# 5. Check file using 'doas relayd -n' +# 6. Reload/start relayd # # doas rcctl enable relayd # # doas rcctl start relayd # @@ -14,31 +15,54 @@ ext_inet="" #ext_inet6="" table { 127.0.0.1 } -table { 127.0.0.1 } -http protocol plerup { # Protocol for upstream pleroma server +# Uncomment next line when you want to serve other services than Pleroma. +# In this example tables are used only as way to differentiate between Pleroma and other services. +# Feel free to rename "httpd_server" everywhere to fit your setup. +#table { 127.0.0.1 } + +http protocol pleroma { # Protocol for upstream Pleroma server #tcp { nodelay, sack, socket buffer 65536, backlog 128 } # Uncomment and adjust as you see fit - tls ciphers "ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA0-POLY1305" - tls ecdhe secp384r1 + tls ciphers "ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:!aNULL:!eNULL:!EXPORT:!DES:!MD5:!PSK:!RC4" + tls ecdhe "X25519,P-256,P-384,secp521r1" # relayd default+secp521r1 - # Forward some paths to the local server (as pleroma won't respond to them as you might want) - pass request quick path "/robots.txt" forward to + return error - # Append a bunch of headers - match request header append "X-Forwarded-For" value "$REMOTE_ADDR" # This two header and the next one are not strictl required by pleroma but adding them won't hurt - match request header append "X-Forwarded-By" value "$SERVER_ADDR:$SERVER_PORT" + # When serving multiple services with different certificates, specify multiple "tls keypair" keywords + # and add forwards to those services before the block keyword near the bottom of the protocol and relay configurations. + # The string in quotes must match the fullchain certificate file create by acme-client. + # For example: + # tls keypair "pleroma.example.tld" + # tls keypair "example.tld" + tls keypair "example.tld" + match request header append "X-Forwarded-For" value "$REMOTE_ADDR" match request header append "Connection" value "upgrade" + # When hosting Pleroma on a subdomain, replace example.tld accordingly (not the base domain). + # From the above example, "example.tld" should be replaced with "pleroma.example.tld" instead. + pass request quick header "Host" value "example.tld" forward to + + # Uncomment when serving media uploads on a different (sub)domain. + # Keep media proxy disabled, as it will NOT work under relayd/httpd. If you want to also setup media proxy, use nginx instead. + #pass request quick header "Host" value "media.example.tld" forward to + + # When serving multiple services, add the forwards here. + # Example: + #pass request quick header "Host" value "example.tld" forward to + + block } relay wwwtls { listen on $ext_inet port https tls # Comment to disable listening on IPv4 -# listen on $ext_inet6 port https tls # Comment to disable listening on IPv6 + #listen on $ext_inet6 port https tls # Comment to disable listening on IPv6 - protocol plerup + protocol pleroma - forward to port 4000 check http "/" code 200 - forward to port 80 check http "/robots.txt" code 200 + forward to port 4000 check tcp timeout 500 # Adjust timeout accordingly when relayd returns 502 while Pleroma is running without problems. + + # When serving multiple services, add the forwards here. + # Example: + #forward to port 8080 } - From 9b39065595ee49dad929c2613bf5ec04413039a7 Mon Sep 17 00:00:00 2001 From: Phantasm Date: Tue, 12 Nov 2024 00:15:07 +0100 Subject: [PATCH 032/128] openbsd: add changelogs --- changelog.d/openbsd-docs-update.skip | 0 changelog.d/openbsd-update-httpd-relayd.change | 1 + changelog.d/openbsd-update-rc.fix | 1 + 3 files changed, 2 insertions(+) create mode 100644 changelog.d/openbsd-docs-update.skip create mode 100644 changelog.d/openbsd-update-httpd-relayd.change create mode 100644 changelog.d/openbsd-update-rc.fix diff --git a/changelog.d/openbsd-docs-update.skip b/changelog.d/openbsd-docs-update.skip new file mode 100644 index 000000000..e69de29bb diff --git a/changelog.d/openbsd-update-httpd-relayd.change b/changelog.d/openbsd-update-httpd-relayd.change new file mode 100644 index 000000000..2ee85c2b0 --- /dev/null +++ b/changelog.d/openbsd-update-httpd-relayd.change @@ -0,0 +1 @@ +Updated relayd/httpd config files to be on par with nginx diff --git a/changelog.d/openbsd-update-rc.fix b/changelog.d/openbsd-update-rc.fix new file mode 100644 index 000000000..2d4263827 --- /dev/null +++ b/changelog.d/openbsd-update-rc.fix @@ -0,0 +1 @@ +replaced depracated flags and functions, renamed service to fit other service files From 427db326032628248b44439d5593f8395b3de428 Mon Sep 17 00:00:00 2001 From: Phantasm Date: Tue, 12 Nov 2024 00:21:33 +0100 Subject: [PATCH 033/128] openbsd relayd: clarify certificate naming --- installation/openbsd/relayd.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/installation/openbsd/relayd.conf b/installation/openbsd/relayd.conf index 816de6de7..b04f122e1 100644 --- a/installation/openbsd/relayd.conf +++ b/installation/openbsd/relayd.conf @@ -16,7 +16,7 @@ ext_inet="" table { 127.0.0.1 } -# Uncomment next line when you want to serve other services than Pleroma. +# Uncomment when you want to serve other services than Pleroma. # In this example tables are used only as way to differentiate between Pleroma and other services. # Feel free to rename "httpd_server" everywhere to fit your setup. #table { 127.0.0.1 } @@ -30,7 +30,7 @@ http protocol pleroma { # Protocol for upstream Pleroma server # When serving multiple services with different certificates, specify multiple "tls keypair" keywords # and add forwards to those services before the block keyword near the bottom of the protocol and relay configurations. - # The string in quotes must match the fullchain certificate file create by acme-client. + # The string in quotes must match the fullchain certificate file created by acme-client without the extension. # For example: # tls keypair "pleroma.example.tld" # tls keypair "example.tld" From d3f2d5919cd426b9cbd6c485ffd513610abc2dd6 Mon Sep 17 00:00:00 2001 From: Phantasm Date: Fri, 22 Nov 2024 19:44:27 +0100 Subject: [PATCH 034/128] docs openbsd: update install instructions for httpd/relayd --- docs/installation/openbsd_en.md | 115 +++++++++----------------------- 1 file changed, 31 insertions(+), 84 deletions(-) diff --git a/docs/installation/openbsd_en.md b/docs/installation/openbsd_en.md index 1e7a011fc..4deed6550 100644 --- a/docs/installation/openbsd_en.md +++ b/docs/installation/openbsd_en.md @@ -247,45 +247,27 @@ If the configuration is correct, you can now enable and reload the nginx service #### httpd -httpd will have three functions: +httpd will have two functions: * redirect requests trying to reach the instance over http to the https URL - * serve a robots.txt file * get Let's Encrypt certificates, with acme-client -Insert the following config in httpd.conf: +As root, copy `/home/_pleroma/pleroma/installation/openbsd/httpd.conf` to `/etc/httpd.conf`, or modify the existing one. +Edit `/etc/httpd.conf` settings and change: + + * `` with your instance's IPv4 address + * All occurances of `example.tld` with your instance's domain name + * When using IPv6 also change: + - Uncomment the `ext_inet6=""` line near the beginning of the file and change `* to your server's address(es). If httpd should only listen on one protocol family, comment one of the two first *listen* options. - -Create the /var/www/htdocs/local/ folder and write the content of your robots.txt in /var/www/htdocs/local/robots.txt. -Check the configuration with `httpd -n`, if it is OK enable and start httpd (as root): +If the configuration is correct, enable and start the `httpd` service: ``` # rcctl enable httpd @@ -295,73 +277,38 @@ Check the configuration with `httpd -n`, if it is OK enable and start httpd (as #### relayd relayd will be used as the reverse proxy sitting in front of pleroma. -Insert the following configuration in /etc/relayd.conf: +As root, copy `/home/_pleroma/pleroma/installation/openbsd/relayd.conf` to `/etc/relayd.conf`, or modify the existing one. + +Edit `/etc/relayd.conf` settings and change: + + * `` with your instance's IPv4 address + * All occurances of `example.tld` with your instance's domain name + * When using IPv6 also change: + - Uncomment the `ext_inet6=""` line near the beginning of the file and change `` to your instance's IPv6 address + - Uncomment the line starting with `listen on $ext_inet6` in the `relay wwwtls` block + +Check the configuration by running: ``` -# $OpenBSD: relayd.conf,v 1.4 2018/03/23 09:55:06 claudio Exp $ - -ext_inet="" -ext_inet6="" - -table { 127.0.0.1 } -table { 127.0.0.1 } - -http protocol plerup { # Protocol for upstream pleroma server - #tcp { nodelay, sack, socket buffer 65536, backlog 128 } # Uncomment and adjust as you see fit - tls ciphers "ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305" - tls ecdhe secp384r1 - - # Forward some paths to the local server (as pleroma won't respond to them as you might want) - pass request quick path "/robots.txt" forward to - - # Append a bunch of headers - match request header append "X-Forwarded-For" value "$REMOTE_ADDR" # This two header and the next one are not strictly required by pleroma but adding them won't hurt - match request header append "X-Forwarded-By" value "$SERVER_ADDR:$SERVER_PORT" - - match response header append "X-XSS-Protection" value "1; mode=block" - match response header append "X-Permitted-Cross-Domain-Policies" value "none" - match response header append "X-Frame-Options" value "DENY" - match response header append "X-Content-Type-Options" value "nosniff" - match response header append "Referrer-Policy" value "same-origin" - match response header append "X-Download-Options" value "noopen" - match response header append "Content-Security-Policy" value "default-src 'none'; base-uri 'self'; form-action 'self'; img-src 'self' data: https:; media-src 'self' https:; style-src 'self' 'unsafe-inline'; font-src 'self'; script-src 'self'; connect-src 'self' wss://CHANGEME.tld; upgrade-insecure-requests;" # Modify "CHANGEME.tld" and set your instance's domain here - match request header append "Connection" value "upgrade" - #match response header append "Strict-Transport-Security" value "max-age=31536000; includeSubDomains" # Uncomment this only after you get HTTPS working. - - # If you do not want remote frontends to be able to access your Pleroma backend server, comment these lines - match response header append "Access-Control-Allow-Origin" value "*" - match response header append "Access-Control-Allow-Methods" value "POST, PUT, DELETE, GET, PATCH, OPTIONS" - match response header append "Access-Control-Allow-Headers" value "Authorization, Content-Type, Idempotency-Key" - match response header append "Access-Control-Expose-Headers" value "Link, X-RateLimit-Reset, X-RateLimit-Limit, X-RateLimit-Remaining, X-Request-Id" - # Stop commenting lines here -} - -relay wwwtls { - listen on $ext_inet port https tls # Comment to disable listening on IPv4 - listen on $ext_inet6 port https tls # Comment to disable listening on IPv6 - - protocol plerup - - forward to port 4000 check http "/" code 200 - forward to port 80 check http "/robots.txt" code 200 -} +# relayd -n ``` -Again, change ** to your server's address(es) and comment one of the two *listen* options if needed. Also change *wss://CHANGEME.tld* to *wss://*. -Check the configuration with `relayd -n`, if it is OK enable and start relayd (as root): +If the configuration is correct, enable and start the `relayd` service: ``` -rcctl enable relayd -rcctl start relayd +# rcctl enable relayd +# rcctl start relayd ``` -##### (Strongly recommended) serve media on another domain +#### (Strongly recommended) serve media on another domain Refer to the [Hardening your instance](../configuration/hardening.md) document on how to serve media on another domain. We STRONGLY RECOMMEND you to do this to minimize attack vectors. + #### pf Enabling and configuring pf is highly recommended. In /etc/pf.conf, insert the following configuration: + ``` # Macros if="" From 0bd21084c42ab6e935c5a53e1ee12aa7bca3b835 Mon Sep 17 00:00:00 2001 From: Phantasm Date: Fri, 22 Nov 2024 19:45:45 +0100 Subject: [PATCH 035/128] docs openbsd: remove firewall configuation from install instructions It isn't in any of the install docs, why should it be here. --- docs/installation/openbsd_en.md | 33 --------------------------------- 1 file changed, 33 deletions(-) diff --git a/docs/installation/openbsd_en.md b/docs/installation/openbsd_en.md index 4deed6550..76b3d69c5 100644 --- a/docs/installation/openbsd_en.md +++ b/docs/installation/openbsd_en.md @@ -304,39 +304,6 @@ If the configuration is correct, enable and start the `relayd` service: Refer to the [Hardening your instance](../configuration/hardening.md) document on how to serve media on another domain. We STRONGLY RECOMMEND you to do this to minimize attack vectors. - -#### pf -Enabling and configuring pf is highly recommended. -In /etc/pf.conf, insert the following configuration: - -``` -# Macros -if="" -authorized_ssh_clients="any" - -# Skip traffic on loopback interface -set skip on lo - -# Default behavior -set block-policy drop -block in log all -pass out quick - -# Security features -match in all scrub (no-df random-id) -block in log from urpf-failed - -# Rules -pass in quick on $if inet proto icmp to ($if) icmp-type { echoreq unreach paramprob trace } # ICMP -pass in quick on $if inet6 proto icmp6 to ($if) icmp6-type { echoreq unreach paramprob timex toobig } # ICMPv6 -pass in quick on $if proto tcp to ($if) port { http https } # relayd/httpd -pass in quick on $if proto tcp from $authorized_ssh_clients to ($if) port ssh -``` - -Replace ** by your server's network interface name (which you can get with ifconfig). Consider replacing the content of the authorized\_ssh\_clients macro by, for example, your home IP address, to avoid SSH connection attempts from bots. - -Check pf's configuration by running `pfctl -nf /etc/pf.conf`, load it with `pfctl -f /etc/pf.conf` and enable pf at boot with `rcctl enable pf`. - ### Starting pleroma at boot Copy the startup script and make sure it's executable: From a21e11f586676f001bb32d1a5786a8ebf7132ba7 Mon Sep 17 00:00:00 2001 From: Phantasm Date: Fri, 22 Nov 2024 19:47:37 +0100 Subject: [PATCH 036/128] openbsd: unify IPvX placeholders in configs --- installation/openbsd/httpd.conf | 10 +++++----- installation/openbsd/relayd.conf | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/installation/openbsd/httpd.conf b/installation/openbsd/httpd.conf index 912a541af..c8ddae629 100644 --- a/installation/openbsd/httpd.conf +++ b/installation/openbsd/httpd.conf @@ -2,8 +2,8 @@ # Default httpd.conf file for Pleroma on OpenBSD # Simple installation instructions # 1. Place file in /etc -# 2. Replace with your public IP address -# 3. If using IPv6, uncomment IPv6 lines and replace with your public IPv6 address +# 2. Replace with your public IP address +# 3. If using IPv6, uncomment IPv6 lines and replace with your public IPv6 address # 4. Replace all occurences of example.tld with your instance's domain name. # 5. Check file using 'doas httpd -n' # 6. Enable and start httpd: @@ -11,12 +11,12 @@ # # doas rcctl start httpd # -ext_inet="" -#ext_inet6="" +ext_inet="" +#ext_inet6="" server "example.tld" { listen on $ext_inet port 80 # Comment to disable listening on IPv4 -# listen on $ext_inet6 port 80 # Comment to disable listening on IPv6 + #listen on $ext_inet6 port 80 # Comment to disable listening on IPv6 listen on 127.0.0.1 port 80 # Do NOT comment this line log syslog diff --git a/installation/openbsd/relayd.conf b/installation/openbsd/relayd.conf index b04f122e1..8b7be4ca6 100644 --- a/installation/openbsd/relayd.conf +++ b/installation/openbsd/relayd.conf @@ -3,7 +3,7 @@ # Simple installation instructions: # 1. Place in /etc # 2. Replace with your public IPv4 address -# 3. If using IPv6i, uncomment IPv6 lines and replace with your public IPv6 address +# 3. If using IPv6, uncomment IPv6 lines and replace with your public IPv6 address # 4. Replace all occurrences of example.tld with your instance's domain # 5. Check file using 'doas relayd -n' # 6. Reload/start relayd From 79c5ca05c9956a3dbbc4faf4c71054f57622b458 Mon Sep 17 00:00:00 2001 From: Phantasm Date: Sun, 24 Nov 2024 16:42:24 +0100 Subject: [PATCH 037/128] docs openbsd: inherit default daemon limits and tweak them su _pleroma commands were also changed in docs to simulate a full login to apply the custom environment from login.conf --- docs/installation/openbsd_en.md | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/docs/installation/openbsd_en.md b/docs/installation/openbsd_en.md index 76b3d69c5..f205aa573 100644 --- a/docs/installation/openbsd_en.md +++ b/docs/installation/openbsd_en.md @@ -57,14 +57,15 @@ To check that PostgreSQL started properly and didn't fail right after starting, ### Configuring Pleroma -Pleroma will be run by a dedicated \_pleroma user. Before creating it, insert the following lines in /etc/login.conf: +Pleroma will be run by a dedicated \_pleroma user. Before creating it, insert the following lines in `/etc/login.conf`: ``` pleroma:\ - :datasize-max=1536M:\ - :datasize-cur=1536M:\ + :datasize=1536M:\ :openfiles-max=4096:\ - :setenv=LC_ALL=en_US.UTF-8 + :openfiles-cur=1024:\ + :setenv=LC_ALL=en_US.UTF-8,VIX_COMPILATION_MODE=PLATFORM_PROVIDED_LIBVIPS:\ + :tc=daemon: ``` This creates a "pleroma" login class and sets higher values than default for datasize and openfiles (see [login.conf(5)](https://man.openbsd.org/login.conf)), this is required to avoid having Pleroma crash some time after starting. @@ -73,19 +74,17 @@ Create the \_pleroma user, assign it the pleroma login class and create its home ``` # useradd -m -L pleroma _pleroma -# echo 'export VIX_COMPILATION_MODE=PLATFORM_PROVIDED_LIBVIPS' >> /home/_pleroma/.profile ``` Switch to the _pleroma user: ``` -# su _pleroma +# su -l _pleroma ``` -Change to the home directory (/home/\_pleroma) and clone the Pleroma repository: +Clone the Pleroma repository: ``` -$ cd $ git clone -b stable https://git.pleroma.social/pleroma/pleroma.git $ cd pleroma ``` @@ -109,8 +108,8 @@ Create the Pleroma database: Switch back to the \_pleroma user and apply database migrations: ``` -# su _pleroma -$ cd /home/_pleroma/pleroma +# su -l _pleroma +$ cd pleroma $ MIX_ENV=prod mix ecto.migrate ``` From ee25acea6d87c036d195c69430bd2e92ea56bd52 Mon Sep 17 00:00:00 2001 From: Phantasm Date: Sun, 24 Nov 2024 23:43:55 +0100 Subject: [PATCH 038/128] docs openbsd: Fix nginx acme challenges, automatic certificate renewals in proper places --- docs/installation/openbsd_en.md | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/docs/installation/openbsd_en.md b/docs/installation/openbsd_en.md index f205aa573..1194a5f07 100644 --- a/docs/installation/openbsd_en.md +++ b/docs/installation/openbsd_en.md @@ -121,7 +121,7 @@ In another SSH session or a tmux window, check that it is working properly by ru ### Configuring acme-client acme-client is used to get SSL/TLS certificates from Let's Encrypt. -Insert the following configuration in /etc/acme-client.conf and replace `example.tld` with your domain: +Insert the following configuration in `/etc/acme-client.conf` and replace `example.tld` with your domain: ``` # @@ -150,12 +150,6 @@ Check the configuration: # acme-client -n ``` -Add auto-renewal by adding acme-client to `/etc/weekly.local`, replace `example.tld` with your domain: - -``` -echo "acme-client example.tld >> /etc/weekly.local -``` - ### Configuring the Web server Pleroma supports two Web servers: @@ -181,7 +175,8 @@ http { ... server_name example.tld; # Replace with your domain - location ~ /.well-known/acme-challenge { + location /.well-known/acme-challenge { + rewrite ^/.well-known/acme-challenge/(.*) /$1 break; root /var/www/acme; } } @@ -195,6 +190,12 @@ Start the nginx service and acquire certificates: # acme-client example.tld ``` +Add certificate auto-renewal by adding acme-client to `/etc/weekly.local`, replace `example.tld` with your domain: + +``` +# echo "acme-client example.tld && rcctl reload nginx" >> /etc/weekly.local +``` + OpenBSD's default nginx configuration does not contain an include directive, which is typically used for multiple sites. Therefore, you will need to first create the required directory as follows: @@ -246,6 +247,8 @@ If the configuration is correct, you can now enable and reload the nginx service #### httpd +***Skip this section when using nginx*** + httpd will have two functions: * redirect requests trying to reach the instance over http to the https URL @@ -275,6 +278,8 @@ If the configuration is correct, enable and start the `httpd` service: #### relayd +***Skip this section when using nginx*** + relayd will be used as the reverse proxy sitting in front of pleroma. As root, copy `/home/_pleroma/pleroma/installation/openbsd/relayd.conf` to `/etc/relayd.conf`, or modify the existing one. @@ -299,6 +304,12 @@ If the configuration is correct, enable and start the `relayd` service: # rcctl start relayd ``` +Add certificate auto-renewal by adding acme-client to `/etc/weekly.local`, replace `example.tld` with your domain: + +``` +# echo "acme-client example.tld && rcctl reload relayd" >> /etc/weekly.local +``` + #### (Strongly recommended) serve media on another domain Refer to the [Hardening your instance](../configuration/hardening.md) document on how to serve media on another domain. We STRONGLY RECOMMEND you to do this to minimize attack vectors. From df492669e576de8feb83a9f83d621533326e3f21 Mon Sep 17 00:00:00 2001 From: Phantasm Date: Sun, 24 Nov 2024 23:45:03 +0100 Subject: [PATCH 039/128] docs openbsd: proper permission for Pleroma service file --- docs/installation/openbsd_en.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/installation/openbsd_en.md b/docs/installation/openbsd_en.md index 1194a5f07..f0d6b9e93 100644 --- a/docs/installation/openbsd_en.md +++ b/docs/installation/openbsd_en.md @@ -320,7 +320,7 @@ Copy the startup script and make sure it's executable: ``` # cp /home/_pleroma/pleroma/installation/openbsd/rc.d/pleroma /etc/rc.d/pleroma -# chmod +x /etc/rc.d/pleroma +# chmod 555 /etc/rc.d/pleroma ``` Enable and start the pleroma service: From b0721ddbf5c0e32fdab6fda09855b061cc0fb1e1 Mon Sep 17 00:00:00 2001 From: Phantasm Date: Mon, 25 Nov 2024 00:03:04 +0100 Subject: [PATCH 040/128] docs openbsd: recommend changing pgsql auth method, remove redundant service check --- docs/installation/openbsd_en.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/installation/openbsd_en.md b/docs/installation/openbsd_en.md index f0d6b9e93..45cd03a9a 100644 --- a/docs/installation/openbsd_en.md +++ b/docs/installation/openbsd_en.md @@ -46,6 +46,11 @@ $ initdb -D /var/postgresql/data -U postgres Running PostgreSQL in a different directory than `/var/postgresql/data` requires changing the `daemon_flags` variable in the `/etc/rc.d/postgresql` script. +For security reasons it is recommended to change the authentication method for `local` and `host` connections with the localhost address to `scram-sha-256`.
+Do not forget to set a password for the `postgres` user before doing so, otherwise you won't be able to log back in unless you change the authentication method back to `trust`.
+Changing the password hashing algorithm is not needed.
+For more information [read](https://www.postgresql.org/docs/16/auth-pg-hba-conf.html) the PostgreSQL documentation. + Enable and start the postgresql service: ``` @@ -53,7 +58,7 @@ Enable and start the postgresql service: # rcctl start postgresql ``` -To check that PostgreSQL started properly and didn't fail right after starting, you can run `ps aux | grep postgres`, there should be multiple lines of output. Or alternatively run `# rcctl check postgresql` which should return `postgresql(ok)`. +To check that PostgreSQL started properly and didn't fail right after starting, run `# rcctl check postgresql` which should return `postgresql(ok)`. ### Configuring Pleroma From e0ba132bce735a5c429fa2280ce90d99fb02ae10 Mon Sep 17 00:00:00 2001 From: Phantasm Date: Tue, 26 Nov 2024 14:53:02 +0100 Subject: [PATCH 041/128] docs openbsd: ensure db has UTF-8 enconding --- docs/installation/openbsd_en.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/installation/openbsd_en.md b/docs/installation/openbsd_en.md index 45cd03a9a..cf3dee5e3 100644 --- a/docs/installation/openbsd_en.md +++ b/docs/installation/openbsd_en.md @@ -41,7 +41,7 @@ Switch to the \_postgresql user and initialize PostgreSQL: ``` # su _postgresql -$ initdb -D /var/postgresql/data -U postgres +$ initdb -D /var/postgresql/data -U postgres --encoding=utf-8 --lc-collate=C ``` Running PostgreSQL in a different directory than `/var/postgresql/data` requires changing the `daemon_flags` variable in the `/etc/rc.d/postgresql` script. From 3b5b3ba4fc1e714c9d8927bb32f85d56e2f6b3d4 Mon Sep 17 00:00:00 2001 From: Phantasm Date: Wed, 27 Nov 2024 21:40:36 +0100 Subject: [PATCH 042/128] openbsd: properly set daemon workdir, use default rc_start, set MIX_ENV in login.conf Setting the MIX_ENV variable in rc_pre() isn't possible, because the environment doesn't persist between rc_pre and rc_start(). This way we can also ditch the custom rc_start() function in favor of the default one which is just: rc_start() { rc_exec "${daemon} ${daemon_flags} } --- docs/installation/openbsd_en.md | 2 +- installation/openbsd/rc.d/pleroma | 7 +------ 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/docs/installation/openbsd_en.md b/docs/installation/openbsd_en.md index cf3dee5e3..8aaa6e8de 100644 --- a/docs/installation/openbsd_en.md +++ b/docs/installation/openbsd_en.md @@ -69,7 +69,7 @@ pleroma:\ :datasize=1536M:\ :openfiles-max=4096:\ :openfiles-cur=1024:\ - :setenv=LC_ALL=en_US.UTF-8,VIX_COMPILATION_MODE=PLATFORM_PROVIDED_LIBVIPS:\ + :setenv=LC_ALL=en_US.UTF-8,VIX_COMPILATION_MODE=PLATFORM_PROVIDED_LIBVIPS,MIX_ENV=prod:\ :tc=daemon: ``` diff --git a/installation/openbsd/rc.d/pleroma b/installation/openbsd/rc.d/pleroma index 9b54d5967..6959c20b0 100755 --- a/installation/openbsd/rc.d/pleroma +++ b/installation/openbsd/rc.d/pleroma @@ -13,8 +13,7 @@ daemon="/usr/local/bin/elixir" daemon_flags="--erl \"-detached\" -S /usr/local/bin/mix phx.server" daemon_user="_pleroma" - -env="MIX_ENV=prod" +daemon_execdir="/home/_pleroma/pleroma" . /etc/rc.d/rc.subr @@ -25,10 +24,6 @@ rc_check() { pgrep -q -U _pleroma -f "phx.server" } -rc_start() { - rc_exec "cd pleroma; export ${env}; ${daemon} ${daemon_flags}" -} - rc_stop() { pkill -q -U _pleroma -f "phx.server" } From accdefb8db480066ca06176db94b7c82c74cd6b9 Mon Sep 17 00:00:00 2001 From: Phantasm Date: Wed, 27 Nov 2024 21:46:50 +0100 Subject: [PATCH 043/128] openbsd httpd: use more appropriate HTTP response code for redirect --- installation/openbsd/httpd.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/installation/openbsd/httpd.conf b/installation/openbsd/httpd.conf index c8ddae629..f37325d91 100644 --- a/installation/openbsd/httpd.conf +++ b/installation/openbsd/httpd.conf @@ -27,7 +27,7 @@ server "example.tld" { request strip 2 } - location "/*" { block return 302 "https://$HTTP_HOST$REQUEST_URI" } + location "/*" { block return 301 "https://$HTTP_HOST$REQUEST_URI" } } # Example of serving a basic static website besides Pleroma using the example configuration in relayd From 49c35f8d95e4fb7e58d62e2b3babc06bb3066429 Mon Sep 17 00:00:00 2001 From: Phantasm Date: Wed, 27 Nov 2024 21:47:13 +0100 Subject: [PATCH 044/128] dosc openbsd: add missing acquire certificate instruction for httpd --- docs/installation/openbsd_en.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/installation/openbsd_en.md b/docs/installation/openbsd_en.md index 8aaa6e8de..d5df310cc 100644 --- a/docs/installation/openbsd_en.md +++ b/docs/installation/openbsd_en.md @@ -281,6 +281,12 @@ If the configuration is correct, enable and start the `httpd` service: # rcctl start httpd ``` +Acquire certificate: + +``` +# acme-client example.tld +``` + #### relayd ***Skip this section when using nginx*** From a323701c3369650736692e360d95f162d62df71f Mon Sep 17 00:00:00 2001 From: Phantasm Date: Wed, 27 Nov 2024 22:21:00 +0100 Subject: [PATCH 045/128] docs openbsd: spellcheck --- docs/installation/openbsd_en.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/installation/openbsd_en.md b/docs/installation/openbsd_en.md index d5df310cc..1135f838c 100644 --- a/docs/installation/openbsd_en.md +++ b/docs/installation/openbsd_en.md @@ -264,7 +264,7 @@ As root, copy `/home/_pleroma/pleroma/installation/openbsd/httpd.conf` to `/etc/ Edit `/etc/httpd.conf` settings and change: * `` with your instance's IPv4 address - * All occurances of `example.tld` with your instance's domain name + * All occurrences of `example.tld` with your instance's domain name * When using IPv6 also change: - Uncomment the `ext_inet6=""` line near the beginning of the file and change `` with your instance's IPv4 address - * All occurances of `example.tld` with your instance's domain name + * All occurrences of `example.tld` with your instance's domain name * When using IPv6 also change: - Uncomment the `ext_inet6=""` line near the beginning of the file and change `` to your instance's IPv6 address - Uncomment the line starting with `listen on $ext_inet6` in the `relay wwwtls` block From 047916445be61d2d86064e22a8acc22b6c017f5b Mon Sep 17 00:00:00 2001 From: Phantasm Date: Fri, 29 Nov 2024 16:00:52 +0100 Subject: [PATCH 046/128] docs openbsd: No need to switch users when creating DB --- docs/installation/openbsd_en.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/docs/installation/openbsd_en.md b/docs/installation/openbsd_en.md index 1135f838c..a98e6022a 100644 --- a/docs/installation/openbsd_en.md +++ b/docs/installation/openbsd_en.md @@ -107,14 +107,12 @@ Note: Answer yes when asked to install Hex and rebar3. This step might take some Create the Pleroma database: ``` -# psql -U postgres -f /home/_pleroma/pleroma/config/setup_db.psql +$ psql -U postgres -f config/setup_db.psql ``` -Switch back to the \_pleroma user and apply database migrations: +Apply database migrations: ``` -# su -l _pleroma -$ cd pleroma $ MIX_ENV=prod mix ecto.migrate ``` @@ -343,9 +341,10 @@ Enable and start the pleroma service: ### Create administrative user -If your instance is up and running, you can create your first user with administrative rights with the following command as the \_pleroma user: +If your instance is up and running, you can create your first user with administrative rights with the following commands as the \_pleroma user: ``` +$ cd pleroma $ MIX_ENV=prod mix pleroma.user new --admin ``` From 0a34e39569c3731a09968e9b51f5e52ac3d06216 Mon Sep 17 00:00:00 2001 From: Phantasm Date: Wed, 5 Feb 2025 23:23:35 +0100 Subject: [PATCH 047/128] docs openbsd: fix certificate acquisition on nginx --- docs/installation/openbsd_en.md | 35 +++++++++++++++++++++++++++++---- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/docs/installation/openbsd_en.md b/docs/installation/openbsd_en.md index a98e6022a..387b0f2ea 100644 --- a/docs/installation/openbsd_en.md +++ b/docs/installation/openbsd_en.md @@ -19,7 +19,8 @@ To install required packages, run the following command: # pkg_add erlang%26 elixir gmake git postgresql-server postgresql-contrib cmake libmagic libvips ``` -Pleroma requires a reverse proxy, OpenBSD has relayd in base (and is used in this guide) and packages/ports are available for nginx (www/nginx) and apache (www/apache-httpd). Independently of the reverse proxy, [acme-client(1)](https://man.openbsd.org/acme-client) can be used to get a certificate from Let's Encrypt. +Pleroma requires a reverse proxy, OpenBSD has relayd in base (and is used in this guide) and packages/ports are available for nginx (www/nginx) and apache (www/apache-httpd). +Independently of the reverse proxy, [acme-client(1)](https://man.openbsd.org/acme-client) can be used to get a certificate from Let's Encrypt. #### Optional software @@ -119,7 +120,8 @@ $ MIX_ENV=prod mix ecto.migrate Note: You will need to run this step again when updating your instance to a newer version with `git pull` or `git checkout tags/NEW_VERSION`. As \_pleroma in /home/\_pleroma/pleroma, you can now run `MIX_ENV=prod mix phx.server` to start your instance. -In another SSH session or a tmux window, check that it is working properly by running `ftp -MVo - http://127.0.0.1:4000/api/v1/instance`, you should get json output. Double-check that the *uri* value near the bottom is your instance's domain name and the instance *title* are correct. +In another SSH session or a tmux window, check that it is working properly by running `ftp -MVo - http://127.0.0.1:4000/api/v1/instance`, you should get json output. +Double-check that the *uri* value near the bottom is your instance's domain name and the instance *title* are correct. ### Configuring acme-client @@ -176,10 +178,10 @@ http { server { ... - server_name example.tld; # Replace with your domain + server_name localhost; # Replace with your domain location /.well-known/acme-challenge { - rewrite ^/.well-known/acme-challenge/(.*) /$1 break; + rewrite ^/\.well-known/acme-challenge/(.*) /$1 break; root /var/www/acme; } } @@ -225,10 +227,32 @@ As root, copy `/home/_pleroma/pleroma/installation/pleroma.nginx` to `/etc/nginx Edit default `/etc/nginx/sites-available/pleroma.nginx` settings and replace `example.tld` with your domain: + * Uncomment the location block for `~ /\.well-known/acme-challenge` in the server block listening on port 80 + - add `rewrite ^/\.well-known/acme-challenge/(.*) /$1 break;` above the `root` location + - change the `root` location to `/var/www/acme;` * Change `ssl_trusted_certificate` to `/etc/ssl/example.tld_cert-only.crt` * Change `ssl_certificate` to `/etc/ssl/example.tld.crt` * Change `ssl_certificate_key` to `/etc/ssl/private/example.tld.key` +Remove the following `location {}` block from `/etc/nginx/nginx.conf`, that was previously added for acquiring certificates and change `server_name` back to `localhost`: + +``` +http { + ... + + server { + ... + server_name example.tld; # Change back to localhost + + # Delete this block + location /.well-known/acme-challenge { + rewrite ^/\.well-known/acme-challenge/(.*) /$1 break; + root /var/www/acme; + } + } +} +``` + Symlink the Pleroma configuration to the enabled sites: ``` @@ -241,6 +265,9 @@ Check nginx configuration syntax by running: # nginx -t ``` +Note: If the above command complains about a `conflicting server name`, check again that the `location {}` block for acquiring certificates has been removed from `/etc/nginx/nginx.conf` and that the `server_name` has been reverted back to `localhost`. +After doing so run `# nginx -t` again. + If the configuration is correct, you can now enable and reload the nginx service: ``` From d0dac30ac6e6808e772a5ec0b378e2b8294fc93a Mon Sep 17 00:00:00 2001 From: mkljczk Date: Sat, 22 Feb 2025 15:53:44 +0100 Subject: [PATCH 048/128] Merge downstream changes Signed-off-by: mkljczk --- lib/pleroma/application_requirements.ex | 18 ++++++++++++++- lib/pleroma/language/translation.ex | 22 +++++++++++++++++++ lib/pleroma/language/translation/deepl.ex | 2 ++ .../language/translation/libretranslate.ex | 4 +++- lib/pleroma/language/translation/provider.ex | 13 +++++++++++ .../{deepl_test.ex => deepl_test.exs} | 0 ...anslation_test.ex => translation_test.exs} | 1 - 7 files changed, 57 insertions(+), 3 deletions(-) rename test/pleroma/language/translation/{deepl_test.ex => deepl_test.exs} (100%) rename test/pleroma/language/{translation_test.ex => translation_test.exs} (94%) diff --git a/lib/pleroma/application_requirements.ex b/lib/pleroma/application_requirements.ex index a334d12ee..211b4882d 100644 --- a/lib/pleroma/application_requirements.ex +++ b/lib/pleroma/application_requirements.ex @@ -189,7 +189,23 @@ defmodule Pleroma.ApplicationRequirements do false end - if Enum.all?([preview_proxy_commands_status | filter_commands_statuses], & &1) do + translation_commands_status = + if Pleroma.Language.Translation.missing_dependencies() == [] do + true + else + Logger.error( + "The following dependencies required by the currently enabled " <> + "translation provider are not installed: " <> + inspect(Pleroma.Language.Translation.missing_dependencies()) + ) + + false + end + + if Enum.all?( + [preview_proxy_commands_status, translation_commands_status | filter_commands_statuses], + & &1 + ) do :ok else {:error, diff --git a/lib/pleroma/language/translation.ex b/lib/pleroma/language/translation.ex index e4916389d..3706e76eb 100644 --- a/lib/pleroma/language/translation.ex +++ b/lib/pleroma/language/translation.ex @@ -11,6 +11,16 @@ defmodule Pleroma.Language.Translation do !!provider and provider.configured? end + def missing_dependencies do + provider = get_provider() + + if provider do + provider.missing_dependencies() + else + [] + end + end + def translate(text, source_language, target_language) do cache_key = get_cache_key(text, source_language, target_language) @@ -23,6 +33,7 @@ defmodule Pleroma.Language.Translation do {:error, :not_found} else provider.translate(text, source_language, target_language) + |> scrub_html() end store_result(result, cache_key) @@ -102,4 +113,15 @@ defmodule Pleroma.Language.Translation do defp store_result(_, _), do: nil defp content_hash(text), do: :crypto.hash(:sha256, text) |> Base.encode64() + + defp scrub_html({:ok, %{content: content} = result}) when is_binary(content) do + scrubbers = Pleroma.Config.get([:markup, :scrub_policy]) + + content + |> Pleroma.HTML.filter_tags(scrubbers) + + {:ok, %{result | content: content}} + end + + defp scrub_html(result), do: result end diff --git a/lib/pleroma/language/translation/deepl.ex b/lib/pleroma/language/translation/deepl.ex index 4f668fbba..e027035b4 100644 --- a/lib/pleroma/language/translation/deepl.ex +++ b/lib/pleroma/language/translation/deepl.ex @@ -7,6 +7,8 @@ defmodule Pleroma.Language.Translation.Deepl do alias Pleroma.Language.Translation.Provider + use Provider + @behaviour Provider @name "DeepL" diff --git a/lib/pleroma/language/translation/libretranslate.ex b/lib/pleroma/language/translation/libretranslate.ex index b793b166e..fd727d1cf 100644 --- a/lib/pleroma/language/translation/libretranslate.ex +++ b/lib/pleroma/language/translation/libretranslate.ex @@ -7,6 +7,8 @@ defmodule Pleroma.Language.Translation.Libretranslate do alias Pleroma.Language.Translation.Provider + use Provider + @behaviour Provider @name "LibreTranslate" @@ -44,7 +46,7 @@ defmodule Pleroma.Language.Translation.Libretranslate do %{ content: content, detected_source_language: source_language, - provider: "LibreTranslate" + provider: @name }} _ -> diff --git a/lib/pleroma/language/translation/provider.ex b/lib/pleroma/language/translation/provider.ex index f12cba2cd..533b5355a 100644 --- a/lib/pleroma/language/translation/provider.ex +++ b/lib/pleroma/language/translation/provider.ex @@ -3,6 +3,10 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Language.Translation.Provider do + alias Pleroma.Language.Translation.Provider + + @callback missing_dependencies() :: [String.t()] + @callback configured?() :: boolean() @callback translate( @@ -24,4 +28,13 @@ defmodule Pleroma.Language.Translation.Provider do @callback languages_matrix() :: {:ok, Map.t()} | {:error, atom()} @callback name() :: String.t() + + defmacro __using__(_opts) do + quote do + @impl Provider + def missing_dependencies, do: [] + + defoverridable missing_dependencies: 0 + end + end end diff --git a/test/pleroma/language/translation/deepl_test.ex b/test/pleroma/language/translation/deepl_test.exs similarity index 100% rename from test/pleroma/language/translation/deepl_test.ex rename to test/pleroma/language/translation/deepl_test.exs diff --git a/test/pleroma/language/translation_test.ex b/test/pleroma/language/translation_test.exs similarity index 94% rename from test/pleroma/language/translation_test.ex rename to test/pleroma/language/translation_test.exs index ecab3d20f..0be7a8d60 100644 --- a/test/pleroma/language/translation_test.ex +++ b/test/pleroma/language/translation_test.exs @@ -2,7 +2,6 @@ defmodule Pleroma.Language.TranslationTest do use Pleroma.Web.ConnCase alias Pleroma.Language.Translation - # use Oban.Testing, repo: Pleroma.Repo setup do: clear_config([Pleroma.Language.Translation, :provider], TranslationMock) From 22bbe55b55eeb3766f31c357043f97d9acd1d8dd Mon Sep 17 00:00:00 2001 From: mkljczk Date: Sat, 22 Feb 2025 16:03:05 +0100 Subject: [PATCH 049/128] fix Signed-off-by: mkljczk --- test/support/translation_mock.ex | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/support/translation_mock.ex b/test/support/translation_mock.ex index 95da738d1..84ed8f696 100644 --- a/test/support/translation_mock.ex +++ b/test/support/translation_mock.ex @@ -5,6 +5,8 @@ defmodule TranslationMock do alias Pleroma.Language.Translation.Provider + use Provider + @behaviour Provider @name "TranslationMock" From d7f9d30b2cad51fa2a9acb1ae02091b3140f829b Mon Sep 17 00:00:00 2001 From: mkljczk Date: Sat, 22 Feb 2025 16:01:50 +0100 Subject: [PATCH 050/128] Merge downstream changes Signed-off-by: mkljczk --- lib/pleroma/language/language_detector.ex | 16 +++++++++++++++- .../web/mastodon_api/views/instance_view.ex | 2 +- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/language/language_detector.ex b/lib/pleroma/language/language_detector.ex index 42d200a28..2efe22d5e 100644 --- a/lib/pleroma/language/language_detector.ex +++ b/lib/pleroma/language/language_detector.ex @@ -3,8 +3,17 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Language.LanguageDetector do + import Pleroma.EctoType.ActivityPub.ObjectValidators.LanguageCode, + only: [good_locale_code?: 1] + @words_threshold 4 + def configured? do + provider = get_provider() + + !!provider and provider.configured? + end + def missing_dependencies do provider = get_provider() @@ -34,7 +43,12 @@ defmodule Pleroma.Language.LanguageDetector do if word_count < @words_threshold or !provider or !provider.configured? do nil else - provider.detect(text) + with language <- provider.detect(text), + true <- good_locale_code?(language) do + language + else + _ -> nil + end end end diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index d9cff1504..00ca06243 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -146,7 +146,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do "pleroma:get:main/ostatus", "pleroma:group_actors", "pleroma:bookmark_folders", - if Config.get([Pleroma.Language.LanguageDetector, :provider]) do + if Pleroma.Language.LanguageDetector.configured?() do "pleroma:language_detection" end ] From 3b74d13147d8b8cef5cf487a75e45eb541899f54 Mon Sep 17 00:00:00 2001 From: mkljczk Date: Sat, 22 Feb 2025 18:31:26 +0100 Subject: [PATCH 051/128] Do not call LanguageDetector when not language is provided Signed-off-by: mkljczk --- .../object_validators/common_fixes.ex | 20 +++++++++++----- .../article_note_page_validator_test.exs | 23 +++++++++++++++++++ 2 files changed, 37 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex index e5f3b0589..f0f3fef90 100644 --- a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex +++ b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex @@ -152,11 +152,19 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes do def maybe_add_language(object) do language = [ - get_language_from_context(object), - get_language_from_content_map(object), - get_language_from_content(object) + &get_language_from_context/1, + &get_language_from_content_map/1, + &get_language_from_content/1 ] - |> Enum.find(&good_locale_code?(&1)) + |> Enum.find_value(fn get_language -> + language = get_language.(object) + + if good_locale_code?(language) do + language + else + nil + end + end) if language do Map.put(object, "language", language) @@ -189,8 +197,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes do defp get_language_from_content_map(_), do: nil - defp get_language_from_content(%{"summary" => summary, "content" => content}) do - LanguageDetector.detect("#{summary} #{content}") + defp get_language_from_content(%{"content" => content} = object) do + LanguageDetector.detect("#{object["summary"] || ""} #{content}") end defp get_language_from_content(_), do: nil diff --git a/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs b/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs index 829598246..b64c554d8 100644 --- a/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs +++ b/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs @@ -9,6 +9,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidatorTest alias Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidator alias Pleroma.Web.ActivityPub.Utils + import Mock import Pleroma.Factory describe "Notes" do @@ -234,6 +235,28 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidatorTest assert object.language == "pl" end + test_with_mock "it doesn't call LanguageDetector when language is specified", + Pleroma.Language.LanguageDetector, + detect: fn _ -> nil end do + user = insert(:user) + + note = %{ + "to" => ["https://www.w3.org/ns/activitystreams#Public"], + "cc" => [], + "id" => Utils.generate_object_id(), + "type" => "Note", + "content" => "a post in English", + "contentMap" => %{ + "en" => "a post in English" + }, + "attributedTo" => user.ap_id + } + + ArticleNotePageValidator.cast_and_apply(note) + + refute called(Pleroma.Language.LanguageDetector.detect(:_)) + end + test "it adds contentMap if language is specified" do user = insert(:user) From 7b69e525643da749afbe4f6fa0bd59cbd6dcc923 Mon Sep 17 00:00:00 2001 From: tusooa Date: Sun, 23 Feb 2025 21:12:08 -0500 Subject: [PATCH 052/128] Fix AssignAppUser migration OOM --- changelog.d/assign-app-user-oom.fix | 1 + .../20240904142434_assign_app_user.exs | 18 +++++++++++------- 2 files changed, 12 insertions(+), 7 deletions(-) create mode 100644 changelog.d/assign-app-user-oom.fix diff --git a/changelog.d/assign-app-user-oom.fix b/changelog.d/assign-app-user-oom.fix new file mode 100644 index 000000000..ac1de7159 --- /dev/null +++ b/changelog.d/assign-app-user-oom.fix @@ -0,0 +1 @@ +Fix AssignAppUser migration OOM diff --git a/priv/repo/migrations/20240904142434_assign_app_user.exs b/priv/repo/migrations/20240904142434_assign_app_user.exs index 11bec529b..74740220d 100644 --- a/priv/repo/migrations/20240904142434_assign_app_user.exs +++ b/priv/repo/migrations/20240904142434_assign_app_user.exs @@ -1,20 +1,24 @@ defmodule Pleroma.Repo.Migrations.AssignAppUser do use Ecto.Migration + import Ecto.Query + alias Pleroma.Repo alias Pleroma.Web.OAuth.App alias Pleroma.Web.OAuth.Token def up do - Repo.all(Token) - |> Enum.group_by(fn x -> Map.get(x, :app_id) end) - |> Enum.each(fn {_app_id, tokens} -> - token = - Enum.filter(tokens, fn x -> not is_nil(x.user_id) end) - |> List.first() - + Token + |> where([t], not is_nil(t.user_id)) + |> group_by([t], t.app_id) + |> select([t], %{app_id: t.app_id, id: min(t.id)}) + |> order_by(asc: :app_id) + |> Repo.stream() + |> Stream.each(fn %{id: id} -> + token = Token.Query.get_by_id(id) |> Repo.one() App.maybe_update_owner(token) end) + |> Stream.run() end def down, do: :ok From ccc6f2b288adf10d0206a63785136def934b2f98 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 25 Feb 2025 12:19:49 +0400 Subject: [PATCH 053/128] Docs: Add mox testing info --- docs/development/index.md | 6 + docs/development/mox_testing.md | 406 ++++++++++++++++++++++++++++++++ 2 files changed, 412 insertions(+) create mode 100644 docs/development/mox_testing.md diff --git a/docs/development/index.md b/docs/development/index.md index 01a617596..6b35321c5 100644 --- a/docs/development/index.md +++ b/docs/development/index.md @@ -1 +1,7 @@ This section contains notes and guidelines for developers. + +- [Setting up a Pleroma development environment](setting_up_pleroma_dev.md) +- [Setting up a Gitlab Runner](setting_up_a_gitlab_runner.md) +- [Authentication & Authorization](authentication_authorization.md) +- [ActivityPub Extensions](ap_extensions.md) +- [Mox Testing Guide](mox_testing.md) diff --git a/docs/development/mox_testing.md b/docs/development/mox_testing.md new file mode 100644 index 000000000..7e04ee52e --- /dev/null +++ b/docs/development/mox_testing.md @@ -0,0 +1,406 @@ +# Using Mox for Testing in Pleroma + +## Introduction + +This guide explains how to use [Mox](https://hexdocs.pm/mox/Mox.html) for testing in Pleroma and how to migrate existing tests from Mock/meck to Mox. Mox is a library for defining concurrent mocks in Elixir that offers several key advantages: + +- **Async-safe testing**: Mox supports concurrent testing with `async: true` +- **Explicit contract through behaviors**: Enforces implementation of behavior callbacks +- **No module redefinition**: Avoids runtime issues caused by redefining modules +- **Expectations scoped to the current process**: Prevents test state from leaking between tests + +## Why Migrate from Mock/meck to Mox? + +### Problems with Mock/meck + +1. **Not async-safe**: Tests using Mock/meck cannot safely run with `async: true`, which slows down the test suite +2. **Global state**: Mocked functions are global, leading to potential cross-test contamination +3. **No explicit contract**: No guarantee that mocked functions match the actual implementation +4. **Module redefinition**: Can lead to hard-to-debug runtime issues + +### Benefits of Mox + +1. **Async-safe testing**: Tests can run concurrently with `async: true`, significantly speeding up the test suite +2. **Process isolation**: Expectations are set per process, preventing leakage between tests +3. **Explicit contracts via behaviors**: Ensures mocks implement all required functions +4. **Compile-time checks**: Prevents mocking non-existent functions +5. **No module redefinition**: Mocks are defined at compile time, not runtime + +## Existing Mox Setup in Pleroma + +Pleroma already has a basic Mox setup in the `Pleroma.DataCase` module, which handles some common mocking scenarios automatically. Here's what's included: + +### Default Mox Configuration + +The `setup` function in `DataCase` does the following: + +1. Sets up Mox for either async or non-async tests +2. Verifies all mock expectations on test exit +3. Stubs common dependencies with their real implementations + +```elixir +# From test/support/data_case.ex +setup tags do + setup_multi_process_mode(tags) + setup_streamer(tags) + stub_pipeline() + + Mox.verify_on_exit!() + + :ok +end +``` + +### Async vs. Non-Async Test Setup + +Pleroma configures Mox differently depending on whether your test is async or not: + +```elixir +def setup_multi_process_mode(tags) do + :ok = Ecto.Adapters.SQL.Sandbox.checkout(Pleroma.Repo) + + if tags[:async] do + # For async tests, use process-specific mocks and stub CachexMock with NullCache + Mox.stub_with(Pleroma.CachexMock, Pleroma.NullCache) + Mox.set_mox_private() + else + # For non-async tests, use global mocks and stub CachexMock with CachexProxy + Ecto.Adapters.SQL.Sandbox.mode(Pleroma.Repo, {:shared, self()}) + + Mox.set_mox_global() + Mox.stub_with(Pleroma.CachexMock, Pleroma.CachexProxy) + clear_cachex() + end + + :ok +end +``` + +### Default Pipeline Stubs + +Pleroma automatically stubs several core components with their real implementations: + +```elixir +def stub_pipeline do + Mox.stub_with(Pleroma.Web.ActivityPub.SideEffectsMock, Pleroma.Web.ActivityPub.SideEffects) + Mox.stub_with(Pleroma.Web.ActivityPub.ObjectValidatorMock, Pleroma.Web.ActivityPub.ObjectValidator) + Mox.stub_with(Pleroma.Web.ActivityPub.MRFMock, Pleroma.Web.ActivityPub.MRF) + Mox.stub_with(Pleroma.Web.ActivityPub.ActivityPubMock, Pleroma.Web.ActivityPub.ActivityPub) + Mox.stub_with(Pleroma.Web.FederatorMock, Pleroma.Web.Federator) + Mox.stub_with(Pleroma.ConfigMock, Pleroma.Config) + Mox.stub_with(Pleroma.StaticStubbedConfigMock, Pleroma.Test.StaticConfig) + Mox.stub_with(Pleroma.StubbedHTTPSignaturesMock, Pleroma.Test.HTTPSignaturesProxy) +end +``` + +This means that by default, these mocks will behave like their real implementations unless you explicitly override them with expectations in your tests. + +## Configuration in Async Tests + +### Understanding `clear_config` Limitations + +The `clear_config` helper is commonly used in Pleroma tests to modify configuration for specific tests. However, it's important to understand that **`clear_config` is not async-safe** and should not be used in tests with `async: true`. + +Here's why: + +```elixir +# Implementation of clear_config in test/support/helpers.ex +defmacro clear_config(config_path, temp_setting) do + quote do + clear_config(unquote(config_path)) do + Config.put(unquote(config_path), unquote(temp_setting)) + end + end +end + +defmacro clear_config(config_path, do: yield) do + quote do + initial_setting = Config.fetch(unquote(config_path)) + + unquote(yield) + + on_exit(fn -> + case initial_setting do + :error -> + Config.delete(unquote(config_path)) + + {:ok, value} -> + Config.put(unquote(config_path), value) + end + end) + + :ok + end +end +``` + +The issue is that `clear_config`: +1. Modifies the global application environment +2. Uses `on_exit` to restore the original value after the test +3. Can lead to race conditions when multiple async tests modify the same configuration + +### Async-Safe Configuration Approaches + +When writing async tests with Mox, use these approaches instead of `clear_config`: + +1. **Dependency Injection with Module Attributes**: + ```elixir + # In your module + @config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config) + + def some_function do + value = @config_impl.get([:some, :config]) + # ... + end + ``` + +2. **Mock the Config Module**: + ```elixir + # In your test + Pleroma.ConfigMock + |> expect(:get, fn [:some, :config] -> "test_value" end) + ``` + +3. **Use Test-Specific Implementations**: + ```elixir + # Define a test-specific implementation + defmodule TestConfig do + def get([:some, :config]), do: "test_value" + def get(_), do: nil + end + + # In your test + Mox.stub_with(Pleroma.ConfigMock, TestConfig) + ``` + +4. **Pass Configuration as Arguments**: + ```elixir + # Refactor functions to accept configuration as arguments + def some_function(config \\ nil) do + config = config || Pleroma.Config.get([:some, :config]) + # ... + end + + # In your test + some_function("test_value") + ``` + +By using these approaches, you can safely run tests with `async: true` without worrying about configuration conflicts. + +## Setting Up Mox in Pleroma + +### Step 1: Define a Behavior + +Start by defining a behavior for the module you want to mock. This specifies the contract that both the real implementation and mocks must follow. + +```elixir +# In your implementation module (e.g., lib/pleroma/uploaders/s3.ex) +defmodule Pleroma.Uploaders.S3.ExAwsAPI do + @callback request(op :: ExAws.Operation.t()) :: {:ok, ExAws.Operation.t()} | {:error, term()} +end +``` + +### Step 2: Make Your Implementation Configurable + +Modify your module to use a configurable implementation. This allows for dependency injection and easier testing. + +```elixir +# In your implementation module +@ex_aws_impl Application.compile_env(:pleroma, [__MODULE__, :ex_aws_impl], ExAws) +@config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config) + +def put_file(%Pleroma.Upload{} = upload) do + # Use @ex_aws_impl instead of ExAws directly + case @ex_aws_impl.request(op) do + {:ok, _} -> + {:ok, {:file, s3_name}} + + error -> + Logger.error("#{__MODULE__}: #{inspect(error)}") + error + end +end +``` + +### Step 3: Define the Mock in test/support/mocks.ex + +Add your mock definition in the central mocks file: + +```elixir +# In test/support/mocks.ex +Mox.defmock(Pleroma.Uploaders.S3.ExAwsMock, for: Pleroma.Uploaders.S3.ExAwsAPI) +``` + +### Step 4: Configure the Mock in Test Environment + +In your test configuration (e.g., `config/test.exs`), specify which mock implementation to use: + +```elixir +config :pleroma, Pleroma.Uploaders.S3, ex_aws_impl: Pleroma.Uploaders.S3.ExAwsMock +config :pleroma, Pleroma.Uploaders.S3, config_impl: Pleroma.UnstubbedConfigMock +``` + +## Writing Tests with Mox + +### Setting Up Your Test + +```elixir +defmodule Pleroma.Uploaders.S3Test do + use Pleroma.DataCase, async: true # Note: async: true is now possible! + + alias Pleroma.Uploaders.S3 + alias Pleroma.Uploaders.S3.ExAwsMock + alias Pleroma.UnstubbedConfigMock, as: ConfigMock + + import Mox # Import Mox functions + + # Note: verify_on_exit! is already called in DataCase setup + # so you don't need to add it explicitly in your test module +end +``` + +### Setting Expectations with Mox + +Mox uses an explicit expectation system. Here's how to use it: + +```elixir +# Basic expectation for a function call +ExAwsMock +|> expect(:request, fn _req -> {:ok, %{status_code: 200}} end) + +# Expectation for multiple calls with same response +ExAwsMock +|> expect(:request, 3, fn _req -> {:ok, %{status_code: 200}} end) + +# Expectation with specific arguments +ExAwsMock +|> expect(:request, fn %{bucket: "test_bucket"} -> {:ok, %{status_code: 200}} end) + +# Complex configuration mocking +ConfigMock +|> expect(:get, fn key -> + [ + {Pleroma.Upload, [uploader: Pleroma.Uploaders.S3, base_url: "https://s3.amazonaws.com"]}, + {Pleroma.Uploaders.S3, [bucket: "test_bucket"]} + ] + |> get_in(key) +end) +``` + +### Understanding Mox Modes in Pleroma + +Pleroma's DataCase automatically configures Mox differently based on whether your test is async or not: + +1. **Async tests** (`async: true`): + - Uses `Mox.set_mox_private()` - expectations are scoped to the current process + - Stubs `Pleroma.CachexMock` with `Pleroma.NullCache` + - Each test process has its own isolated mock expectations + +2. **Non-async tests** (`async: false`): + - Uses `Mox.set_mox_global()` - expectations are shared across processes + - Stubs `Pleroma.CachexMock` with `Pleroma.CachexProxy` + - Mock expectations can be set in one process and called from another + +Choose the appropriate mode based on your test requirements. For most tests, async mode is preferred for better performance. + +## Migrating from Mock/meck to Mox + +Here's a step-by-step guide for migrating existing tests from Mock/meck to Mox: + +### 1. Identify the Module to Mock + +Look for `with_mock` or `test_with_mock` calls in your tests: + +```elixir +# Old approach with Mock +with_mock ExAws, request: fn _ -> {:ok, :ok} end do + assert S3.put_file(file_upload) == {:ok, {:file, "test_folder/image-tet.jpg"}} +end +``` + +### 2. Define a Behavior for the Module + +Create a behavior that defines the functions you want to mock: + +```elixir +defmodule Pleroma.Uploaders.S3.ExAwsAPI do + @callback request(op :: ExAws.Operation.t()) :: {:ok, ExAws.Operation.t()} | {:error, term()} +end +``` + +### 3. Update Your Implementation to Use a Configurable Dependency + +```elixir +# Old +def put_file(%Pleroma.Upload{} = upload) do + case ExAws.request(op) do + # ... + end +end + +# New +@ex_aws_impl Application.compile_env(:pleroma, [__MODULE__, :ex_aws_impl], ExAws) + +def put_file(%Pleroma.Upload{} = upload) do + case @ex_aws_impl.request(op) do + # ... + end +end +``` + +### 4. Define the Mock in mocks.ex + +```elixir +Mox.defmock(Pleroma.Uploaders.S3.ExAwsMock, for: Pleroma.Uploaders.S3.ExAwsAPI) +``` + +### 5. Configure the Test Environment + +```elixir +config :pleroma, Pleroma.Uploaders.S3, ex_aws_impl: Pleroma.Uploaders.S3.ExAwsMock +``` + +### 6. Update Your Tests to Use Mox + +```elixir +# Old (with Mock) +test_with_mock "save file", ExAws, request: fn _ -> {:ok, :ok} end do + assert S3.put_file(file_upload) == {:ok, {:file, "test_folder/image-tet.jpg"}} + assert_called(ExAws.request(:_)) +end + +# New (with Mox) +test "save file" do + ExAwsMock + |> expect(:request, fn _req -> {:ok, %{status_code: 200}} end) + + assert S3.put_file(file_upload) == {:ok, {:file, "test_folder/image-tet.jpg"}} +end +``` + +### 7. Enable Async Testing + +Now you can safely enable `async: true` in your test module: + +```elixir +use Pleroma.DataCase, async: true +``` + +## Best Practices + +1. **Always define behaviors**: They serve as contracts and documentation +2. **Keep mocks in a central location**: Use test/support/mocks.ex for all mock definitions +3. **Use verify_on_exit!**: This is already set up in DataCase, ensuring all expected calls were made +4. **Use specific expectations**: Be as specific as possible with your expectations +5. **Enable async: true**: Take advantage of Mox's concurrent testing capability +6. **Don't over-mock**: Only mock external dependencies that are difficult to test directly +7. **Leverage existing stubs**: Use the default stubs provided by DataCase when possible +8. **Avoid clear_config in async tests**: Use dependency injection and mocking instead + +## Example: Complete Migration + +For a complete example of migrating a test from Mock/meck to Mox, you can refer to commit `90a47ca050c5839e8b4dc3bac315dc436d49152d` in the Pleroma repository, which shows how the S3 uploader tests were migrated. + +## Conclusion + +Migrating tests from Mock/meck to Mox provides significant benefits for the Pleroma test suite, including faster test execution through async testing, better isolation between tests, and more robust mocking through explicit contracts. By following this guide, you can successfully migrate existing tests and write new tests using Mox. \ No newline at end of file From edfb1deb1c578e48c0cbeebb1961c90b76b28beb Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 25 Feb 2025 12:20:19 +0400 Subject: [PATCH 054/128] Application: Don't verify requirements during test at startup. --- lib/pleroma/application.ex | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 3f199c002..d7975d2d1 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -56,7 +56,10 @@ defmodule Pleroma.Application do Pleroma.Web.Plugs.HTTPSecurityPlug.warn_if_disabled() end - Pleroma.ApplicationRequirements.verify!() + if Mix.env() != :test do + Pleroma.ApplicationRequirements.verify!() + end + load_custom_modules() Pleroma.Docs.JSON.compile() limiters_setup() From 35814de0dff761e347e6977afb40b80099df4f4e Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 25 Feb 2025 12:31:19 +0400 Subject: [PATCH 055/128] LanguageDetectorTests: Switch to mox --- config/test.exs | 1 + lib/pleroma/language/language_detector.ex | 3 +- .../language/language_detector_test.ex | 31 ++++++++++++-- .../article_note_page_validator_test.exs | 41 ++++++++++++++++--- test/support/language_detector_mock.ex | 18 -------- test/support/mocks.ex | 4 ++ 6 files changed, 70 insertions(+), 28 deletions(-) delete mode 100644 test/support/language_detector_mock.ex diff --git a/config/test.exs b/config/test.exs index 6fe84478a..141a206fa 100644 --- a/config/test.exs +++ b/config/test.exs @@ -152,6 +152,7 @@ config :pleroma, Pleroma.User.Backup, config_impl: Pleroma.UnstubbedConfigMock config :pleroma, Pleroma.Uploaders.S3, ex_aws_impl: Pleroma.Uploaders.S3.ExAwsMock config :pleroma, Pleroma.Uploaders.S3, config_impl: Pleroma.UnstubbedConfigMock config :pleroma, Pleroma.Upload, config_impl: Pleroma.UnstubbedConfigMock +config :pleroma, Pleroma.Language.LanguageDetector, config_impl: Pleroma.UnstubbedConfigMock config :pleroma, Pleroma.ScheduledActivity, config_impl: Pleroma.UnstubbedConfigMock config :pleroma, Pleroma.Web.RichMedia.Helpers, config_impl: Pleroma.StaticStubbedConfigMock config :pleroma, Pleroma.Uploaders.IPFS, config_impl: Pleroma.UnstubbedConfigMock diff --git a/lib/pleroma/language/language_detector.ex b/lib/pleroma/language/language_detector.ex index 2efe22d5e..16e2d4faa 100644 --- a/lib/pleroma/language/language_detector.ex +++ b/lib/pleroma/language/language_detector.ex @@ -7,6 +7,7 @@ defmodule Pleroma.Language.LanguageDetector do only: [good_locale_code?: 1] @words_threshold 4 + @config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config) def configured? do provider = get_provider() @@ -53,6 +54,6 @@ defmodule Pleroma.Language.LanguageDetector do end defp get_provider do - Pleroma.Config.get([__MODULE__, :provider]) + @config_impl.get([__MODULE__, :provider]) end end diff --git a/test/pleroma/language/language_detector_test.ex b/test/pleroma/language/language_detector_test.ex index 4d9af33bf..b867fca19 100644 --- a/test/pleroma/language/language_detector_test.ex +++ b/test/pleroma/language/language_detector_test.ex @@ -3,26 +3,51 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Language.LanguageDetectorTest do - use Pleroma.Web.ConnCase + use Pleroma.DataCase, async: true alias Pleroma.Language.LanguageDetector + alias Pleroma.Language.LanguageDetectorMock + alias Pleroma.UnstubbedConfigMock - setup do: clear_config([Pleroma.Language.LanguageDetector, :provider], LanguageDetectorMock) + import Mox + + setup do + # Stub the UnstubbedConfigMock to return our mock for the provider + UnstubbedConfigMock + |> stub(:get, fn + [Pleroma.Language.LanguageDetector, :provider] -> LanguageDetectorMock + _other -> nil + end) + + # Stub the LanguageDetectorMock with default implementations + LanguageDetectorMock + |> stub(:missing_dependencies, fn -> [] end) + |> stub(:configured?, fn -> true end) + + :ok + end test "it detects text language" do + LanguageDetectorMock + |> expect(:detect, fn _text -> "fr" end) + detected_language = LanguageDetector.detect("Je viens d'atterrir en Tchéquie.") assert detected_language == "fr" end test "it returns nil if text is not long enough" do + # No need to set expectations as the word count check happens before the provider is called + detected_language = LanguageDetector.detect("it returns nil") assert detected_language == nil end test "it returns nil if no provider specified" do - clear_config([Pleroma.Language.LanguageDetector, :provider], nil) + # Override the stub to return nil for the provider + UnstubbedConfigMock + |> expect(:get, fn [Pleroma.Language.LanguageDetector, :provider] -> nil end) detected_language = LanguageDetector.detect("this should also return nil") diff --git a/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs b/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs index b64c554d8..63e2b173a 100644 --- a/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs +++ b/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs @@ -8,10 +8,30 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidatorTest alias Pleroma.Web.ActivityPub.ObjectValidator alias Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidator alias Pleroma.Web.ActivityPub.Utils + alias Pleroma.Language.LanguageDetectorMock + alias Pleroma.UnstubbedConfigMock - import Mock + import Mox import Pleroma.Factory + # Setup for all tests + setup do + # Stub the UnstubbedConfigMock to return our mock for the provider + UnstubbedConfigMock + |> stub(:get, fn + [Pleroma.Language.LanguageDetector, :provider] -> LanguageDetectorMock + _other -> nil + end) + + # Stub the LanguageDetectorMock with default implementations + LanguageDetectorMock + |> stub(:missing_dependencies, fn -> [] end) + |> stub(:configured?, fn -> true end) + |> stub(:detect, fn _text -> nil end) + + :ok + end + describe "Notes" do setup do user = insert(:user) @@ -235,9 +255,20 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidatorTest assert object.language == "pl" end - test_with_mock "it doesn't call LanguageDetector when language is specified", - Pleroma.Language.LanguageDetector, - detect: fn _ -> nil end do + test "it doesn't call LanguageDetector when language is specified" do + # Set up expectation that detect should not be called + LanguageDetectorMock + |> expect(:detect, 0, fn _ -> flunk("LanguageDetector.detect should not be called") end) + |> stub(:missing_dependencies, fn -> [] end) + |> stub(:configured?, fn -> true end) + + # Stub the UnstubbedConfigMock to return our mock for the provider + UnstubbedConfigMock + |> stub(:get, fn + [Pleroma.Language.LanguageDetector, :provider] -> LanguageDetectorMock + _other -> nil + end) + user = insert(:user) note = %{ @@ -253,8 +284,6 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidatorTest } ArticleNotePageValidator.cast_and_apply(note) - - refute called(Pleroma.Language.LanguageDetector.detect(:_)) end test "it adds contentMap if language is specified" do diff --git a/test/support/language_detector_mock.ex b/test/support/language_detector_mock.ex deleted file mode 100644 index 3e6a258ae..000000000 --- a/test/support/language_detector_mock.ex +++ /dev/null @@ -1,18 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2022 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule LanguageDetectorMock do - alias Pleroma.Language.LanguageDetector.Provider - - @behaviour Provider - - @impl Provider - def missing_dependencies, do: [] - - @impl Provider - def configured?, do: true - - @impl Provider - def detect(_text), do: "fr" -end diff --git a/test/support/mocks.ex b/test/support/mocks.ex index d84958e15..68b0de565 100644 --- a/test/support/mocks.ex +++ b/test/support/mocks.ex @@ -33,3 +33,7 @@ Mox.defmock(Pleroma.StubbedHTTPSignaturesMock, for: Pleroma.HTTPSignaturesAPI) Mox.defmock(Pleroma.LoggerMock, for: Pleroma.Logging) Mox.defmock(Pleroma.Uploaders.S3.ExAwsMock, for: Pleroma.Uploaders.S3.ExAwsAPI) + +Mox.defmock(Pleroma.Language.LanguageDetectorMock, + for: Pleroma.Language.LanguageDetector.Provider +) From 1e35ea785afc09a50c4d4a9b6e4a4624239ce2bf Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 25 Feb 2025 12:39:31 +0400 Subject: [PATCH 056/128] LanguageDetector: Use StaticStubbedConfigMock. --- config/test.exs | 2 +- test/pleroma/language/language_detector_test.ex | 8 ++++---- .../article_note_page_validator_test.exs | 10 +++++----- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/config/test.exs b/config/test.exs index 141a206fa..a20e720b6 100644 --- a/config/test.exs +++ b/config/test.exs @@ -152,7 +152,7 @@ config :pleroma, Pleroma.User.Backup, config_impl: Pleroma.UnstubbedConfigMock config :pleroma, Pleroma.Uploaders.S3, ex_aws_impl: Pleroma.Uploaders.S3.ExAwsMock config :pleroma, Pleroma.Uploaders.S3, config_impl: Pleroma.UnstubbedConfigMock config :pleroma, Pleroma.Upload, config_impl: Pleroma.UnstubbedConfigMock -config :pleroma, Pleroma.Language.LanguageDetector, config_impl: Pleroma.UnstubbedConfigMock +config :pleroma, Pleroma.Language.LanguageDetector, config_impl: Pleroma.StaticStubbedConfigMock config :pleroma, Pleroma.ScheduledActivity, config_impl: Pleroma.UnstubbedConfigMock config :pleroma, Pleroma.Web.RichMedia.Helpers, config_impl: Pleroma.StaticStubbedConfigMock config :pleroma, Pleroma.Uploaders.IPFS, config_impl: Pleroma.UnstubbedConfigMock diff --git a/test/pleroma/language/language_detector_test.ex b/test/pleroma/language/language_detector_test.ex index b867fca19..ccb81d5bd 100644 --- a/test/pleroma/language/language_detector_test.ex +++ b/test/pleroma/language/language_detector_test.ex @@ -7,13 +7,13 @@ defmodule Pleroma.Language.LanguageDetectorTest do alias Pleroma.Language.LanguageDetector alias Pleroma.Language.LanguageDetectorMock - alias Pleroma.UnstubbedConfigMock + alias Pleroma.StaticStubbedConfigMock import Mox setup do - # Stub the UnstubbedConfigMock to return our mock for the provider - UnstubbedConfigMock + # Stub the StaticStubbedConfigMock to return our mock for the provider + StaticStubbedConfigMock |> stub(:get, fn [Pleroma.Language.LanguageDetector, :provider] -> LanguageDetectorMock _other -> nil @@ -46,7 +46,7 @@ defmodule Pleroma.Language.LanguageDetectorTest do test "it returns nil if no provider specified" do # Override the stub to return nil for the provider - UnstubbedConfigMock + StaticStubbedConfigMock |> expect(:get, fn [Pleroma.Language.LanguageDetector, :provider] -> nil end) detected_language = LanguageDetector.detect("this should also return nil") diff --git a/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs b/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs index 63e2b173a..9bd792f25 100644 --- a/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs +++ b/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs @@ -9,15 +9,15 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidatorTest alias Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidator alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Language.LanguageDetectorMock - alias Pleroma.UnstubbedConfigMock + alias Pleroma.StaticStubbedConfigMock import Mox import Pleroma.Factory # Setup for all tests setup do - # Stub the UnstubbedConfigMock to return our mock for the provider - UnstubbedConfigMock + # Stub the StaticStubbedConfigMock to return our mock for the provider + StaticStubbedConfigMock |> stub(:get, fn [Pleroma.Language.LanguageDetector, :provider] -> LanguageDetectorMock _other -> nil @@ -262,8 +262,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidatorTest |> stub(:missing_dependencies, fn -> [] end) |> stub(:configured?, fn -> true end) - # Stub the UnstubbedConfigMock to return our mock for the provider - UnstubbedConfigMock + # Stub the StaticStubbedConfigMock to return our mock for the provider + StaticStubbedConfigMock |> stub(:get, fn [Pleroma.Language.LanguageDetector, :provider] -> LanguageDetectorMock _other -> nil From 584e4efaafad1b3890e2fc62a3b1debee795d8cb Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 25 Feb 2025 12:49:10 +0400 Subject: [PATCH 057/128] mox_testing.md: Update with more information --- docs/development/mox_testing.md | 79 +++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/docs/development/mox_testing.md b/docs/development/mox_testing.md index 7e04ee52e..673064022 100644 --- a/docs/development/mox_testing.md +++ b/docs/development/mox_testing.md @@ -95,6 +95,85 @@ end This means that by default, these mocks will behave like their real implementations unless you explicitly override them with expectations in your tests. +### Understanding Config Mock Types + +Pleroma has three different Config mock implementations, each with a specific purpose and different characteristics regarding async test safety: + +#### 1. ConfigMock + +- Defined in `test/support/mocks.ex` as `Mox.defmock(Pleroma.ConfigMock, for: Pleroma.Config.Getting)` +- It's stubbed with the real `Pleroma.Config` by default in `DataCase`: `Mox.stub_with(Pleroma.ConfigMock, Pleroma.Config)` +- This means it falls back to the normal configuration behavior unless explicitly overridden +- Used for general mocking of configuration in tests where you want most config to behave normally +- ⚠️ **NOT ASYNC-SAFE**: Since it's stubbed with the real `Pleroma.Config`, it modifies global application state +- Can not be used in tests with `async: true` + +#### 2. StaticStubbedConfigMock + +- Defined in `test/support/mocks.ex` as `Mox.defmock(Pleroma.StaticStubbedConfigMock, for: Pleroma.Config.Getting)` +- It's stubbed with `Pleroma.Test.StaticConfig` (defined in `test/test_helper.exs`) +- `Pleroma.Test.StaticConfig` creates a completely static configuration snapshot at the start of the test run: + ```elixir + defmodule Pleroma.Test.StaticConfig do + @moduledoc """ + This module provides a Config that is completely static, built at startup time from the environment. + It's safe to use in testing as it will not modify any state. + """ + + @behaviour Pleroma.Config.Getting + @config Application.get_all_env(:pleroma) + + def get(path, default \\ nil) do + get_in(@config, path) || default + end + end + ``` +- Configuration is frozen at startup time and doesn't change during the test run +- ✅ **ASYNC-SAFE**: Never modifies global state since it uses a frozen snapshot of the configuration + +#### 3. UnstubbedConfigMock + +- Defined in `test/support/mocks.ex` as `Mox.defmock(Pleroma.UnstubbedConfigMock, for: Pleroma.Config.Getting)` +- Unlike the other two mocks, it's not automatically stubbed with any implementation in `DataCase` +- Starts completely "unstubbed" and requires tests to explicitly set expectations or stub it +- The most commonly used configuration mock in the test suite +- Often aliased as `ConfigMock` in individual test files: `alias Pleroma.UnstubbedConfigMock, as: ConfigMock` +- Set as the default config implementation in `config/test.exs`: `config :pleroma, :config_impl, Pleroma.UnstubbedConfigMock` +- Offers maximum flexibility for tests that need precise control over configuration values +- ✅ **ASYNC-SAFE**: Safe if used with `expect()` to set up test-specific expectations (since expectations are process-scoped) + +#### Configuring Components to Use Specific Mocks + +In `config/test.exs`, different components can be configured to use different configuration mocks: + +```elixir +# Components using UnstubbedConfigMock +config :pleroma, Pleroma.Upload, config_impl: Pleroma.UnstubbedConfigMock +config :pleroma, Pleroma.User.Backup, config_impl: Pleroma.UnstubbedConfigMock +config :pleroma, Pleroma.Uploaders.S3, config_impl: Pleroma.UnstubbedConfigMock + +# Components using StaticStubbedConfigMock (async-safe) +config :pleroma, Pleroma.Language.LanguageDetector, config_impl: Pleroma.StaticStubbedConfigMock +config :pleroma, Pleroma.Web.RichMedia.Helpers, config_impl: Pleroma.StaticStubbedConfigMock +config :pleroma, Pleroma.Web.Plugs.HTTPSecurityPlug, config_impl: Pleroma.StaticStubbedConfigMock +``` + +This allows different parts of the application to use the most appropriate configuration mocking strategy based on their specific needs. + +#### When to Use Each Config Mock Type + +- **ConfigMock**: ⚠️ For non-async tests only, when you want most configuration to behave normally with occasional overrides +- **StaticStubbedConfigMock**: ✅ For async tests where modifying global state would be problematic and a static configuration is sufficient +- **UnstubbedConfigMock**: ⚠️ Use carefully in async tests; set specific expectations rather than stubbing with implementations that modify global state + +#### Summary of Async Safety + +| Mock Type | Async-Safe? | Best Use Case | +|-----------|-------------|--------------| +| ConfigMock | ❌ No | Non-async tests that need minimal configuration overrides | +| StaticStubbedConfigMock | ✅ Yes | Async tests that need configuration values without modification | +| UnstubbedConfigMock | ⚠️ Depends | Any test with careful usage; set expectations rather than stubbing | + ## Configuration in Async Tests ### Understanding `clear_config` Limitations From 7ccf3395238aebef1e15d70ccd0faf1d00c35635 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 25 Feb 2025 13:18:32 +0400 Subject: [PATCH 058/128] LanguageDetectorTest: Rename --- .../{language_detector_test.ex => language_detector_test.exs} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename test/pleroma/language/{language_detector_test.ex => language_detector_test.exs} (100%) diff --git a/test/pleroma/language/language_detector_test.ex b/test/pleroma/language/language_detector_test.exs similarity index 100% rename from test/pleroma/language/language_detector_test.ex rename to test/pleroma/language/language_detector_test.exs From d3e310d7697873e61358d810ec13bd226fbb7f51 Mon Sep 17 00:00:00 2001 From: mkljczk Date: Tue, 25 Feb 2025 10:40:51 +0100 Subject: [PATCH 059/128] Credo Signed-off-by: mkljczk --- .../object_validators/article_note_page_validator_test.exs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs b/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs index 9bd792f25..3c7ff0eeb 100644 --- a/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs +++ b/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs @@ -5,11 +5,11 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidatorTest do use Pleroma.DataCase, async: true + alias Pleroma.Language.LanguageDetectorMock + alias Pleroma.StaticStubbedConfigMock alias Pleroma.Web.ActivityPub.ObjectValidator alias Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidator alias Pleroma.Web.ActivityPub.Utils - alias Pleroma.Language.LanguageDetectorMock - alias Pleroma.StaticStubbedConfigMock import Mox import Pleroma.Factory From bee8b64fa79b74a8fa9a862956d80018eebc2966 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Fri, 14 Mar 2025 19:41:46 +0400 Subject: [PATCH 060/128] Migrations: Add activities_actor_type index --- changelog.d/activity_type_index.change | 1 + .../20250314153704_add_activities_actor_type_index.exs | 7 +++++++ 2 files changed, 8 insertions(+) create mode 100644 changelog.d/activity_type_index.change create mode 100644 priv/repo/migrations/20250314153704_add_activities_actor_type_index.exs diff --git a/changelog.d/activity_type_index.change b/changelog.d/activity_type_index.change new file mode 100644 index 000000000..ea2d7adbe --- /dev/null +++ b/changelog.d/activity_type_index.change @@ -0,0 +1 @@ +Add new activity actor/type index. Greatly speeds up retrieval of rare types (like "Listen") diff --git a/priv/repo/migrations/20250314153704_add_activities_actor_type_index.exs b/priv/repo/migrations/20250314153704_add_activities_actor_type_index.exs new file mode 100644 index 000000000..3713beea1 --- /dev/null +++ b/priv/repo/migrations/20250314153704_add_activities_actor_type_index.exs @@ -0,0 +1,7 @@ +defmodule Pleroma.Repo.Migrations.AddActivitiesActorTypeIndex do + use Ecto.Migration + + def change do + create(index(:activities, ["actor", "(data ->> 'type'::text)", "id DESC NULLS LAST"])) + end +end From ad79912a0723f4a3e428c125a9c2946831b2cfa8 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Fri, 14 Mar 2025 19:53:06 +0400 Subject: [PATCH 061/128] Create the index concurrently --- .../20250314153704_add_activities_actor_type_index.exs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/priv/repo/migrations/20250314153704_add_activities_actor_type_index.exs b/priv/repo/migrations/20250314153704_add_activities_actor_type_index.exs index 3713beea1..a0fac28a8 100644 --- a/priv/repo/migrations/20250314153704_add_activities_actor_type_index.exs +++ b/priv/repo/migrations/20250314153704_add_activities_actor_type_index.exs @@ -1,7 +1,14 @@ defmodule Pleroma.Repo.Migrations.AddActivitiesActorTypeIndex do use Ecto.Migration + @disable_ddl_transaction true def change do - create(index(:activities, ["actor", "(data ->> 'type'::text)", "id DESC NULLS LAST"])) + create( + index( + :activities, + ["actor", "(data ->> 'type'::text)", "id DESC NULLS LAST"], + concurrently: true + ) + ) end end From 016df5093dd3296b6bdf60cf1c25cd76f8190392 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sun, 16 Mar 2025 12:23:22 +0400 Subject: [PATCH 062/128] Config: Use advisory lock --- config/config.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/config.exs b/config/config.exs index 50672cfc8..a231c5ba0 100644 --- a/config/config.exs +++ b/config/config.exs @@ -48,7 +48,7 @@ config :pleroma, ecto_repos: [Pleroma.Repo] config :pleroma, Pleroma.Repo, telemetry_event: [Pleroma.Repo.Instrumenter], - migration_lock: nil + migration_lock: :pg_advisory_lock config :pleroma, Pleroma.Captcha, enabled: true, From fc7ca2ccf4fe593bf47c90f26484cff8b5d99269 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 18 Mar 2025 15:25:54 +0400 Subject: [PATCH 063/128] Federator: More specific logging for rejections --- lib/pleroma/web/federator.ex | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/pleroma/web/federator.ex b/lib/pleroma/web/federator.ex index 58260afa8..676fc5137 100644 --- a/lib/pleroma/web/federator.ex +++ b/lib/pleroma/web/federator.ex @@ -122,6 +122,10 @@ defmodule Pleroma.Web.Federator do Logger.debug("Unhandled actor #{actor}, #{inspect(e)}") {:error, e} + {:reject, reason} = e -> + Logger.debug("Rejected by MRF: #{inspect(reason)}") + {:error, e} + e -> # Just drop those for now Logger.debug(fn -> "Unhandled activity\n" <> Jason.encode!(params, pretty: true) end) From e19ca7606dc25ccb5a68c276dbe95eebe372a677 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 18 Mar 2025 15:53:27 +0400 Subject: [PATCH 064/128] Transmogrifier: Also accept mitra emoji likes. --- .../web/activity_pub/transmogrifier.ex | 13 ++++- test/fixtures/misskey-custom-emoji-like.json | 54 +++++++++++++++++++ test/fixtures/mitra-custom-emoji-like.json | 46 ++++++++++++++++ .../transmogrifier/like_handling_test.exs | 51 ++++++++++++++++++ 4 files changed, 163 insertions(+), 1 deletion(-) create mode 100644 test/fixtures/misskey-custom-emoji-like.json create mode 100644 test/fixtures/mitra-custom-emoji-like.json diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 1e6ee7dc8..19d036c0d 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -495,12 +495,23 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do defp handle_incoming_normalized( %{ "type" => "Like", - "_misskey_reaction" => reaction + "content" => _ } = data, options ) do data |> Map.put("type", "EmojiReact") + |> handle_incoming_normalized(options) + end + + defp handle_incoming_normalized( + %{ + "type" => "Like", + "_misskey_reaction" => reaction + } = data, + options + ) do + data |> Map.put("content", @misskey_reactions[reaction] || reaction) |> handle_incoming_normalized(options) end diff --git a/test/fixtures/misskey-custom-emoji-like.json b/test/fixtures/misskey-custom-emoji-like.json new file mode 100644 index 000000000..51a825d42 --- /dev/null +++ b/test/fixtures/misskey-custom-emoji-like.json @@ -0,0 +1,54 @@ + { + "@context": [ + "https://www.w3.org/ns/activitystreams", + "https://w3id.org/security/v1", + { + "Emoji": "toot:Emoji", + "Hashtag": "as:Hashtag", + "PropertyValue": "schema:PropertyValue", + "_misskey_content": "misskey:_misskey_content", + "_misskey_quote": "misskey:_misskey_quote", + "_misskey_reaction": "misskey:_misskey_reaction", + "_misskey_summary": "misskey:_misskey_summary", + "_misskey_votes": "misskey:_misskey_votes", + "backgroundUrl": "sharkey:backgroundUrl", + "discoverable": "toot:discoverable", + "featured": "toot:featured", + "fedibird": "http://fedibird.com/ns#", + "firefish": "https://joinfirefish.org/ns#", + "isCat": "misskey:isCat", + "listenbrainz": "sharkey:listenbrainz", + "manuallyApprovesFollowers": "as:manuallyApprovesFollowers", + "misskey": "https://misskey-hub.net/ns#", + "quoteUri": "fedibird:quoteUri", + "quoteUrl": "as:quoteUrl", + "schema": "http://schema.org#", + "sensitive": "as:sensitive", + "sharkey": "https://joinsharkey.org/ns#", + "speakAsCat": "firefish:speakAsCat", + "toot": "http://joinmastodon.org/ns#", + "value": "schema:value", + "vcard": "http://www.w3.org/2006/vcard/ns#" + } + ], + "_misskey_reaction": ":blobwtfnotlikethis:", + "actor": "https://mai.waifuism.life/users/9otxaeemjqy70001", + "content": ":blobwtfnotlikethis:", + "id": "https://mai.waifuism.life/likes/9q2xifhrdnb0001b", + "object": "https://bungle.online/notes/9q2xi2sy4k", + "tag": [ + { + "icon": { + "mediaType": "image/png", + "type": "Image", + "url": "https://mai.waifuism.life/files/1b0510f2-1fb4-43f5-a399-10053bbd8f0f" + }, + "id": "https://mai.waifuism.life/emojis/blobwtfnotlikethis", + "name": ":blobwtfnotlikethis:", + "type": "Emoji", + "updated": "2024-02-07T02:21:46.497Z" + } + ], + "type": "Like" +} + diff --git a/test/fixtures/mitra-custom-emoji-like.json b/test/fixtures/mitra-custom-emoji-like.json new file mode 100644 index 000000000..4d727febd --- /dev/null +++ b/test/fixtures/mitra-custom-emoji-like.json @@ -0,0 +1,46 @@ +{ + "@context": [ + "https://www.w3.org/ns/activitystreams", + "https://w3id.org/security/v1", + "https://w3id.org/security/data-integrity/v1", + { + "Emoji": "toot:Emoji", + "Hashtag": "as:Hashtag", + "sensitive": "as:sensitive", + "toot": "http://joinmastodon.org/ns#" + } + ], + "actor": "https://mitra.social/users/silverpill", + "cc": [], + "content": ":ablobcatheartsqueeze:", + "id": "https://mitra.social/activities/like/0195a89a-a3a0-ead4-3a1c-aa6311397cfd", + "object": "https://framapiaf.org/users/peertube/statuses/114182703352270287", + "proof": { + "created": "2025-03-18T09:34:21.610678375Z", + "cryptosuite": "eddsa-jcs-2022", + "proofPurpose": "assertionMethod", + "proofValue": "z5AvpwkXQGFpTneRVDNeF48Jo9qYG6PgrE5HaPPpQNdNyc31ULMN4Vxd4aFXELo4Rk5Y9hd9nDy254xP8v5uGGWp1", + "type": "DataIntegrityProof", + "verificationMethod": "https://mitra.social/users/silverpill#ed25519-key" + }, + "tag": [ + { + "attributedTo": "https://mitra.social/actor", + "icon": { + "mediaType": "image/png", + "type": "Image", + "url": "https://mitra.social/media/a08e153441b25e512ab1b2e8922f5d8cd928322c8b79958cd48297ac722a4117.png" + }, + "id": "https://mitra.social/objects/emojis/ablobcatheartsqueeze", + "name": ":ablobcatheartsqueeze:", + "type": "Emoji", + "updated": "1970-01-01T00:00:00Z" + } + ], + "to": [ + "https://framapiaf.org/users/peertube", + "https://www.w3.org/ns/activitystreams#Public" + ], + "type": "Like" +} + diff --git a/test/pleroma/web/activity_pub/transmogrifier/like_handling_test.exs b/test/pleroma/web/activity_pub/transmogrifier/like_handling_test.exs index c02f66d77..560f31dac 100644 --- a/test/pleroma/web/activity_pub/transmogrifier/like_handling_test.exs +++ b/test/pleroma/web/activity_pub/transmogrifier/like_handling_test.exs @@ -6,6 +6,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.LikeHandlingTest do use Pleroma.DataCase, async: true alias Pleroma.Activity + alias Pleroma.Object alias Pleroma.Web.ActivityPub.Transmogrifier alias Pleroma.Web.CommonAPI @@ -75,4 +76,54 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.LikeHandlingTest do assert activity_data["object"] == activity.data["object"] assert activity_data["content"] == "⭐" end + + test "it works for misskey likes with custom emoji" do + user = insert(:user) + + {:ok, activity} = CommonAPI.post(user, %{status: "hello"}) + + data = + File.read!("test/fixtures/misskey-custom-emoji-like.json") + |> Jason.decode!() + |> Map.put("object", activity.data["object"]) + + _actor = insert(:user, ap_id: data["actor"], local: false) + + {:ok, %Activity{data: activity_data, local: false}} = Transmogrifier.handle_incoming(data) + + assert activity_data["actor"] == data["actor"] + assert activity_data["type"] == "EmojiReact" + assert activity_data["id"] == data["id"] + assert activity_data["object"] == activity.data["object"] + assert activity_data["content"] == ":blobwtfnotlikethis:" + + assert [["blobwtfnotlikethis", _, _]] = + Object.get_by_ap_id(activity.data["object"]) + |> Object.get_emoji_reactions() + end + + test "it works for mitra likes with custom emoji" do + user = insert(:user) + + {:ok, activity} = CommonAPI.post(user, %{status: "hello"}) + + data = + File.read!("test/fixtures/mitra-custom-emoji-like.json") + |> Jason.decode!() + |> Map.put("object", activity.data["object"]) + + _actor = insert(:user, ap_id: data["actor"], local: false) + + {:ok, %Activity{data: activity_data, local: false}} = Transmogrifier.handle_incoming(data) + + assert activity_data["actor"] == data["actor"] + assert activity_data["type"] == "EmojiReact" + assert activity_data["id"] == data["id"] + assert activity_data["object"] == activity.data["object"] + assert activity_data["content"] == ":ablobcatheartsqueeze:" + + assert [["ablobcatheartsqueeze", _, _]] = + Object.get_by_ap_id(activity.data["object"]) + |> Object.get_emoji_reactions() + end end From ef216c922fbd2b96de2f1e99bae8d4ddb3700fdc Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 18 Mar 2025 15:54:33 +0400 Subject: [PATCH 065/128] Add changelog --- changelog.d/emoji_likes.add | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/emoji_likes.add diff --git a/changelog.d/emoji_likes.add b/changelog.d/emoji_likes.add new file mode 100644 index 000000000..13c91a950 --- /dev/null +++ b/changelog.d/emoji_likes.add @@ -0,0 +1 @@ +Support Mitra-style emoji likes. From 950bf60765cd4eff8f29717dd7a487b8cdf395f8 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Wed, 19 Mar 2025 15:57:08 +0400 Subject: [PATCH 066/128] LikeHandlingTest: Add test for invalid content --- .../transmogrifier/like_handling_test.exs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/test/pleroma/web/activity_pub/transmogrifier/like_handling_test.exs b/test/pleroma/web/activity_pub/transmogrifier/like_handling_test.exs index 560f31dac..023c2530f 100644 --- a/test/pleroma/web/activity_pub/transmogrifier/like_handling_test.exs +++ b/test/pleroma/web/activity_pub/transmogrifier/like_handling_test.exs @@ -126,4 +126,20 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.LikeHandlingTest do Object.get_by_ap_id(activity.data["object"]) |> Object.get_emoji_reactions() end + + test "it works for likes with wrong content" do + user = insert(:user) + + {:ok, activity} = CommonAPI.post(user, %{status: "hello"}) + + data = + File.read!("test/fixtures/mitra-custom-emoji-like.json") + |> Jason.decode!() + |> Map.put("object", activity.data["object"]) + |> Map.put("content", 1) + + _actor = insert(:user, ap_id: data["actor"], local: false) + + assert {:error, _} = Transmogrifier.handle_incoming(data) + end end From f9bff8f5e5408e0b8eee9a7b0019e4c92c54e1f9 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Wed, 19 Mar 2025 16:00:27 +0400 Subject: [PATCH 067/128] Transmogrifier: Keep likes as likes if the content is obviously wrong --- lib/pleroma/web/activity_pub/transmogrifier.ex | 5 +++-- .../web/activity_pub/transmogrifier/like_handling_test.exs | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 19d036c0d..6517f5eff 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -495,10 +495,11 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do defp handle_incoming_normalized( %{ "type" => "Like", - "content" => _ + "content" => content } = data, options - ) do + ) + when is_binary(content) do data |> Map.put("type", "EmojiReact") |> handle_incoming_normalized(options) diff --git a/test/pleroma/web/activity_pub/transmogrifier/like_handling_test.exs b/test/pleroma/web/activity_pub/transmogrifier/like_handling_test.exs index 023c2530f..fc04c1391 100644 --- a/test/pleroma/web/activity_pub/transmogrifier/like_handling_test.exs +++ b/test/pleroma/web/activity_pub/transmogrifier/like_handling_test.exs @@ -140,6 +140,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.LikeHandlingTest do _actor = insert(:user, ap_id: data["actor"], local: false) - assert {:error, _} = Transmogrifier.handle_incoming(data) + assert {:ok, activity} = Transmogrifier.handle_incoming(data) + assert activity.data["type"] == "Like" end end From 25a3ee2256c8cf24575c8ed31eaa851d4c8dbea1 Mon Sep 17 00:00:00 2001 From: mkljczk Date: Wed, 19 Mar 2025 17:59:42 +0100 Subject: [PATCH 068/128] InstanceView: do not repeat information Signed-off-by: mkljczk --- lib/pleroma/web/mastodon_api/views/instance_view.ex | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index bd1ecc2f7..4b0480f66 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -157,9 +157,6 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do "pleroma:bookmark_folders", if Pleroma.Language.LanguageDetector.configured?() do "pleroma:language_detection" - end, - if Pleroma.Language.Translation.configured?() do - "translation" end ] |> Enum.filter(& &1) From 7763b9a87fe534bd85892884fdbb4bbb6b31c982 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 19 Mar 2025 10:29:45 -0700 Subject: [PATCH 069/128] Truncate the length of Rich Media title and description fields Some sites like Instagram are serving obnoxiously long metadata fields --- changelog.d/truncate-rich-media.change | 1 + lib/pleroma/web/rich_media/parser.ex | 13 +++ .../rich_media/instagram_longtext.html | 90 +++++++++++++++++++ test/pleroma/web/rich_media/parser_test.exs | 7 ++ test/support/http_request_mock.ex | 8 +- 5 files changed, 118 insertions(+), 1 deletion(-) create mode 100644 changelog.d/truncate-rich-media.change create mode 100644 test/fixtures/rich_media/instagram_longtext.html diff --git a/changelog.d/truncate-rich-media.change b/changelog.d/truncate-rich-media.change new file mode 100644 index 000000000..1df064be1 --- /dev/null +++ b/changelog.d/truncate-rich-media.change @@ -0,0 +1 @@ +Truncate the length of Rich Media title and description fields diff --git a/lib/pleroma/web/rich_media/parser.ex b/lib/pleroma/web/rich_media/parser.ex index a3a522d7a..9c8ec7a9f 100644 --- a/lib/pleroma/web/rich_media/parser.ex +++ b/lib/pleroma/web/rich_media/parser.ex @@ -4,6 +4,7 @@ defmodule Pleroma.Web.RichMedia.Parser do alias Pleroma.Web.RichMedia.Helpers + import Pleroma.Web.Metadata.Utils, only: [scrub_html_and_truncate: 2] require Logger @config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config) @@ -63,8 +64,20 @@ defmodule Pleroma.Web.RichMedia.Parser do not match?({:ok, _}, Jason.encode(%{key => val})) end) |> Map.new() + |> truncate_title() + |> truncate_desc() end + defp truncate_title(%{"title" => title} = data) when is_binary(title), + do: %{data | "title" => scrub_html_and_truncate(title, 120)} + + defp truncate_title(data), do: data + + defp truncate_desc(%{"description" => desc} = data) when is_binary(desc), + do: %{data | "description" => scrub_html_and_truncate(desc, 200)} + + defp truncate_desc(data), do: data + @spec validate_page_url(URI.t() | binary()) :: :ok | :error defp validate_page_url(page_url) when is_binary(page_url) do validate_tld = @config_impl.get([Pleroma.Formatter, :validate_tld]) diff --git a/test/fixtures/rich_media/instagram_longtext.html b/test/fixtures/rich_media/instagram_longtext.html new file mode 100644 index 000000000..e833f408c --- /dev/null +++ b/test/fixtures/rich_media/instagram_longtext.html @@ -0,0 +1,90 @@ + + + + + + + + + + + + + + + + +CAPTURE THE ATLAS | ✨ A Once-in-a-Lifetime Shot: Total Lunar Eclipse + Aurora Substorm! 🔴💚 + +Last Thursday night, under the freezing skies of Northern Alaska, I... | Instagram + +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/pleroma/web/rich_media/parser_test.exs b/test/pleroma/web/rich_media/parser_test.exs index 20f61badc..1f01d657a 100644 --- a/test/pleroma/web/rich_media/parser_test.exs +++ b/test/pleroma/web/rich_media/parser_test.exs @@ -61,6 +61,13 @@ defmodule Pleroma.Web.RichMedia.ParserTest do }} end + test "truncates title and description fields" do + {:ok, parsed} = Parser.parse("https://instagram.com/longtext") + + assert String.length(parsed["title"]) == 120 + assert String.length(parsed["description"]) == 200 + end + test "parses OEmbed and filters HTML tags" do assert Parser.parse("https://example.com/oembed") == {:ok, diff --git a/test/support/http_request_mock.ex b/test/support/http_request_mock.ex index 1c472fca9..a8f954af9 100644 --- a/test/support/http_request_mock.ex +++ b/test/support/http_request_mock.ex @@ -1494,6 +1494,11 @@ defmodule HttpRequestMock do {:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/twitter_card.html")}} end + def get("https://instagram.com/longtext", _, _, _) do + {:ok, + %Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/instagram_longtext.html")}} + end + def get("https://example.com/non-ogp", _, _, _) do {:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/non_ogp_embed.html")}} @@ -1720,7 +1725,8 @@ defmodule HttpRequestMock do "https://example.com/twitter-card", "https://google.com/", "https://pleroma.local/notice/9kCP7V", - "https://yahoo.com/" + "https://yahoo.com/", + "https://instagram.com/longtext" ] def head(url, _query, _body, _headers) when url in @rich_media_mocks do From 638d047a5c2f50b5c41ad35ba223092a1acd2872 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 19 Mar 2025 10:47:32 -0700 Subject: [PATCH 070/128] Fix releases by not relying on Mix --- changelog.d/releases.fix | 1 + lib/pleroma/application.ex | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/releases.fix diff --git a/changelog.d/releases.fix b/changelog.d/releases.fix new file mode 100644 index 000000000..5436accc7 --- /dev/null +++ b/changelog.d/releases.fix @@ -0,0 +1 @@ +Fix release builds diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index d7975d2d1..78ac0443f 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -56,7 +56,7 @@ defmodule Pleroma.Application do Pleroma.Web.Plugs.HTTPSecurityPlug.warn_if_disabled() end - if Mix.env() != :test do + if Config.get(:env) != :test do Pleroma.ApplicationRequirements.verify!() end From 3af9692352a54f6f85d5c9b7eeba00bca605db69 Mon Sep 17 00:00:00 2001 From: Moon Man Date: Thu, 20 Mar 2025 15:25:00 +0000 Subject: [PATCH 071/128] return json if no accept is specified --- lib/pleroma/web/web_finger/web_finger_controller.ex | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/lib/pleroma/web/web_finger/web_finger_controller.ex b/lib/pleroma/web/web_finger/web_finger_controller.ex index 021df9bc5..0a9ee2d3b 100644 --- a/lib/pleroma/web/web_finger/web_finger_controller.ex +++ b/lib/pleroma/web/web_finger/web_finger_controller.ex @@ -41,5 +41,17 @@ defmodule Pleroma.Web.WebFinger.WebFingerController do end end + # Default to JSON when no format is specified or format is not recognized + def webfinger(%{assigns: %{format: _format}} = conn, %{"resource" => resource}) do + with {:ok, response} <- WebFinger.webfinger(resource, "JSON") do + json(conn, response) + else + _e -> + conn + |> put_status(404) + |> json("Couldn't find user") + end + end + def webfinger(conn, _params), do: send_resp(conn, 400, "Bad Request") end From edfa372fdb572e429c28c4346dc7c8ccb1d342c7 Mon Sep 17 00:00:00 2001 From: Moon Man Date: Thu, 20 Mar 2025 15:30:41 +0000 Subject: [PATCH 072/128] changelog update --- changelog.d/webfinger.change | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/webfinger.change diff --git a/changelog.d/webfinger.change b/changelog.d/webfinger.change new file mode 100644 index 000000000..353e65a89 --- /dev/null +++ b/changelog.d/webfinger.change @@ -0,0 +1 @@ +Don't require an Accept header for WebFinger queries and default to JSON. \ No newline at end of file From 7624af92cf95b8ae17bff59c2327853eb606b26d Mon Sep 17 00:00:00 2001 From: Moon Man Date: Thu, 20 Mar 2025 16:42:46 +0000 Subject: [PATCH 073/128] tests for webfinger --- .../web_finger/web_finger_controller_test.exs | 46 +++++++++++++++---- 1 file changed, 37 insertions(+), 9 deletions(-) diff --git a/test/pleroma/web/web_finger/web_finger_controller_test.exs b/test/pleroma/web/web_finger/web_finger_controller_test.exs index 80e072163..b89849e68 100644 --- a/test/pleroma/web/web_finger/web_finger_controller_test.exs +++ b/test/pleroma/web/web_finger/web_finger_controller_test.exs @@ -55,6 +55,26 @@ defmodule Pleroma.Web.WebFinger.WebFingerControllerTest do ] end + test "Webfinger defaults to JSON when no Accept header is provided" do + user = + insert(:user, + ap_id: "https://hyrule.world/users/zelda", + also_known_as: ["https://mushroom.kingdom/users/toad"] + ) + + response = + build_conn() + |> get("/.well-known/webfinger?resource=acct:#{user.nickname}@localhost") + |> json_response(200) + + assert response["subject"] == "acct:#{user.nickname}@localhost" + + assert response["aliases"] == [ + "https://hyrule.world/users/zelda", + "https://mushroom.kingdom/users/toad" + ] + end + test "reach user on tld, while pleroma is running on subdomain" do clear_config([Pleroma.Web.Endpoint, :url, :host], "sub.example.com") @@ -109,16 +129,24 @@ defmodule Pleroma.Web.WebFinger.WebFingerControllerTest do assert result == "Couldn't find user" end - test "Sends a 404 when invalid format" do - user = insert(:user) + test "Returns JSON when format is not supported" do + user = + insert(:user, + ap_id: "https://hyrule.world/users/zelda", + also_known_as: ["https://mushroom.kingdom/users/toad"] + ) - assert capture_log(fn -> - assert_raise Phoenix.NotAcceptableError, fn -> - build_conn() - |> put_req_header("accept", "text/html") - |> get("/.well-known/webfinger?resource=acct:#{user.nickname}@localhost") - end - end) =~ "no supported media type in accept header" + response = + build_conn() + |> put_req_header("accept", "text/html") + |> get("/.well-known/webfinger?resource=acct:#{user.nickname}@localhost") + |> json_response(200) + + assert response["subject"] == "acct:#{user.nickname}@localhost" + assert response["aliases"] == [ + "https://hyrule.world/users/zelda", + "https://mushroom.kingdom/users/toad" + ] end test "Sends a 400 when resource param is missing" do From 43a124bb14d385382c8b16da7d229d9ec7cd1205 Mon Sep 17 00:00:00 2001 From: Moon Man Date: Thu, 20 Mar 2025 12:51:43 -0400 Subject: [PATCH 074/128] formatting --- test/pleroma/web/web_finger/web_finger_controller_test.exs | 1 + 1 file changed, 1 insertion(+) diff --git a/test/pleroma/web/web_finger/web_finger_controller_test.exs b/test/pleroma/web/web_finger/web_finger_controller_test.exs index b89849e68..d60e8a585 100644 --- a/test/pleroma/web/web_finger/web_finger_controller_test.exs +++ b/test/pleroma/web/web_finger/web_finger_controller_test.exs @@ -143,6 +143,7 @@ defmodule Pleroma.Web.WebFinger.WebFingerControllerTest do |> json_response(200) assert response["subject"] == "acct:#{user.nickname}@localhost" + assert response["aliases"] == [ "https://hyrule.world/users/zelda", "https://mushroom.kingdom/users/toad" From 890ac8ff86e28af464f56fc023d9d7e2f4bc2f1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Sun, 30 Apr 2023 17:33:11 +0200 Subject: [PATCH 075/128] Expose markup configuration in InstanceView MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicole Mikołajczyk --- lib/pleroma/web/mastodon_api/views/instance_view.ex | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index 4b0480f66..af6a63e92 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -270,7 +270,8 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do post_formats: Config.get([:instance, :allowed_post_formats]), birthday_required: Config.get([:instance, :birthday_required]), birthday_min_age: Config.get([:instance, :birthday_min_age]), - translation: supported_languages() + translation: supported_languages(), + markup: markup() }, stats: %{mau: Pleroma.User.active_user_count()}, vapid_public_key: Keyword.get(Pleroma.Web.Push.vapid_config(), :public_key) @@ -321,4 +322,12 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do target_languages: target_languages } end + + defp markup() do + %{ + allow_inline_images: Config.get([:markup, :allow_inline_images]), + allow_headings: Config.get([:markup, :allow_headings]), + allow_tables: Config.get([:markup, :allow_tables]) + } + end end From 4d4174c339b0450aab4bb90473ee8285f87936f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicole=20Miko=C5=82ajczyk?= Date: Fri, 28 Mar 2025 18:47:00 +0100 Subject: [PATCH 076/128] fix a few typos MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicole Mikołajczyk --- changelog.d/typos.skip | 0 lib/pleroma/web/api_spec.ex | 2 +- .../web/api_spec/operations/admin/rule_operation.ex | 8 ++++---- lib/pleroma/web/api_spec/operations/instance_operation.ex | 2 +- test/pleroma/web/activity_pub/activity_pub_test.exs | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) create mode 100644 changelog.d/typos.skip diff --git a/changelog.d/typos.skip b/changelog.d/typos.skip new file mode 100644 index 000000000..e69de29bb diff --git a/lib/pleroma/web/api_spec.ex b/lib/pleroma/web/api_spec.ex index 63409870e..e5339097f 100644 --- a/lib/pleroma/web/api_spec.ex +++ b/lib/pleroma/web/api_spec.ex @@ -97,7 +97,7 @@ defmodule Pleroma.Web.ApiSpec do "Frontend management", "Instance configuration", "Instance documents", - "Instance rule managment", + "Instance rule management", "Invites", "MediaProxy cache", "OAuth application management", diff --git a/lib/pleroma/web/api_spec/operations/admin/rule_operation.ex b/lib/pleroma/web/api_spec/operations/admin/rule_operation.ex index c3a3ecc7c..6d06728f4 100644 --- a/lib/pleroma/web/api_spec/operations/admin/rule_operation.ex +++ b/lib/pleroma/web/api_spec/operations/admin/rule_operation.ex @@ -16,7 +16,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.RuleOperation do def index_operation do %Operation{ - tags: ["Instance rule managment"], + tags: ["Instance rule management"], summary: "Retrieve list of instance rules", operationId: "AdminAPI.RuleController.index", security: [%{"oAuth" => ["admin:read"]}], @@ -33,7 +33,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.RuleOperation do def create_operation do %Operation{ - tags: ["Instance rule managment"], + tags: ["Instance rule management"], summary: "Create new rule", operationId: "AdminAPI.RuleController.create", security: [%{"oAuth" => ["admin:write"]}], @@ -49,7 +49,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.RuleOperation do def update_operation do %Operation{ - tags: ["Instance rule managment"], + tags: ["Instance rule management"], summary: "Modify existing rule", operationId: "AdminAPI.RuleController.update", security: [%{"oAuth" => ["admin:write"]}], @@ -65,7 +65,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.RuleOperation do def delete_operation do %Operation{ - tags: ["Instance rule managment"], + tags: ["Instance rule management"], summary: "Delete rule", operationId: "AdminAPI.RuleController.delete", parameters: [Operation.parameter(:id, :path, :string, "Rule ID")], diff --git a/lib/pleroma/web/api_spec/operations/instance_operation.ex b/lib/pleroma/web/api_spec/operations/instance_operation.ex index 84e5b314d..911ffb994 100644 --- a/lib/pleroma/web/api_spec/operations/instance_operation.ex +++ b/lib/pleroma/web/api_spec/operations/instance_operation.ex @@ -52,7 +52,7 @@ defmodule Pleroma.Web.ApiSpec.InstanceOperation do summary: "Retrieve list of instance rules", operationId: "InstanceController.rules", responses: %{ - 200 => Operation.response("Array of domains", "application/json", array_of_rules()) + 200 => Operation.response("Array of rules", "application/json", array_of_rules()) } } end diff --git a/test/pleroma/web/activity_pub/activity_pub_test.exs b/test/pleroma/web/activity_pub/activity_pub_test.exs index dbc3aa532..c16f081f6 100644 --- a/test/pleroma/web/activity_pub/activity_pub_test.exs +++ b/test/pleroma/web/activity_pub/activity_pub_test.exs @@ -826,7 +826,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do assert object.data["repliesCount"] == 2 end - test "increates quotes count", %{user: user} do + test "increases quotes count", %{user: user} do user2 = insert(:user) {:ok, activity} = CommonAPI.post(user, %{status: "1", visibility: "public"}) From d1b9d03302b4f8ea83174d0934f71360709d7585 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicole=20Miko=C5=82ajczyk?= Date: Fri, 28 Mar 2025 17:00:36 +0100 Subject: [PATCH 077/128] update changelog MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicole Mikołajczyk --- changelog.d/expose-markup-configuration.add | 1 + lib/pleroma/web/mastodon_api/views/instance_view.ex | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/expose-markup-configuration.add diff --git a/changelog.d/expose-markup-configuration.add b/changelog.d/expose-markup-configuration.add new file mode 100644 index 000000000..8c7f35697 --- /dev/null +++ b/changelog.d/expose-markup-configuration.add @@ -0,0 +1 @@ +Expose markup configuration in InstanceView diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index af6a63e92..848bf1a22 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -323,7 +323,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do } end - defp markup() do + defp markup do %{ allow_inline_images: Config.get([:markup, :allow_inline_images]), allow_headings: Config.get([:markup, :allow_headings]), From f60a1e7d44e94824ef0b2c38f69540b14cb3693e Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 31 Mar 2025 20:17:18 -0700 Subject: [PATCH 078/128] Set PATH in the FreeBSD rc script to avoid failures starting the service --- changelog.d/freebsd-rc.fix | 1 + installation/freebsd/rc.d/pleroma | 2 ++ 2 files changed, 3 insertions(+) create mode 100644 changelog.d/freebsd-rc.fix diff --git a/changelog.d/freebsd-rc.fix b/changelog.d/freebsd-rc.fix new file mode 100644 index 000000000..1f59d4596 --- /dev/null +++ b/changelog.d/freebsd-rc.fix @@ -0,0 +1 @@ +Set PATH in the FreeBSD rc script to avoid failures starting the service diff --git a/installation/freebsd/rc.d/pleroma b/installation/freebsd/rc.d/pleroma index f62aef18d..149b40838 100755 --- a/installation/freebsd/rc.d/pleroma +++ b/installation/freebsd/rc.d/pleroma @@ -24,4 +24,6 @@ command=/usr/local/bin/elixir command_args="--erl \"-detached\" -S /usr/local/bin/mix phx.server" procname="*beam.smp" +PATH="${PATH}:/usr/local/sbin:/usr/local/bin" + run_rc_command "$1" From 93aa563cfe0bca64be3fa5d4bc74843d87f03937 Mon Sep 17 00:00:00 2001 From: Moon Man Date: Wed, 2 Apr 2025 07:00:45 -0400 Subject: [PATCH 079/128] implemented --- changelog.d/siteinfo-baseurls.add | 3 ++ .../web/mastodon_api/views/instance_view.ex | 18 +++++++++- .../controllers/instance_controller_test.exs | 33 +++++++++++++++++++ 3 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 changelog.d/siteinfo-baseurls.add diff --git a/changelog.d/siteinfo-baseurls.add b/changelog.d/siteinfo-baseurls.add new file mode 100644 index 000000000..d0ff986d7 --- /dev/null +++ b/changelog.d/siteinfo-baseurls.add @@ -0,0 +1,3 @@ +### Added + +- Add `base_urls` to the /api/v1/instance pleroma metadata which provides information about the base URLs for media_proxy and uploads when configured \ No newline at end of file diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index 4b0480f66..fd72e2f91 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -261,6 +261,21 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do end defp pleroma_configuration(instance) do + base_urls = %{} + + base_urls = + if Config.get([:media_proxy, :enabled]) do + Map.put(base_urls, :media_proxy, Config.get([:media_proxy, :base_url])) + else + base_urls + end + + base_urls = + case Config.get([Pleroma.Upload, :base_url]) do + nil -> base_urls + url -> Map.put(base_urls, :upload, url) + end + %{ metadata: %{ account_activation_required: Keyword.get(instance, :account_activation_required), @@ -270,7 +285,8 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do post_formats: Config.get([:instance, :allowed_post_formats]), birthday_required: Config.get([:instance, :birthday_required]), birthday_min_age: Config.get([:instance, :birthday_min_age]), - translation: supported_languages() + translation: supported_languages(), + base_urls: base_urls }, stats: %{mau: Pleroma.User.active_user_count()}, vapid_public_key: Keyword.get(Pleroma.Web.Push.vapid_config(), :public_key) diff --git a/test/pleroma/web/mastodon_api/controllers/instance_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/instance_controller_test.exs index 38b547770..8a0fe5259 100644 --- a/test/pleroma/web/mastodon_api/controllers/instance_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/instance_controller_test.exs @@ -161,4 +161,37 @@ defmodule Pleroma.Web.MastodonAPI.InstanceControllerTest do |> get("/api/v1/instance/translation_languages") |> json_response_and_validate_schema(200) end + + test "base_urls in pleroma metadata", %{conn: conn} do + media_proxy_base_url = "https://media.example.org" + upload_base_url = "https://uploads.example.org" + + clear_config([:media_proxy, :enabled], true) + clear_config([:media_proxy, :base_url], media_proxy_base_url) + clear_config([Pleroma.Upload, :base_url], upload_base_url) + + conn = get(conn, "/api/v1/instance") + + assert result = json_response_and_validate_schema(conn, 200) + assert result["pleroma"]["metadata"]["base_urls"]["media_proxy"] == media_proxy_base_url + assert result["pleroma"]["metadata"]["base_urls"]["upload"] == upload_base_url + + # Test when media_proxy is disabled + clear_config([:media_proxy, :enabled], false) + + conn = get(conn, "/api/v1/instance") + + assert result = json_response_and_validate_schema(conn, 200) + refute Map.has_key?(result["pleroma"]["metadata"]["base_urls"], "media_proxy") + assert result["pleroma"]["metadata"]["base_urls"]["upload"] == upload_base_url + + # Test when upload base_url is not set + clear_config([Pleroma.Upload, :base_url], nil) + + conn = get(conn, "/api/v1/instance") + + assert result = json_response_and_validate_schema(conn, 200) + refute Map.has_key?(result["pleroma"]["metadata"]["base_urls"], "media_proxy") + refute Map.has_key?(result["pleroma"]["metadata"]["base_urls"], "upload") + end end From 8322134a2112cca2deb98c5ad324ed7c7e76f704 Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 2 Apr 2025 12:30:32 +0000 Subject: [PATCH 080/128] Edit siteinfo-baseurls.add --- changelog.d/siteinfo-baseurls.add | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/changelog.d/siteinfo-baseurls.add b/changelog.d/siteinfo-baseurls.add index d0ff986d7..6f0f19847 100644 --- a/changelog.d/siteinfo-baseurls.add +++ b/changelog.d/siteinfo-baseurls.add @@ -1,3 +1 @@ -### Added - -- Add `base_urls` to the /api/v1/instance pleroma metadata which provides information about the base URLs for media_proxy and uploads when configured \ No newline at end of file +Add `base_urls` to the /api/v1/instance pleroma metadata which provides information about the base URLs for media_proxy and uploads when configured \ No newline at end of file From 1266b180b912036fde72fa688eb7de99686ce47e Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 10 Apr 2025 14:32:31 -0700 Subject: [PATCH 081/128] Improved performance of status search queries using the default GIN index --- changelog.d/gin-search.fix | 1 + lib/pleroma/search/database_search.ex | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 changelog.d/gin-search.fix diff --git a/changelog.d/gin-search.fix b/changelog.d/gin-search.fix new file mode 100644 index 000000000..ba9977b6e --- /dev/null +++ b/changelog.d/gin-search.fix @@ -0,0 +1 @@ +Improved performance of status search queries using the default GIN index diff --git a/lib/pleroma/search/database_search.ex b/lib/pleroma/search/database_search.ex index aef5d1e74..e88d632cb 100644 --- a/lib/pleroma/search/database_search.ex +++ b/lib/pleroma/search/database_search.ex @@ -102,7 +102,8 @@ defmodule Pleroma.Search.DatabaseSearch do ^tsc, o.data, ^search_query - ) + ), + order_by: [desc: :inserted_at] ) end From 51a0cee405e0244585fcc85e6d59a8813dbea5d3 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Wed, 9 Apr 2025 22:50:28 +0300 Subject: [PATCH 082/128] Add expiring blocks - `/api/v1/accounts/:id/block` now has a "duration" parameter - `/api/v1/blocks` returns "block_expires_at" to indicate when the block will expire - MuteExpireWorker also processes block expiration - Remove unused OpenAPI parameters from mute endpoint - Add pleroma:block_expiration to nodeinfo features --- changelog.d/expiring-blocks.add | 1 + lib/pleroma/user.ex | 37 +++++++++++++++---- lib/pleroma/web/activity_pub/builder.ex | 6 +-- lib/pleroma/web/activity_pub/side_effects.ex | 2 +- .../api_spec/operations/account_operation.ex | 35 +++++++++++------- lib/pleroma/web/api_spec/schemas/account.ex | 1 + lib/pleroma/web/common_api.ex | 6 +-- .../controllers/account_controller.ex | 13 +++++-- .../web/mastodon_api/views/account_view.ex | 11 ++++++ .../web/mastodon_api/views/instance_view.ex | 3 +- lib/pleroma/workers/mute_expire_worker.ex | 19 +++++++++- test/pleroma/web/common_api_test.exs | 11 ++++++ 12 files changed, 112 insertions(+), 33 deletions(-) create mode 100644 changelog.d/expiring-blocks.add diff --git a/changelog.d/expiring-blocks.add b/changelog.d/expiring-blocks.add new file mode 100644 index 000000000..29989af15 --- /dev/null +++ b/changelog.d/expiring-blocks.add @@ -0,0 +1 @@ +Add `duration` to the block endpoint, which makes block expire \ No newline at end of file diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index d9da9ede1..316541343 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -1708,7 +1708,9 @@ defmodule Pleroma.User do end end - def block(%User{} = blocker, %User{} = blocked) do + def block(blocker, blocked, params \\ %{}) + + def block(%User{} = blocker, %User{} = blocked, params) do # sever any follow relationships to prevent leaks per activitypub (Pleroma issue #213) blocker = if following?(blocker, blocked) do @@ -1738,12 +1740,33 @@ defmodule Pleroma.User do {:ok, blocker} = update_follower_count(blocker) {:ok, blocker, _} = Participation.mark_all_as_read(blocker, blocked) - add_to_block(blocker, blocked) + + duration = Map.get(params, :duration, 0) + + expires_at = + if duration > 0 do + DateTime.utc_now() + |> DateTime.add(duration) + else + nil + end + + user_block = add_to_block(blocker, blocked, expires_at) + + if duration > 0 do + Pleroma.Workers.MuteExpireWorker.new( + %{"op" => "unblock_user", "blocker_id" => blocker.id, "blocked_id" => blocked.id}, + scheduled_at: expires_at + ) + |> Oban.insert() + end + + user_block end # helper to handle the block given only an actor's AP id - def block(%User{} = blocker, %{ap_id: ap_id}) do - block(blocker, get_cached_by_ap_id(ap_id)) + def block(%User{} = blocker, %{ap_id: ap_id}, params) do + block(blocker, get_cached_by_ap_id(ap_id), params) end def unblock(%User{} = blocker, %User{} = blocked) do @@ -2779,10 +2802,10 @@ defmodule Pleroma.User do set_domain_blocks(user, List.delete(user.domain_blocks, domain_blocked)) end - @spec add_to_block(User.t(), User.t()) :: + @spec add_to_block(User.t(), User.t(), integer() | nil) :: {:ok, UserRelationship.t()} | {:error, Ecto.Changeset.t()} - defp add_to_block(%User{} = user, %User{} = blocked) do - with {:ok, relationship} <- UserRelationship.create_block(user, blocked) do + defp add_to_block(%User{} = user, %User{} = blocked, expires_at) do + with {:ok, relationship} <- UserRelationship.create_block(user, blocked, expires_at) do @cachex.del(:user_cache, "blocked_users_ap_ids:#{user.ap_id}") {:ok, relationship} end diff --git a/lib/pleroma/web/activity_pub/builder.ex b/lib/pleroma/web/activity_pub/builder.ex index 2a1e56278..ecb6df1f0 100644 --- a/lib/pleroma/web/activity_pub/builder.ex +++ b/lib/pleroma/web/activity_pub/builder.ex @@ -327,8 +327,8 @@ defmodule Pleroma.Web.ActivityPub.Builder do }, []} end - @spec block(User.t(), User.t()) :: {:ok, map(), keyword()} - def block(blocker, blocked) do + @spec block(User.t(), User.t(), map()) :: {:ok, map(), keyword()} + def block(blocker, blocked, params) do {:ok, %{ "id" => Utils.generate_activity_id(), @@ -336,7 +336,7 @@ defmodule Pleroma.Web.ActivityPub.Builder do "actor" => blocker.ap_id, "object" => blocked.ap_id, "to" => [blocked.ap_id] - }, []} + }, Keyword.new(params)} end @spec announce(User.t(), Object.t(), keyword()) :: {:ok, map(), keyword()} diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index d6d403671..52cdc3c3f 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -145,7 +145,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do ) do with %User{} = blocker <- User.get_cached_by_ap_id(blocking_user), %User{} = blocked <- User.get_cached_by_ap_id(blocked_user) do - User.block(blocker, blocked) + User.block(blocker, blocked, Enum.into(meta, %{})) end {:ok, object, meta} diff --git a/lib/pleroma/web/api_spec/operations/account_operation.ex b/lib/pleroma/web/api_spec/operations/account_operation.ex index 21a779dcb..d63e92d16 100644 --- a/lib/pleroma/web/api_spec/operations/account_operation.ex +++ b/lib/pleroma/web/api_spec/operations/account_operation.ex @@ -284,18 +284,6 @@ defmodule Pleroma.Web.ApiSpec.AccountOperation do :query, %Schema{allOf: [BooleanLike], default: true}, "Mute notifications in addition to statuses? Defaults to `true`." - ), - Operation.parameter( - :duration, - :query, - %Schema{type: :integer}, - "Expire the mute in `duration` seconds. Default 0 for infinity" - ), - Operation.parameter( - :expires_in, - :query, - %Schema{type: :integer, default: 0}, - "Deprecated, use `duration` instead" ) ], responses: %{ @@ -323,16 +311,37 @@ defmodule Pleroma.Web.ApiSpec.AccountOperation do tags: ["Account actions"], summary: "Block", operationId: "AccountController.block", + requestBody: request_body("Parameters", block_request()), security: [%{"oAuth" => ["follow", "write:blocks"]}], description: "Block the given account. Clients should filter statuses from this account if received (e.g. due to a boost in the Home timeline)", - parameters: [%Reference{"$ref": "#/components/parameters/accountIdOrNickname"}], + parameters: [ + %Reference{"$ref": "#/components/parameters/accountIdOrNickname"} + ], responses: %{ 200 => Operation.response("Relationship", "application/json", AccountRelationship) } } end + defp block_request do + %Schema{ + title: "AccountBlockRequest", + description: "POST body for blocking an account", + type: :object, + properties: %{ + duration: %Schema{ + type: :integer, + nullable: true, + description: "Expire the mute in `duration` seconds. Default 0 for infinity" + } + }, + example: %{ + "duration" => 86_400 + } + } + end + def unblock_operation do %Operation{ tags: ["Account actions"], diff --git a/lib/pleroma/web/api_spec/schemas/account.ex b/lib/pleroma/web/api_spec/schemas/account.ex index 1f73ef60c..19827e996 100644 --- a/lib/pleroma/web/api_spec/schemas/account.ex +++ b/lib/pleroma/web/api_spec/schemas/account.ex @@ -34,6 +34,7 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Account do id: FlakeID, locked: %Schema{type: :boolean}, mute_expires_at: %Schema{type: :string, format: "date-time", nullable: true}, + block_expires_at: %Schema{type: :string, format: "date-time", nullable: true}, note: %Schema{type: :string, format: :html}, statuses_count: %Schema{type: :integer}, url: %Schema{type: :string, format: :uri}, diff --git a/lib/pleroma/web/common_api.ex b/lib/pleroma/web/common_api.ex index 412424dae..ae554d0b9 100644 --- a/lib/pleroma/web/common_api.ex +++ b/lib/pleroma/web/common_api.ex @@ -27,9 +27,9 @@ defmodule Pleroma.Web.CommonAPI do require Logger @spec block(User.t(), User.t()) :: {:ok, Activity.t()} | Pipeline.errors() - def block(blocked, blocker) do - with {:ok, block_data, _} <- Builder.block(blocker, blocked), - {:ok, block, _} <- Pipeline.common_pipeline(block_data, local: true) do + def block(blocked, blocker, params \\ %{}) do + with {:ok, block_data, meta} <- Builder.block(blocker, blocked, params), + {:ok, block, _} <- Pipeline.common_pipeline(block_data, meta ++ [local: true]) do {:ok, block} end end diff --git a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex index 68157b0c4..d374e8c01 100644 --- a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex @@ -501,8 +501,14 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do end @doc "POST /api/v1/accounts/:id/block" - def block(%{assigns: %{user: blocker, account: blocked}} = conn, _params) do - with {:ok, _activity} <- CommonAPI.block(blocked, blocker) do + def block( + %{ + assigns: %{user: blocker, account: blocked}, + private: %{open_api_spex: %{body_params: params}} + } = conn, + _params + ) do + with {:ok, _activity} <- CommonAPI.block(blocked, blocker, params) do render(conn, "relationship.json", user: blocker, target: blocked) else {:error, message} -> json_response(conn, :forbidden, %{error: message}) @@ -607,7 +613,8 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do users: users, for: user, as: :user, - embed_relationships: embed_relationships?(params) + embed_relationships: embed_relationships?(params), + blocks: true ) end diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index f6727d29d..8d28dd69a 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -340,6 +340,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do |> maybe_put_unread_notification_count(user, opts[:for]) |> maybe_put_email_address(user, opts[:for]) |> maybe_put_mute_expires_at(user, opts[:for], opts) + |> maybe_put_block_expires_at(user, opts[:for], opts) |> maybe_show_birthday(user, opts[:for]) end @@ -476,6 +477,16 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do defp maybe_put_mute_expires_at(data, _, _, _), do: data + defp maybe_put_block_expires_at(data, %User{} = user, target, %{blocks: true}) do + Map.put( + data, + :block_expires_at, + UserRelationship.get_block_expire_date(target, user) + ) + end + + defp maybe_put_block_expires_at(data, _, _, _), do: data + defp maybe_show_birthday(data, %User{id: user_id} = user, %User{id: user_id}) do data |> Kernel.put_in([:pleroma, :birthday], user.birthday) diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index fd72e2f91..5894c764b 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -157,7 +157,8 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do "pleroma:bookmark_folders", if Pleroma.Language.LanguageDetector.configured?() do "pleroma:language_detection" - end + end, + "pleroma:block_expiration" ] |> Enum.filter(& &1) end diff --git a/lib/pleroma/workers/mute_expire_worker.ex b/lib/pleroma/workers/mute_expire_worker.ex index 8356a775d..9a04fc486 100644 --- a/lib/pleroma/workers/mute_expire_worker.ex +++ b/lib/pleroma/workers/mute_expire_worker.ex @@ -5,9 +5,13 @@ defmodule Pleroma.Workers.MuteExpireWorker do use Oban.Worker, queue: :background + alias Pleroma.User + @impl true - def perform(%Job{args: %{"op" => "unmute_user", "muter_id" => muter_id, "mutee_id" => mutee_id}}) do - Pleroma.User.unmute(muter_id, mutee_id) + def perform(%Job{ + args: %{"op" => "unmute_user", "muter_id" => muter_id, "mutee_id" => mutee_id} + }) do + User.unmute(muter_id, mutee_id) :ok end @@ -18,6 +22,17 @@ defmodule Pleroma.Workers.MuteExpireWorker do :ok end + def perform(%Job{ + args: %{"op" => "unblock_user", "blocker_id" => blocker_id, "blocked_id" => blocked_id} + }) do + Pleroma.Web.CommonAPI.unblock( + User.get_cached_by_id(blocked_id), + User.get_cached_by_id(blocker_id) + ) + + :ok + end + @impl true def timeout(_job), do: :timer.seconds(5) end diff --git a/test/pleroma/web/common_api_test.exs b/test/pleroma/web/common_api_test.exs index 73230a58c..6b5d31537 100644 --- a/test/pleroma/web/common_api_test.exs +++ b/test/pleroma/web/common_api_test.exs @@ -111,6 +111,17 @@ defmodule Pleroma.Web.CommonAPITest do end end + test "add expiring block", %{blocker: blocker, blocked: blocked} do + {:ok, _} = CommonAPI.block(blocked, blocker, %{expires_in: 60}) + assert User.blocks?(blocker, blocked) + + worker = Pleroma.Workers.MuteExpireWorker + args = %{"op" => "unblock_user", "blocker_id" => blocker.id, "blocked_id" => blocked.id} + + assert :ok = perform_job(worker, args) + refute User.blocks?(blocker, blocked) + end + test "it blocks and does not federate if outgoing blocks are disabled", %{ blocker: blocker, blocked: blocked From ded40182b0aa6848b55febe73ec7e41eace1e0f6 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Mon, 5 May 2025 15:28:02 +0400 Subject: [PATCH 083/128] Public getting stripped from unlisted activity CC: Add possible tests --- test/fixtures/poast_unlisted.json | 65 +++++++++++++++++++ .../transmogrifier/note_handling_test.exs | 31 +++++++++ .../web/activity_pub/transmogrifier_test.exs | 37 +++++++++++ 3 files changed, 133 insertions(+) create mode 100644 test/fixtures/poast_unlisted.json diff --git a/test/fixtures/poast_unlisted.json b/test/fixtures/poast_unlisted.json new file mode 100644 index 000000000..fa23153ba --- /dev/null +++ b/test/fixtures/poast_unlisted.json @@ -0,0 +1,65 @@ +{ + "@context" : [ + "https://www.w3.org/ns/activitystreams", + "https://poa.st/schemas/litepub-0.1.jsonld", + { + "@language" : "und" + } + ], + "actor" : "https://poa.st/users/TrevorGoodchild", + "attachment" : [], + "attributedTo" : "https://poa.st/users/TrevorGoodchild", + "cc" : [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "context" : "https://poa.st/contexts/c6d125f1-4e7f-43bd-aa31-33de4d90d049", + "conversation" : "https://poa.st/contexts/c6d125f1-4e7f-43bd-aa31-33de4d90d049", + "directMessage" : false, + "id" : "https://poa.st/activities/bbd3347a-4a89-4cdb-bf86-4f9eed9506e3", + "object" : { + "actor" : "https://poa.st/users/TrevorGoodchild", + "attachment" : [], + "attributedTo" : "https://poa.st/users/TrevorGoodchild", + "cc" : [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "content" : "@HoroTheWhiteWolf >please let this be his zero fucks given final statement before he joins the 52%+ tranny club", + "context" : "https://poa.st/contexts/c6d125f1-4e7f-43bd-aa31-33de4d90d049", + "conversation" : "https://poa.st/contexts/c6d125f1-4e7f-43bd-aa31-33de4d90d049", + "id" : "https://poa.st/objects/7eb785d5-a556-4070-9091-f4afb226466c", + "inReplyTo" : "https://poa.st/objects/71995b41-cfb2-48ce-abce-76d570d54edc", + "published" : "2025-05-03T23:54:07.489885Z", + "repliesCount" : 2, + "sensitive" : false, + "source" : { + "content" : ">please let this be his zero fucks given final statement before he joins the 52%+ tranny club", + "mediaType" : "text/plain" + }, + "summary" : "", + "tag" : [ + { + "href" : "https://poa.st/users/HoroTheWhiteWolf", + "name" : "@HoroTheWhiteWolf", + "type" : "Mention" + } + ], + "to" : [ + "https://poa.st/users/HoroTheWhiteWolf", + "https://poa.st/users/TrevorGoodchild/followers" + ], + "type" : "Note" + }, + "published" : "2025-05-03T23:54:07.489837Z", + "tag" : [ + { + "href" : "https://poa.st/users/HoroTheWhiteWolf", + "name" : "@HoroTheWhiteWolf", + "type" : "Mention" + } + ], + "to" : [ + "https://poa.st/users/HoroTheWhiteWolf", + "https://poa.st/users/TrevorGoodchild/followers" + ], + "type" : "Create" +} diff --git a/test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs b/test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs index fd7a3c772..13982940a 100644 --- a/test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs +++ b/test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs @@ -786,4 +786,35 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.NoteHandlingTest do assert object.data["context"] == object.data["inReplyTo"] assert modified.data["context"] == object.data["inReplyTo"] end + + test "it keeps the public address in cc in the activity when it is present" do + data = + File.read!("test/fixtures/mastodon-post-activity.json") + |> Jason.decode!() + + object = + data["object"] + |> Map.put("cc", ["https://www.w3.org/ns/activitystreams#Public"]) + |> Map.put("to", []) + + data = + data + |> Map.put("object", object) + |> Map.put("cc", ["https://www.w3.org/ns/activitystreams#Public"]) + |> Map.put("to", []) + + {:ok, %Activity{} = modified} = Transmogrifier.handle_incoming(data) + assert modified.data["cc"] == ["https://www.w3.org/ns/activitystreams#Public"] + end + + test "it tries it with the real poast_unlisted.json, ensuring that public is in the cc" do + data = + File.read!("test/fixtures/poast_unlisted.json") + |> Jason.decode!() + + _user = insert(:user, ap_id: data["actor"]) + + {:ok, %Activity{} = modified} = Transmogrifier.handle_incoming(data) + assert modified.data["cc"] == ["https://www.w3.org/ns/activitystreams#Public"] + end end diff --git a/test/pleroma/web/activity_pub/transmogrifier_test.exs b/test/pleroma/web/activity_pub/transmogrifier_test.exs index e0395d7bb..ef6e004f1 100644 --- a/test/pleroma/web/activity_pub/transmogrifier_test.exs +++ b/test/pleroma/web/activity_pub/transmogrifier_test.exs @@ -757,6 +757,43 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do refute recipient.follower_address in fixed_object["cc"] refute recipient.follower_address in fixed_object["to"] end + + test "preserves public URL in cc even when not explicitly mentioned", %{user: user} do + public_url = "https://www.w3.org/ns/activitystreams#Public" + + # Case 1: Public URL in cc but no mentions + object = %{ + "actor" => user.ap_id, + "to" => ["https://social.beepboop.ga/users/dirb"], + "cc" => [public_url], + "tag" => [] + } + + fixed_object = Transmogrifier.fix_explicit_addressing(object, user.follower_address) + assert public_url in fixed_object["cc"] + + # Case 2: Public URL in cc, with mentions but public not in to + object = %{ + "actor" => user.ap_id, + "to" => ["https://pleroma.gold/users/user1"], + "cc" => [public_url], + "tag" => [%{"type" => "Mention", "href" => "https://pleroma.gold/users/user1"}] + } + + fixed_object = Transmogrifier.fix_explicit_addressing(object, user.follower_address) + assert public_url in fixed_object["cc"] + + # Case 3: Public URL in to, it should be moved to to + object = %{ + "actor" => user.ap_id, + "to" => [public_url], + "cc" => [], + "tag" => [] + } + + fixed_object = Transmogrifier.fix_explicit_addressing(object, user.follower_address) + assert public_url in fixed_object["to"] + end end describe "fix_summary/1" do From 31071973b73fd545a7e2c9ae0119539c7bcc301a Mon Sep 17 00:00:00 2001 From: mkljczk Date: Tue, 6 May 2025 21:48:17 +0200 Subject: [PATCH 084/128] Fix typo in account_status function doc --- lib/pleroma/user.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index d9da9ede1..a5672fe4a 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -308,7 +308,7 @@ defmodule Pleroma.User do def binary_id(%User{} = user), do: binary_id(user.id) - @doc "Returns status account" + @doc "Returns account status" @spec account_status(User.t()) :: account_status() def account_status(%User{is_active: false}), do: :deactivated def account_status(%User{password_reset_pending: true}), do: :password_reset_pending From ccb5b81179395a65cceb38a27a53f8c8241d6d70 Mon Sep 17 00:00:00 2001 From: mkljczk Date: Tue, 6 May 2025 21:48:39 +0200 Subject: [PATCH 085/128] Update changelog --- changelog.d/doc-typo.skip | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 changelog.d/doc-typo.skip diff --git a/changelog.d/doc-typo.skip b/changelog.d/doc-typo.skip new file mode 100644 index 000000000..e69de29bb From 63afd9a22d80beefc6bf182373db2e4cea256c0a Mon Sep 17 00:00:00 2001 From: mkljczk Date: Wed, 7 May 2025 17:29:27 +0200 Subject: [PATCH 086/128] Fix condition for moderation log force_password_reset action --- changelog.d/admin-api-log-fix.skip | 0 .../admin_api/controllers/admin_api_controller.ex | 12 ++++++------ 2 files changed, 6 insertions(+), 6 deletions(-) create mode 100644 changelog.d/admin-api-log-fix.skip diff --git a/changelog.d/admin-api-log-fix.skip b/changelog.d/admin-api-log-fix.skip new file mode 100644 index 000000000..e69de29bb diff --git a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex index 0f22dd538..b35f5cdcd 100644 --- a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex @@ -335,13 +335,13 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do if params["password"] do User.force_password_reset_async(user) - end - ModerationLog.insert_log(%{ - actor: admin, - subject: [user], - action: "force_password_reset" - }) + ModerationLog.insert_log(%{ + actor: admin, + subject: [user], + action: "force_password_reset" + }) + end json(conn, %{status: "success"}) else From 68a5c6011356457cd2639d1f4b4da6347f8b4f9f Mon Sep 17 00:00:00 2001 From: mkljczk Date: Thu, 8 May 2025 13:45:22 +0200 Subject: [PATCH 087/128] another doc update --- lib/pleroma/user.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index a5672fe4a..8fd8e164d 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -2615,7 +2615,7 @@ defmodule Pleroma.User do end end - # Internal function; public one is `deactivate/2` + # Internal function; public one is `set_activation/2` defp set_activation_status(user, status) do user |> cast(%{is_active: status}, [:is_active]) From 53d7b205e8539795f57ded3b8c4329f0f9bbee22 Mon Sep 17 00:00:00 2001 From: Phantasm Date: Mon, 12 May 2025 16:17:32 +0200 Subject: [PATCH 088/128] Elixir 1.18 <%# deprecated syntax warning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit warning: <%# is deprecated, use <%!-- or add a space between <% and # instead │ 5 │ <%# Note: using hidden input with `unchecked_value` in order to distinguish user's empty selection from `scope` param being omitted %> │ ~ │ └─ lib/pleroma/web/templates/o_auth/o_auth/_scopes.html.eex:5: (file) --- .../web/templates/email/digest.html.eex | 20 +++++++++---------- .../templates/email/new_users_digest.html.eex | 10 +++++----- .../templates/layout/email_styled.html.eex | 4 ++-- .../templates/o_auth/o_auth/_scopes.html.eex | 2 +- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/lib/pleroma/web/templates/email/digest.html.eex b/lib/pleroma/web/templates/email/digest.html.eex index 1efc76e1a..d2918bc6f 100644 --- a/lib/pleroma/web/templates/email/digest.html.eex +++ b/lib/pleroma/web/templates/email/digest.html.eex @@ -231,8 +231,8 @@ <%= for %{data: mention, object: object, from: from} <- @mentions do %> - <%# mention START %> - <%# user card START %> + <% # mention START %> + <% # user card START %>
@@ -291,7 +291,7 @@
- <%# user card END %> + <% # user card END %>
- <%# mention END %> + <% # mention END %> <% end %> <%= if @followers != [] do %> - <%# new followers header START %> + <% # new followers header START %>
@@ -397,10 +397,10 @@
- <%# new followers header END %> + <% # new followers header END %> <%= for %{data: follow, from: from} <- @followers do %> - <%# user card START %> + <% # user card START %>
@@ -459,13 +459,13 @@
- <%# user card END %> + <% # user card END %> <% end %> <% end %> - <%# divider start %> + <% # divider start %>
@@ -514,7 +514,7 @@
- <%# divider end %> + <% # divider end %>
diff --git a/lib/pleroma/web/templates/email/new_users_digest.html.eex b/lib/pleroma/web/templates/email/new_users_digest.html.eex index 40d9b8381..78b8ac4f9 100644 --- a/lib/pleroma/web/templates/email/new_users_digest.html.eex +++ b/lib/pleroma/web/templates/email/new_users_digest.html.eex @@ -1,5 +1,5 @@ <%= for {user, total_statuses, latest_status} <- @users_and_statuses do %> - <%# user card START %> + <% # user card START %>
@@ -60,7 +60,7 @@
- <%# user card END %> + <% # user card END %> <%= if latest_status do %>
@@ -104,7 +104,7 @@
<% end %> - <%# divider start %> + <% # divider start %>
@@ -153,6 +153,6 @@
- <%# divider end %> - <%# user card END %> + <% # divider end %> + <% # user card END %> <% end %> diff --git a/lib/pleroma/web/templates/layout/email_styled.html.eex b/lib/pleroma/web/templates/layout/email_styled.html.eex index 82cabd889..a1ed4ece3 100644 --- a/lib/pleroma/web/templates/layout/email_styled.html.eex +++ b/lib/pleroma/web/templates/layout/email_styled.html.eex @@ -111,7 +111,7 @@ - <%# header %> + <% # header %>
@@ -145,7 +145,7 @@
- <%# title %> + <% # title %> <%= if @title do %>
<%= for scope <- @available_scopes do %> - <%# Note: using hidden input with `unchecked_value` in order to distinguish user's empty selection from `scope` param being omitted %> + <% # Note: using hidden input with `unchecked_value` in order to distinguish user's empty selection from `scope` param being omitted %> <%= if scope in @scopes do %>
<%= checkbox @form, :"scope_#{scope}", value: scope in @scopes && scope, checked_value: scope, unchecked_value: "", name: "authorization[scope][]" %> From 25e7b12a6bd870d96becbd79167818147f6b501c Mon Sep 17 00:00:00 2001 From: Phantasm Date: Mon, 12 May 2025 17:21:41 +0200 Subject: [PATCH 089/128] Elixir 1.18 Remove seemingly unneeded cond MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit warning: this clause in cond will always match: <<"#", name::binary>> since it has type: binary() where "name" was given the type: %{"type" => "Hashtag", "name" => name} = data typing violation found at: │ 55 │ "#" <> name -> name │ ~ │ └─ lib/pleroma/web/activity_pub/object_validators/tag_validator.ex:55:21: Pleroma.Web.ActivityPub.ObjectValidators.TagValidator.changeset/2 --- .../web/activity_pub/object_validators/tag_validator.ex | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex b/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex index 47cf7b415..411517045 100644 --- a/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex @@ -50,12 +50,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.TagValidator do end def changeset(struct, %{"type" => "Hashtag", "name" => name} = data) do - name = - cond do - "#" <> name -> name - name -> name - end - |> String.downcase() + name = String.downcase(name) data = Map.put(data, "name", name) From 59d17a5b20bac485c189bcfdeafffe7fb06c8277 Mon Sep 17 00:00:00 2001 From: Phantasm Date: Mon, 12 May 2025 17:23:33 +0200 Subject: [PATCH 090/128] Elixir 1.18 Move Update activity validation to separate function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit warning: Pleroma.Web.ActivityPub.ObjectValidators.AcceptRejectValidator.cast_and_validate/2 is undefined or private. Did you mean: * cast_and_validate/1 │ 227 │ validator == UpdateValidator -> fn o -> validator.cast_and_validate(o, meta) end │ ~ │ └─ lib/pleroma/web/activity_pub/object_validator.ex:227:57: Pleroma.Web.ActivityPub.ObjectValidator.validate/2 --- .../web/activity_pub/object_validator.ex | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/lib/pleroma/web/activity_pub/object_validator.ex b/lib/pleroma/web/activity_pub/object_validator.ex index ee12f3ebf..17652a0de 100644 --- a/lib/pleroma/web/activity_pub/object_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validator.ex @@ -200,14 +200,13 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do end def validate(%{"type" => type} = object, meta) - when type in ~w[Accept Reject Follow Update Like EmojiReact Announce + when type in ~w[Accept Reject Follow Like EmojiReact Announce ChatMessage Answer] do validator = case type do "Accept" -> AcceptRejectValidator "Reject" -> AcceptRejectValidator "Follow" -> FollowValidator - "Update" -> UpdateValidator "Like" -> LikeValidator "EmojiReact" -> EmojiReactValidator "Announce" -> AnnounceValidator @@ -215,16 +214,19 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do "Answer" -> AnswerValidator end - cast_func = - if type == "Update" do - fn o -> validator.cast_and_validate(o, meta) end - else - fn o -> validator.cast_and_validate(o) end - end - with {:ok, object} <- object - |> cast_func.() + |> validator.cast_and_validate() + |> Ecto.Changeset.apply_action(:insert) do + object = stringify_keys(object) + {:ok, object, meta} + end + end + + def validate(%{"type" => type} = object, meta) when type == "Update" do + with {:ok, object} <- + object + |> UpdateValidator.cast_and_validate(meta) |> Ecto.Changeset.apply_action(:insert) do object = stringify_keys(object) {:ok, object, meta} From 63cbc1208d2654ed174f7d319334aca3e08f69d7 Mon Sep 17 00:00:00 2001 From: Phantasm Date: Mon, 12 May 2025 17:25:38 +0200 Subject: [PATCH 091/128] Elixir 1.18 Replace Tuple.append/2 with Tuple.insert_at/3 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit warning: Tuple.append/2 is deprecated. Use insert_at instead │ 305 │ Enum.reduce(entity, {}, &Tuple.append(&2, to_elixir_types(&1))) │ ~ │ └─ lib/pleroma/config_db.ex:305:36: Pleroma.ConfigDB.to_elixir_types/1 --- lib/pleroma/config_db.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/config_db.ex b/lib/pleroma/config_db.ex index 89d3050d6..e9990fa35 100644 --- a/lib/pleroma/config_db.ex +++ b/lib/pleroma/config_db.ex @@ -302,7 +302,7 @@ defmodule Pleroma.ConfigDB do end def to_elixir_types(%{"tuple" => entity}) do - Enum.reduce(entity, {}, &Tuple.append(&2, to_elixir_types(&1))) + Enum.reduce(entity, {}, &Tuple.insert_at(&2, tuple_size(&2), to_elixir_types(&1))) end def to_elixir_types(entity) when is_map(entity) do From 5addbf39fbdc67d93f6b8605ce02157e14c3edb1 Mon Sep 17 00:00:00 2001 From: Phantasm Date: Tue, 13 May 2025 00:01:34 +0200 Subject: [PATCH 092/128] Elixir 1.18 Deal with :warnings_as_errors deprecation in compiler_options/1 warning: :warnings_as_errors is deprecated as part of Code.get_compiler_option/1 (elixir 1.18.3) lib/code.ex:1597: Code.get_compiler_option/1 (elixir 1.18.3) lib/code.ex:1572: anonymous fn/2 in Code.compiler_options/1 (elixir 1.18.3) lib/enum.ex:2546: Enum."-reduce/3-lists^foldl/2-0-"/3 (elixir 1.18.3) lib/code.ex:1571: Code.compiler_options/1 (pleroma 2.9.1-77-g8ec49c59-elixir-1-18+test) lib/pleroma/application.ex:104: Pleroma.Application.start/2 (kernel 10.2.6) application_master.erl:295: :application_master.start_it_old/4 warning: :warnings_as_errors is deprecated as part of Code.put_compiler_option/2, instead you must pass it as a --warnings-as-errors flag. If you need to set it as a default in a mix task, you can also set it under aliases: [compile: "compile --warnings-as-errors"] (elixir 1.18.3) lib/code.ex:1710: Code.put_compiler_option/2 (elixir 1.18.3) lib/code.ex:1573: anonymous fn/2 in Code.compiler_options/1 (elixir 1.18.3) lib/enum.ex:2546: Enum."-reduce/3-lists^foldl/2-0-"/3 (elixir 1.18.3) lib/code.ex:1571: Code.compiler_options/1 (pleroma 2.9.1-77-g8ec49c59-elixir-1-18+test) lib/pleroma/application.ex:104: Pleroma.Application.start/2 (kernel 10.2.6) application_master.erl:295: :application_master.start_it_old/4 --- lib/mix/tasks/pleroma/test_runner.ex | 2 +- lib/pleroma/application.ex | 18 +++++++++++++++--- mix.exs | 2 +- test/test_helper.exs | 2 -- 4 files changed, 17 insertions(+), 7 deletions(-) diff --git a/lib/mix/tasks/pleroma/test_runner.ex b/lib/mix/tasks/pleroma/test_runner.ex index 69fefb001..d9cf0d445 100644 --- a/lib/mix/tasks/pleroma/test_runner.ex +++ b/lib/mix/tasks/pleroma/test_runner.ex @@ -4,7 +4,7 @@ defmodule Mix.Tasks.Pleroma.TestRunner do use Mix.Task def run(args \\ []) do - case System.cmd("mix", ["test"] ++ args, into: IO.stream(:stdio, :line)) do + case System.cmd("mix", ["test", "--warnings-as-errors"] ++ args, into: IO.stream(:stdio, :line)) do {_, 0} -> :ok diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 497623ee1..fd3c66c63 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -43,9 +43,6 @@ defmodule Pleroma.Application do # every time the application is restarted, so we disable module # conflicts at runtime Code.compiler_options(ignore_module_conflict: true) - # Disable warnings_as_errors at runtime, it breaks Phoenix live reload - # due to protocol consolidation warnings - Code.compiler_options(warnings_as_errors: false) Pleroma.Telemetry.Logger.attach() Config.Holder.save_default() Pleroma.HTML.compile_scrubbers() @@ -93,6 +90,21 @@ defmodule Pleroma.Application do end end + # Disable warnings_as_errors at runtime, it breaks Phoenix live reload + # due to protocol consolidation warnings + # :warnings_as_errors is deprecated via Code.compiler_options/2 since 1.18 + if elixir_version = System.version() do + [major, minor] = + elixir_version + |> String.split(".") + |> Enum.map(&String.to_integer/1) + |> Enum.take(2) + + if major == 1 and minor < 18 do + Code.compiler_options(warnings_as_errors: false) + end + end + # Define workers and child supervisors to be supervised children = [ diff --git a/mix.exs b/mix.exs index 808a2b12c..dc6c2492f 100644 --- a/mix.exs +++ b/mix.exs @@ -236,7 +236,7 @@ defmodule Pleroma.Mixfile do "ecto.rollback": ["pleroma.ecto.rollback"], "ecto.setup": ["ecto.create", "ecto.migrate", "run priv/repo/seeds.exs"], "ecto.reset": ["ecto.drop", "ecto.setup"], - test: ["ecto.create --quiet", "ecto.migrate", "test"], + test: ["ecto.create --quiet", "ecto.migrate", "test --warnings-as-errors"], docs: ["pleroma.docs", "docs"], analyze: ["credo --strict --only=warnings,todo,fixme,consistency,readability"], copyright: &add_copyright/1, diff --git a/test/test_helper.exs b/test/test_helper.exs index 94661353b..dc6c05a74 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -2,8 +2,6 @@ # Copyright © 2017-2022 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -Code.put_compiler_option(:warnings_as_errors, true) - ExUnit.configure(capture_log: true, max_cases: System.schedulers_online()) ExUnit.start(exclude: [:federated]) From 7c13abb3d98fdac4fdab67828e7fe509ad868431 Mon Sep 17 00:00:00 2001 From: Phantasm Date: Wed, 14 May 2025 16:37:43 +0200 Subject: [PATCH 093/128] Elixir 1.18 Use NaiveDateTime.compare/2 instead of <>= comparisons MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit warning: comparison with structs found: left <= right given types: dynamic() <= dynamic(%NaiveDateTime{}) where "left" (context ExUnit.Assertions) was given the type: # type: dynamic() # from: test/pleroma/web/plugs/user_tracking_plug_test.exs:25 left = user.last_active_at where "right" (context ExUnit.Assertions) was given the type: # type: dynamic(%NaiveDateTime{}) # from: test/pleroma/web/plugs/user_tracking_plug_test.exs:25 right = NaiveDateTime.truncate(NaiveDateTime.utc_now(), :second) Comparison operators (>, <, >=, <=, min, and max) perform structural and not semantic comparison. Comparing with a struct won't give meaningful results. Structs that can be compared typically define a compare/2 function within their modules that can be used for semantic comparison. typing violation found at: │ 25 │ assert user.last_active_at <= NaiveDateTime.truncate(NaiveDateTime.utc_now(), :second) │ ~ │ └─ test/pleroma/web/plugs/user_tracking_plug_test.exs:25:32: Pleroma.Web.Plugs.UserTrackingPlugTest."test updates last_active_at for a new user"/1 --- test/pleroma/user_test.exs | 10 +++++----- test/pleroma/web/plugs/user_tracking_plug_test.exs | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/test/pleroma/user_test.exs b/test/pleroma/user_test.exs index 176e70ef9..79a480f85 100644 --- a/test/pleroma/user_test.exs +++ b/test/pleroma/user_test.exs @@ -2669,8 +2669,8 @@ defmodule Pleroma.UserTest do assert {:ok, user} = User.update_last_active_at(user) - assert user.last_active_at >= test_started_at - assert user.last_active_at <= NaiveDateTime.truncate(NaiveDateTime.utc_now(), :second) + assert NaiveDateTime.compare(user.last_active_at, test_started_at) in [:gt, :eq] + assert NaiveDateTime.compare(user.last_active_at, NaiveDateTime.truncate(NaiveDateTime.utc_now(), :second)) in [:lt, :eq] last_active_at = NaiveDateTime.utc_now() @@ -2681,11 +2681,11 @@ defmodule Pleroma.UserTest do user |> cast(%{last_active_at: last_active_at}, [:last_active_at]) |> User.update_and_set_cache() + assert NaiveDateTime.compare(user.last_active_at, last_active_at) == :eq - assert user.last_active_at == last_active_at assert {:ok, user} = User.update_last_active_at(user) - assert user.last_active_at >= test_started_at - assert user.last_active_at <= NaiveDateTime.truncate(NaiveDateTime.utc_now(), :second) + assert NaiveDateTime.compare(user.last_active_at, test_started_at) in [:gt, :eq] + assert NaiveDateTime.compare(user.last_active_at, NaiveDateTime.truncate(NaiveDateTime.utc_now(), :second)) in [:lt, :eq] end test "active_user_count/1" do diff --git a/test/pleroma/web/plugs/user_tracking_plug_test.exs b/test/pleroma/web/plugs/user_tracking_plug_test.exs index 742f04fea..5c67a7735 100644 --- a/test/pleroma/web/plugs/user_tracking_plug_test.exs +++ b/test/pleroma/web/plugs/user_tracking_plug_test.exs @@ -21,8 +21,8 @@ defmodule Pleroma.Web.Plugs.UserTrackingPlugTest do |> assign(:user, user) |> UserTrackingPlug.call(%{}) - assert user.last_active_at >= test_started_at - assert user.last_active_at <= NaiveDateTime.truncate(NaiveDateTime.utc_now(), :second) + assert NaiveDateTime.compare(user.last_active_at, test_started_at) in [:gt, :eq] + assert NaiveDateTime.compare(user.last_active_at, NaiveDateTime.truncate(NaiveDateTime.utc_now(), :second)) in [:lt, :eq] end test "doesn't update last_active_at if it was updated recently", %{conn: conn} do @@ -38,7 +38,7 @@ defmodule Pleroma.Web.Plugs.UserTrackingPlugTest do |> assign(:user, user) |> UserTrackingPlug.call(%{}) - assert user.last_active_at == last_active_at + assert NaiveDateTime.compare(user.last_active_at, last_active_at) == :eq end test "skips updating last_active_at if user ID is nil", %{conn: conn} do From af81f7bf82ff4ee0ed2f5794cdf4e28a5a43eca2 Mon Sep 17 00:00:00 2001 From: Phantasm Date: Wed, 14 May 2025 17:00:19 +0200 Subject: [PATCH 094/128] Don't use deprecated function invocation syntax warning: using map.field notation (without parentheses) to invoke function TranslationMock.configured?() is deprecated, you must add parentheses instead: remote.function() --- lib/pleroma/language/language_detector.ex | 4 ++-- lib/pleroma/language/translation.ex | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/language/language_detector.ex b/lib/pleroma/language/language_detector.ex index 16e2d4faa..68d243562 100644 --- a/lib/pleroma/language/language_detector.ex +++ b/lib/pleroma/language/language_detector.ex @@ -12,7 +12,7 @@ defmodule Pleroma.Language.LanguageDetector do def configured? do provider = get_provider() - !!provider and provider.configured? + !!provider and provider.configured?() end def missing_dependencies do @@ -41,7 +41,7 @@ defmodule Pleroma.Language.LanguageDetector do text = prepare_text(text) word_count = text |> String.split(~r/\s+/) |> Enum.count() - if word_count < @words_threshold or !provider or !provider.configured? do + if word_count < @words_threshold or !provider or !provider.configured?() do nil else with language <- provider.detect(text), diff --git a/lib/pleroma/language/translation.ex b/lib/pleroma/language/translation.ex index 3706e76eb..64f115ed8 100644 --- a/lib/pleroma/language/translation.ex +++ b/lib/pleroma/language/translation.ex @@ -8,7 +8,7 @@ defmodule Pleroma.Language.Translation do def configured? do provider = get_provider() - !!provider and provider.configured? + !!provider and provider.configured?() end def missing_dependencies do From 4c8a93a06d7c8226f4da8a692e16d9f9610450c9 Mon Sep 17 00:00:00 2001 From: mkljczk Date: Wed, 7 May 2025 19:32:13 +0200 Subject: [PATCH 095/128] Pleroma.User: Mark some functions as private MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicole Mikołajczyk --- changelog.d/private-functions.skip | 0 lib/pleroma/user.ex | 20 ++++++++++---------- 2 files changed, 10 insertions(+), 10 deletions(-) create mode 100644 changelog.d/private-functions.skip diff --git a/changelog.d/private-functions.skip b/changelog.d/private-functions.skip new file mode 100644 index 000000000..e69de29bb diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index d9da9ede1..8b00cf522 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -895,7 +895,7 @@ defmodule Pleroma.User do end) end - def validate_email_not_in_blacklisted_domain(changeset, field) do + defp validate_email_not_in_blacklisted_domain(changeset, field) do validate_change(changeset, field, fn _, value -> valid? = Config.get([User, :email_blacklist]) @@ -912,9 +912,9 @@ defmodule Pleroma.User do end) end - def maybe_validate_required_email(changeset, true), do: changeset + defp maybe_validate_required_email(changeset, true), do: changeset - def maybe_validate_required_email(changeset, _) do + defp maybe_validate_required_email(changeset, _) do if Config.get([:instance, :account_activation_required]) do validate_required(changeset, [:email]) else @@ -1109,15 +1109,15 @@ defmodule Pleroma.User do defp maybe_send_registration_email(_), do: {:ok, :noop} - def needs_update?(%User{local: true}), do: false + defp needs_update?(%User{local: true}), do: false - def needs_update?(%User{local: false, last_refreshed_at: nil}), do: true + defp needs_update?(%User{local: false, last_refreshed_at: nil}), do: true - def needs_update?(%User{local: false} = user) do + defp needs_update?(%User{local: false} = user) do NaiveDateTime.diff(NaiveDateTime.utc_now(), user.last_refreshed_at) >= 86_400 end - def needs_update?(_), do: true + defp needs_update?(_), do: true @spec maybe_direct_follow(User.t(), User.t()) :: {:ok, User.t(), User.t()} | {:error, String.t()} @@ -1984,7 +1984,7 @@ defmodule Pleroma.User do end @spec purge_user_changeset(User.t()) :: Ecto.Changeset.t() - def purge_user_changeset(user) do + defp purge_user_changeset(user) do # "Right to be forgotten" # https://gdpr.eu/right-to-be-forgotten/ change(user, %{ @@ -2156,7 +2156,7 @@ defmodule Pleroma.User do Repo.all(query) end - def delete_notifications_from_user_activities(%User{ap_id: ap_id}) do + defp delete_notifications_from_user_activities(%User{ap_id: ap_id}) do Notification |> join(:inner, [n], activity in assoc(n, :activity)) |> where([n, a], fragment("? = ?", a.actor, ^ap_id)) @@ -2634,7 +2634,7 @@ defmodule Pleroma.User do |> update_and_set_cache() end - def validate_fields(changeset, remote? \\ false) do + defp validate_fields(changeset, remote?) do limit_name = if remote?, do: :max_remote_account_fields, else: :max_account_fields limit = Config.get([:instance, limit_name], 0) From 6b38ec310a636ff3e9aab2ea85fe2d019a5a7720 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicole=20Miko=C5=82ajczyk?= Date: Thu, 22 May 2025 20:52:07 +0200 Subject: [PATCH 096/128] Fix 'Create a user' description in admin api docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicole Mikołajczyk --- changelog.d/admin-api-docs-fix.skip | 1 + docs/development/API/admin_api.md | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 changelog.d/admin-api-docs-fix.skip diff --git a/changelog.d/admin-api-docs-fix.skip b/changelog.d/admin-api-docs-fix.skip new file mode 100644 index 000000000..5c1c68ea0 --- /dev/null +++ b/changelog.d/admin-api-docs-fix.skip @@ -0,0 +1 @@ +Fix 'Create a user' description in admin api docs diff --git a/docs/development/API/admin_api.md b/docs/development/API/admin_api.md index 409e78a1e..64c06ca2b 100644 --- a/docs/development/API/admin_api.md +++ b/docs/development/API/admin_api.md @@ -70,6 +70,8 @@ The `/api/v1/pleroma/admin/*` path is backwards compatible with `/api/pleroma/ad - `nicknames` - Response: Array of user nicknames +## `POST /api/v1/pleroma/admin/users` + ### Create a user - Method: `POST` @@ -81,7 +83,7 @@ The `/api/v1/pleroma/admin/*` path is backwards compatible with `/api/pleroma/ad `password` } ] -- Response: User’s nickname +- Response: Array of user objects ## `POST /api/v1/pleroma/admin/users/follow` From a0dfa12b78d071164c13e88d23336a60d9bfa9a8 Mon Sep 17 00:00:00 2001 From: Phantasm Date: Sat, 24 May 2025 21:59:24 +0200 Subject: [PATCH 097/128] Elixir 1.18 Update supported versions for Erlang OTP and Elixir --- docs/installation/generic_dependencies.include | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/installation/generic_dependencies.include b/docs/installation/generic_dependencies.include index 9f07f62c6..769347a3c 100644 --- a/docs/installation/generic_dependencies.include +++ b/docs/installation/generic_dependencies.include @@ -1,8 +1,8 @@ ## Required dependencies * PostgreSQL >=11.0 -* Elixir >=1.14.0 <1.17 -* Erlang OTP >=23.0.0 (supported: <27) +* Elixir >=1.14.0 <1.19 +* Erlang OTP >=23.0.0 (supported: <28) * git * file / libmagic * gcc or clang From 2b513fd450d0caab4ccfc7bdb8fa4c6a84764978 Mon Sep 17 00:00:00 2001 From: Phantasm Date: Sat, 24 May 2025 22:03:23 +0200 Subject: [PATCH 098/128] Elixir 1.18 add changelog --- changelog.d/elixir-1-18.fix | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/elixir-1-18.fix diff --git a/changelog.d/elixir-1-18.fix b/changelog.d/elixir-1-18.fix new file mode 100644 index 000000000..d4d5a3493 --- /dev/null +++ b/changelog.d/elixir-1-18.fix @@ -0,0 +1 @@ +Elixir 1.18: Fixed warnings and new deprecations From 286204913d6a1e65a8f75fc7277d2003827f9857 Mon Sep 17 00:00:00 2001 From: Phantasm Date: Sat, 24 May 2025 22:17:38 +0200 Subject: [PATCH 099/128] Replace Elixir 1.17 with 1.18 for build unit-testing pipelines --- .gitlab-ci.yml | 8 ++++---- ci/elixir-1.18.3-otp-27/Dockerfile | 8 ++++++++ ci/elixir-1.18.3-otp-27/build_and_push.sh | 1 + 3 files changed, 13 insertions(+), 4 deletions(-) create mode 100644 ci/elixir-1.18.3-otp-27/Dockerfile create mode 100755 ci/elixir-1.18.3-otp-27/build_and_push.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 675d0e067..29ee24a05 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -79,12 +79,12 @@ build-1.14.5-otp-25: script: - mix compile --force -build-1.17.1-otp-26: +build-1.18.3-otp-27: extends: - .build_changes_policy - .using-ci-base stage: build - image: git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.17.1-otp-26 + image: git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.18.3-otp-27 script: - mix compile --force @@ -142,12 +142,12 @@ unit-testing-1.14.5-otp-25: coverage_format: cobertura path: coverage.xml -unit-testing-1.17.1-otp-26: +unit-testing-1.18.3-otp-27: extends: - .build_changes_policy - .using-ci-base stage: test - image: git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.17.1-otp-26 + image: git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.18.3-otp-27 cache: *testing_cache_policy services: *testing_services script: *testing_script diff --git a/ci/elixir-1.18.3-otp-27/Dockerfile b/ci/elixir-1.18.3-otp-27/Dockerfile new file mode 100644 index 000000000..2b42aa90d --- /dev/null +++ b/ci/elixir-1.18.3-otp-27/Dockerfile @@ -0,0 +1,8 @@ +FROM elixir:1.18.3-otp-27 + +# Single RUN statement, otherwise intermediate images are created +# https://docs.docker.com/develop/develop-images/dockerfile_best-practices/#run +RUN apt-get update &&\ + apt-get install -y libmagic-dev cmake libimage-exiftool-perl ffmpeg &&\ + mix local.hex --force &&\ + mix local.rebar --force diff --git a/ci/elixir-1.18.3-otp-27/build_and_push.sh b/ci/elixir-1.18.3-otp-27/build_and_push.sh new file mode 100755 index 000000000..8a564fbf2 --- /dev/null +++ b/ci/elixir-1.18.3-otp-27/build_and_push.sh @@ -0,0 +1 @@ +docker buildx build --platform linux/amd64,linux/arm64 -t git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.18.3-otp-27 --push . From 374e8c85a789d401ceb42567f4c5030124f261f3 Mon Sep 17 00:00:00 2001 From: lain Date: Thu, 29 May 2025 08:17:31 +0000 Subject: [PATCH 100/128] Apply lambadalambda's suggestion(s) to 1 file(s) --- lib/pleroma/web/web_finger/web_finger_controller.ex | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/lib/pleroma/web/web_finger/web_finger_controller.ex b/lib/pleroma/web/web_finger/web_finger_controller.ex index 0a9ee2d3b..8a291e28e 100644 --- a/lib/pleroma/web/web_finger/web_finger_controller.ex +++ b/lib/pleroma/web/web_finger/web_finger_controller.ex @@ -42,15 +42,8 @@ defmodule Pleroma.Web.WebFinger.WebFingerController do end # Default to JSON when no format is specified or format is not recognized - def webfinger(%{assigns: %{format: _format}} = conn, %{"resource" => resource}) do - with {:ok, response} <- WebFinger.webfinger(resource, "JSON") do - json(conn, response) - else - _e -> - conn - |> put_status(404) - |> json("Couldn't find user") - end + def webfinger(%{assigns: %{format: _format}} = conn, %{"resource" => _resource} = params) do + webfinger(put_in(conn.assigns.format, "json"), params) end def webfinger(conn, _params), do: send_resp(conn, 400, "Bad Request") From 9386863019b17175d965c202be24568de2651ac3 Mon Sep 17 00:00:00 2001 From: Phantasm Date: Tue, 3 Jun 2025 23:08:51 +0200 Subject: [PATCH 101/128] openbsd: update install docs for 7.7 Explicitely installing OTP 26 is no longer needed. --- docs/installation/openbsd_en.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/installation/openbsd_en.md b/docs/installation/openbsd_en.md index 387b0f2ea..1de016cdd 100644 --- a/docs/installation/openbsd_en.md +++ b/docs/installation/openbsd_en.md @@ -2,7 +2,7 @@ {! backend/installation/otp_vs_from_source_source.include !} -This guide describes the installation and configuration of Pleroma (and the required software to run it) on a single OpenBSD 7.6 server. +This guide describes the installation and configuration of Pleroma (and the required software to run it) on a single OpenBSD 7.7 server. For any additional information regarding commands and configuration files mentioned here, check the man pages [online](https://man.openbsd.org/) or directly on your server with the man command. @@ -16,7 +16,7 @@ For any additional information regarding commands and configuration files mentio To install required packages, run the following command: ``` -# pkg_add erlang%26 elixir gmake git postgresql-server postgresql-contrib cmake libmagic libvips +# pkg_add elixir gmake git postgresql-server postgresql-contrib cmake libmagic libvips ``` Pleroma requires a reverse proxy, OpenBSD has relayd in base (and is used in this guide) and packages/ports are available for nginx (www/nginx) and apache (www/apache-httpd). From 9710063fdc92ca3df9005ef57f678fd78680a4f0 Mon Sep 17 00:00:00 2001 From: Phantasm Date: Sun, 1 Jun 2025 21:25:38 +0000 Subject: [PATCH 102/128] Apply suggestions to 2 files. --- lib/pleroma/application.ex | 12 ++---------- .../activity_pub/object_validators/tag_validator.ex | 7 ++++++- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index fd3c66c63..57ee7ce1f 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -93,16 +93,8 @@ defmodule Pleroma.Application do # Disable warnings_as_errors at runtime, it breaks Phoenix live reload # due to protocol consolidation warnings # :warnings_as_errors is deprecated via Code.compiler_options/2 since 1.18 - if elixir_version = System.version() do - [major, minor] = - elixir_version - |> String.split(".") - |> Enum.map(&String.to_integer/1) - |> Enum.take(2) - - if major == 1 and minor < 18 do - Code.compiler_options(warnings_as_errors: false) - end + if Version.compare(System.version(), "1.18.0") == :lt do + Code.compiler_options(warnings_as_errors: false) end # Define workers and child supervisors to be supervised diff --git a/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex b/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex index 411517045..5ce9ab36a 100644 --- a/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex @@ -50,7 +50,12 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.TagValidator do end def changeset(struct, %{"type" => "Hashtag", "name" => name} = data) do - name = String.downcase(name) + name = + case name do + "#" <> name -> name + name -> name + end + data = Map.put(data, "name", name) From 0e53cb494038b45d8281b9daba11a4a9dae2115b Mon Sep 17 00:00:00 2001 From: Phantasm Date: Mon, 2 Jun 2025 23:04:45 +0200 Subject: [PATCH 103/128] Remove unreachable checks for OTP < 22.2 OTP 22 is no longer supported at all. Pleroma's dependencies cannot be built with Elixir 1.13 and Elixir 1.14 cannot be built with OTP 22 since it depends on features not present in OTP 22. Hence why these checks cannot get triggered anymore. --- lib/pleroma/application.ex | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 57ee7ce1f..1df38b0bd 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -68,27 +68,6 @@ defmodule Pleroma.Application do Finch.start_link(name: MyFinch) end - if adapter == Tesla.Adapter.Gun do - if version = Pleroma.OTPVersion.version() do - [major, minor] = - version - |> String.split(".") - |> Enum.map(&String.to_integer/1) - |> Enum.take(2) - - if (major == 22 and minor < 2) or major < 22 do - raise " - !!!OTP VERSION WARNING!!! - You are using gun adapter with OTP version #{version}, which doesn't support correct handling of unordered certificates chains. Please update your Erlang/OTP to at least 22.2. - " - end - else - raise " - !!!OTP VERSION WARNING!!! - To support correct handling of unordered certificates chains - OTP version must be > 22.2. - " - end - end # Disable warnings_as_errors at runtime, it breaks Phoenix live reload # due to protocol consolidation warnings From 1be8deda73add2dde23127be1f4da802dcb25b45 Mon Sep 17 00:00:00 2001 From: Phantasm Date: Tue, 3 Jun 2025 23:17:39 +0200 Subject: [PATCH 104/128] Remove Pleroma.OTPVersion module Its last use was a check in lib/application.ex that was removed in commit 0e53cb494038b45d8281b9daba11a4a9dae2115b Major OTP version can be fetched with System.otp_release/0. If checking against minor versions and patch levels is needed, revert this commit since it uses the recommended way of getting a full OTP version string. --- lib/pleroma/otp_version.ex | 28 ----------------- test/fixtures/warnings/otp_version/21.1 | 1 - test/fixtures/warnings/otp_version/22.1 | 1 - test/fixtures/warnings/otp_version/22.4 | 1 - test/fixtures/warnings/otp_version/23.0 | 1 - test/pleroma/otp_version_test.exs | 42 ------------------------- 6 files changed, 74 deletions(-) delete mode 100644 lib/pleroma/otp_version.ex delete mode 100644 test/fixtures/warnings/otp_version/21.1 delete mode 100644 test/fixtures/warnings/otp_version/22.1 delete mode 100644 test/fixtures/warnings/otp_version/22.4 delete mode 100644 test/fixtures/warnings/otp_version/23.0 delete mode 100644 test/pleroma/otp_version_test.exs diff --git a/lib/pleroma/otp_version.ex b/lib/pleroma/otp_version.ex deleted file mode 100644 index 80b15275a..000000000 --- a/lib/pleroma/otp_version.ex +++ /dev/null @@ -1,28 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2022 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.OTPVersion do - @spec version() :: String.t() | nil - def version do - # OTP Version https://erlang.org/doc/system_principles/versions.html#otp-version - [ - Path.join(:code.root_dir(), "OTP_VERSION"), - Path.join([:code.root_dir(), "releases", :erlang.system_info(:otp_release), "OTP_VERSION"]) - ] - |> get_version_from_files() - end - - @spec get_version_from_files([Path.t()]) :: String.t() | nil - def get_version_from_files([]), do: nil - - def get_version_from_files([path | paths]) do - if File.exists?(path) do - path - |> File.read!() - |> String.replace(~r/\r|\n|\s/, "") - else - get_version_from_files(paths) - end - end -end diff --git a/test/fixtures/warnings/otp_version/21.1 b/test/fixtures/warnings/otp_version/21.1 deleted file mode 100644 index 90cd64c4f..000000000 --- a/test/fixtures/warnings/otp_version/21.1 +++ /dev/null @@ -1 +0,0 @@ -21.1 \ No newline at end of file diff --git a/test/fixtures/warnings/otp_version/22.1 b/test/fixtures/warnings/otp_version/22.1 deleted file mode 100644 index d9b314368..000000000 --- a/test/fixtures/warnings/otp_version/22.1 +++ /dev/null @@ -1 +0,0 @@ -22.1 \ No newline at end of file diff --git a/test/fixtures/warnings/otp_version/22.4 b/test/fixtures/warnings/otp_version/22.4 deleted file mode 100644 index 1da8ccd28..000000000 --- a/test/fixtures/warnings/otp_version/22.4 +++ /dev/null @@ -1 +0,0 @@ -22.4 \ No newline at end of file diff --git a/test/fixtures/warnings/otp_version/23.0 b/test/fixtures/warnings/otp_version/23.0 deleted file mode 100644 index 4266d8634..000000000 --- a/test/fixtures/warnings/otp_version/23.0 +++ /dev/null @@ -1 +0,0 @@ -23.0 \ No newline at end of file diff --git a/test/pleroma/otp_version_test.exs b/test/pleroma/otp_version_test.exs deleted file mode 100644 index 21701d5a8..000000000 --- a/test/pleroma/otp_version_test.exs +++ /dev/null @@ -1,42 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2022 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.OTPVersionTest do - use ExUnit.Case, async: true - - alias Pleroma.OTPVersion - - describe "check/1" do - test "22.4" do - assert OTPVersion.get_version_from_files(["test/fixtures/warnings/otp_version/22.4"]) == - "22.4" - end - - test "22.1" do - assert OTPVersion.get_version_from_files(["test/fixtures/warnings/otp_version/22.1"]) == - "22.1" - end - - test "21.1" do - assert OTPVersion.get_version_from_files(["test/fixtures/warnings/otp_version/21.1"]) == - "21.1" - end - - test "23.0" do - assert OTPVersion.get_version_from_files(["test/fixtures/warnings/otp_version/23.0"]) == - "23.0" - end - - test "with nonexistent file" do - assert OTPVersion.get_version_from_files([ - "test/fixtures/warnings/otp_version/non-exising", - "test/fixtures/warnings/otp_version/22.4" - ]) == "22.4" - end - - test "empty paths" do - assert OTPVersion.get_version_from_files([]) == nil - end - end -end From 6fa4f08e67a2ebebca2337259e1a5b5b6862b5ef Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Wed, 4 Jun 2025 11:43:18 +0300 Subject: [PATCH 105/128] Add back String.downcase that was accidentally removed from tag_validator --- lib/pleroma/web/activity_pub/object_validators/tag_validator.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex b/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex index 5ce9ab36a..91aeb9dd7 100644 --- a/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex @@ -55,7 +55,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.TagValidator do "#" <> name -> name name -> name end - + |> String.downcase() data = Map.put(data, "name", name) From d95e1066b9858997c9137097bf00ddc2fa57e5e1 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Wed, 4 Jun 2025 12:03:54 +0300 Subject: [PATCH 106/128] Fix formatting --- lib/mix/tasks/pleroma/test_runner.ex | 4 +++- lib/pleroma/application.ex | 1 - test/pleroma/user_test.exs | 13 +++++++++++-- test/pleroma/web/plugs/user_tracking_plug_test.exs | 6 +++++- 4 files changed, 19 insertions(+), 5 deletions(-) diff --git a/lib/mix/tasks/pleroma/test_runner.ex b/lib/mix/tasks/pleroma/test_runner.ex index d9cf0d445..67820247e 100644 --- a/lib/mix/tasks/pleroma/test_runner.ex +++ b/lib/mix/tasks/pleroma/test_runner.ex @@ -4,7 +4,9 @@ defmodule Mix.Tasks.Pleroma.TestRunner do use Mix.Task def run(args \\ []) do - case System.cmd("mix", ["test", "--warnings-as-errors"] ++ args, into: IO.stream(:stdio, :line)) do + case System.cmd("mix", ["test", "--warnings-as-errors"] ++ args, + into: IO.stream(:stdio, :line) + ) do {_, 0} -> :ok diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 1df38b0bd..8e1c5de0d 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -68,7 +68,6 @@ defmodule Pleroma.Application do Finch.start_link(name: MyFinch) end - # Disable warnings_as_errors at runtime, it breaks Phoenix live reload # due to protocol consolidation warnings # :warnings_as_errors is deprecated via Code.compiler_options/2 since 1.18 diff --git a/test/pleroma/user_test.exs b/test/pleroma/user_test.exs index 79a480f85..44e2d0d65 100644 --- a/test/pleroma/user_test.exs +++ b/test/pleroma/user_test.exs @@ -2670,7 +2670,11 @@ defmodule Pleroma.UserTest do assert {:ok, user} = User.update_last_active_at(user) assert NaiveDateTime.compare(user.last_active_at, test_started_at) in [:gt, :eq] - assert NaiveDateTime.compare(user.last_active_at, NaiveDateTime.truncate(NaiveDateTime.utc_now(), :second)) in [:lt, :eq] + + assert NaiveDateTime.compare( + user.last_active_at, + NaiveDateTime.truncate(NaiveDateTime.utc_now(), :second) + ) in [:lt, :eq] last_active_at = NaiveDateTime.utc_now() @@ -2681,11 +2685,16 @@ defmodule Pleroma.UserTest do user |> cast(%{last_active_at: last_active_at}, [:last_active_at]) |> User.update_and_set_cache() + assert NaiveDateTime.compare(user.last_active_at, last_active_at) == :eq assert {:ok, user} = User.update_last_active_at(user) assert NaiveDateTime.compare(user.last_active_at, test_started_at) in [:gt, :eq] - assert NaiveDateTime.compare(user.last_active_at, NaiveDateTime.truncate(NaiveDateTime.utc_now(), :second)) in [:lt, :eq] + + assert NaiveDateTime.compare( + user.last_active_at, + NaiveDateTime.truncate(NaiveDateTime.utc_now(), :second) + ) in [:lt, :eq] end test "active_user_count/1" do diff --git a/test/pleroma/web/plugs/user_tracking_plug_test.exs b/test/pleroma/web/plugs/user_tracking_plug_test.exs index 5c67a7735..cd9c66448 100644 --- a/test/pleroma/web/plugs/user_tracking_plug_test.exs +++ b/test/pleroma/web/plugs/user_tracking_plug_test.exs @@ -22,7 +22,11 @@ defmodule Pleroma.Web.Plugs.UserTrackingPlugTest do |> UserTrackingPlug.call(%{}) assert NaiveDateTime.compare(user.last_active_at, test_started_at) in [:gt, :eq] - assert NaiveDateTime.compare(user.last_active_at, NaiveDateTime.truncate(NaiveDateTime.utc_now(), :second)) in [:lt, :eq] + + assert NaiveDateTime.compare( + user.last_active_at, + NaiveDateTime.truncate(NaiveDateTime.utc_now(), :second) + ) in [:lt, :eq] end test "doesn't update last_active_at if it was updated recently", %{conn: conn} do From 7ddae61414a2e0f04560d2afe46dc51c5ac32c85 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Wed, 4 Jun 2025 12:25:06 +0300 Subject: [PATCH 107/128] Change the test that assumes that a hashtag with # will remain as-is This does not seem to be the intended behaviour, as the code that produces it did not actually ever do anything and just returned the tag as-is. See lib/pleroma/web/activity_pub/object_validators/tag_validator.ex and https://git.pleroma.social/pleroma/pleroma/-/merge_requests/4358#note_112681 At least Mastodon and Misskey output tags without the # from their API, so in reality tags with the hash should rarely happen. --- .../web/activity_pub/transmogrifier/note_handling_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs b/test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs index fd7a3c772..648326929 100644 --- a/test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs +++ b/test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs @@ -200,7 +200,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.NoteHandlingTest do assert match?( %{ "href" => "http://mastodon.example.org/tags/moo", - "name" => "#moo", + "name" => "moo", "type" => "Hashtag" }, Enum.at(object.data["tag"], 1) From dc26f749617dda6ed2b538f515056567519d9246 Mon Sep 17 00:00:00 2001 From: Phantasm Date: Wed, 4 Jun 2025 18:32:25 +0200 Subject: [PATCH 108/128] Revert to previous tag_validator behavior This paritally reverts commit 9710063fdc92ca3df9005ef57f678fd78680a4f0 and reverts commit 7ddae61414a2e0f04560d2afe46dc51c5ac32c85 See thread: https://git.pleroma.social/pleroma/pleroma/-/merge_requests/4358#note_112761 --- .../web/activity_pub/object_validators/tag_validator.ex | 8 +------- .../activity_pub/transmogrifier/note_handling_test.exs | 2 +- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex b/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex index 91aeb9dd7..dc2770189 100644 --- a/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex @@ -50,13 +50,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.TagValidator do end def changeset(struct, %{"type" => "Hashtag", "name" => name} = data) do - name = - case name do - "#" <> name -> name - name -> name - end - |> String.downcase() - + name = String.downcase(name) data = Map.put(data, "name", name) struct diff --git a/test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs b/test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs index 648326929..fd7a3c772 100644 --- a/test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs +++ b/test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs @@ -200,7 +200,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.NoteHandlingTest do assert match?( %{ "href" => "http://mastodon.example.org/tags/moo", - "name" => "moo", + "name" => "#moo", "type" => "Hashtag" }, Enum.at(object.data["tag"], 1) From ff69b00eaef8354ca7224aa6af5eb158ca502125 Mon Sep 17 00:00:00 2001 From: Phantasm Date: Wed, 4 Jun 2025 19:18:01 +0200 Subject: [PATCH 109/128] Elixir 1.18 Update credo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit warning: Credo.CLI.Command.Info.Output.Default.print_after_info/4 is undefined or private. Did you mean: * print/2 │ 4 │ use Credo.CLI.Output.FormatDelegator, │ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ │ └─ lib/credo/cli/command/info/info_output.ex:4: Credo.CLI.Command.Info.InfoOutput.print_after_info/4 --- mix.exs | 2 +- mix.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mix.exs b/mix.exs index dc6c2492f..e34ee0cbc 100644 --- a/mix.exs +++ b/mix.exs @@ -213,7 +213,7 @@ defmodule Pleroma.Mixfile do {:poison, "~> 3.0", only: :test}, {:ex_doc, "~> 0.22", only: :dev, runtime: false}, {:ex_machina, "~> 2.4", only: :test}, - {:credo, "~> 1.6", only: [:dev, :test], runtime: false}, + {:credo, "~> 1.7", only: [:dev, :test], runtime: false}, {:mock, "~> 0.3.5", only: :test}, {:covertool, "~> 2.0", only: :test}, {:hackney, "~> 1.18.0", override: true}, diff --git a/mix.lock b/mix.lock index 9b53ede62..f7f37b7e1 100644 --- a/mix.lock +++ b/mix.lock @@ -23,7 +23,7 @@ "cowboy": {:hex, :cowboy, "2.12.0", "f276d521a1ff88b2b9b4c54d0e753da6c66dd7be6c9fca3d9418b561828a3731", [:make, :rebar3], [{:cowlib, "2.13.0", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "1.8.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "8a7abe6d183372ceb21caa2709bec928ab2b72e18a3911aa1771639bef82651e"}, "cowboy_telemetry": {:hex, :cowboy_telemetry, "0.4.0", "f239f68b588efa7707abce16a84d0d2acf3a0f50571f8bb7f56a15865aae820c", [:rebar3], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "7d98bac1ee4565d31b62d59f8823dfd8356a169e7fcbb83831b8a5397404c9de"}, "cowlib": {:hex, :cowlib, "2.13.0", "db8f7505d8332d98ef50a3ef34b34c1afddec7506e4ee4dd4a3a266285d282ca", [:make, :rebar3], [], "hexpm", "e1e1284dc3fc030a64b1ad0d8382ae7e99da46c3246b815318a4b848873800a4"}, - "credo": {:hex, :credo, "1.7.7", "771445037228f763f9b2afd612b6aa2fd8e28432a95dbbc60d8e03ce71ba4446", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "8bc87496c9aaacdc3f90f01b7b0582467b69b4bd2441fe8aae3109d843cc2f2e"}, + "credo": {:hex, :credo, "1.7.12", "9e3c20463de4b5f3f23721527fcaf16722ec815e70ff6c60b86412c695d426c1", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "8493d45c656c5427d9c729235b99d498bd133421f3e0a683e5c1b561471291e5"}, "crontab": {:hex, :crontab, "1.1.8", "2ce0e74777dfcadb28a1debbea707e58b879e6aa0ffbf9c9bb540887bce43617", [:mix], [{:ecto, "~> 1.0 or ~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm"}, "custom_base": {:hex, :custom_base, "0.2.1", "4a832a42ea0552299d81652aa0b1f775d462175293e99dfbe4d7dbaab785a706", [:mix], [], "hexpm", "8df019facc5ec9603e94f7270f1ac73ddf339f56ade76a721eaa57c1493ba463"}, "db_connection": {:hex, :db_connection, "2.7.0", "b99faa9291bb09892c7da373bb82cba59aefa9b36300f6145c5f201c7adf48ec", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "dcf08f31b2701f857dfc787fbad78223d61a32204f217f15e881dd93e4bdd3ff"}, From ae2c97fad8121a26146643c7e4a361d8f4d289a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?nicole=20miko=C5=82ajczyk?= Date: Wed, 4 Jun 2025 21:32:30 +0200 Subject: [PATCH 110/128] Use JSON for DeepL API requests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: nicole mikołajczyk --- changelog.d/deepl-json.fix | 1 + lib/pleroma/language/translation/deepl.ex | 18 ++++++++---------- 2 files changed, 9 insertions(+), 10 deletions(-) create mode 100644 changelog.d/deepl-json.fix diff --git a/changelog.d/deepl-json.fix b/changelog.d/deepl-json.fix new file mode 100644 index 000000000..ee6f8664e --- /dev/null +++ b/changelog.d/deepl-json.fix @@ -0,0 +1 @@ +Use JSON for DeepL API requests diff --git a/lib/pleroma/language/translation/deepl.ex b/lib/pleroma/language/translation/deepl.ex index e027035b4..aaaac9b0f 100644 --- a/lib/pleroma/language/translation/deepl.ex +++ b/lib/pleroma/language/translation/deepl.ex @@ -24,17 +24,15 @@ defmodule Pleroma.Language.Translation.Deepl do |> URI.to_string() case Pleroma.HTTP.post( - endpoint <> - "?" <> - URI.encode_query(%{ - text: content, - source_lang: source_language |> String.upcase(), - target_lang: target_language, - tag_handling: "html" - }), - "", + endpoint, + Jason.encode!(%{ + text: [content], + source_lang: source_language |> String.upcase(), + target_lang: target_language, + tag_handling: "html" + }), [ - {"Content-Type", "application/x-www-form-urlencoded"}, + {"Content-Type", "application/json"}, {"Authorization", "DeepL-Auth-Key #{api_key()}"} ] ) do From a817f1800ed335ed5ef2353adce3235bfb0e44c3 Mon Sep 17 00:00:00 2001 From: Phantasm Date: Thu, 5 Jun 2025 16:40:52 +0200 Subject: [PATCH 111/128] Remove forgotten Pleroma.OTPVersion usage in mix.exs This was used in OTP releases where the normal OTP_VERSION file is unavailable. If checking against OTP minor versions and patch levels is needed again, revert this commit and commit mentioned below. Context: 1be8deda73add2dde23127be1f4da802dcb25b45 --- changelog.d/remove-forgotten-OTPVersion-usage.skip | 0 mix.exs | 11 +---------- 2 files changed, 1 insertion(+), 10 deletions(-) create mode 100644 changelog.d/remove-forgotten-OTPVersion-usage.skip diff --git a/changelog.d/remove-forgotten-OTPVersion-usage.skip b/changelog.d/remove-forgotten-OTPVersion-usage.skip new file mode 100644 index 000000000..e69de29bb diff --git a/mix.exs b/mix.exs index e34ee0cbc..971084f94 100644 --- a/mix.exs +++ b/mix.exs @@ -37,22 +37,13 @@ defmodule Pleroma.Mixfile do pleroma: [ include_executables_for: [:unix], applications: [ex_syslogger: :load, syslog: :load, eldap: :transient], - steps: [:assemble, &put_otp_version/1, ©_files/1, ©_nginx_config/1], + steps: [:assemble, ©_files/1, ©_nginx_config/1], config_providers: [{Pleroma.Config.ReleaseRuntimeProvider, []}] ] ] ] end - def put_otp_version(%{path: target_path} = release) do - File.write!( - Path.join([target_path, "OTP_VERSION"]), - Pleroma.OTPVersion.version() - ) - - release - end - def copy_files(%{path: target_path} = release) do File.cp_r!("./rel/files", target_path) release From 8ae4ed0807151f3a1c364c9e7da608cda2387178 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Thu, 5 Jun 2025 22:12:06 +0300 Subject: [PATCH 112/128] Make the opts in ActivityPub.Builder.block optional --- lib/pleroma/web/activity_pub/builder.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/activity_pub/builder.ex b/lib/pleroma/web/activity_pub/builder.ex index ecb6df1f0..046316024 100644 --- a/lib/pleroma/web/activity_pub/builder.ex +++ b/lib/pleroma/web/activity_pub/builder.ex @@ -328,7 +328,7 @@ defmodule Pleroma.Web.ActivityPub.Builder do end @spec block(User.t(), User.t(), map()) :: {:ok, map(), keyword()} - def block(blocker, blocked, params) do + def block(blocker, blocked, params \\ %{}) do {:ok, %{ "id" => Utils.generate_activity_id(), From a2ad2d8d23196801de228fcec9121d6bed03fa25 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 5 Jun 2025 16:54:05 -0700 Subject: [PATCH 113/128] Remove unused import --- test/pleroma/web/web_finger/web_finger_controller_test.exs | 1 - 1 file changed, 1 deletion(-) diff --git a/test/pleroma/web/web_finger/web_finger_controller_test.exs b/test/pleroma/web/web_finger/web_finger_controller_test.exs index d60e8a585..be44e3a8b 100644 --- a/test/pleroma/web/web_finger/web_finger_controller_test.exs +++ b/test/pleroma/web/web_finger/web_finger_controller_test.exs @@ -5,7 +5,6 @@ defmodule Pleroma.Web.WebFinger.WebFingerControllerTest do use Pleroma.Web.ConnCase - import ExUnit.CaptureLog import Pleroma.Factory import Tesla.Mock From 48316d168c644eeb622e03daf751983fcb5bbcdd Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 5 Jun 2025 16:55:07 -0700 Subject: [PATCH 114/128] Fix failing tests due to Builder.block/2 becoming Builder.block/3 with no default value --- lib/pleroma/web/activity_pub/builder.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/activity_pub/builder.ex b/lib/pleroma/web/activity_pub/builder.ex index ecb6df1f0..046316024 100644 --- a/lib/pleroma/web/activity_pub/builder.ex +++ b/lib/pleroma/web/activity_pub/builder.ex @@ -328,7 +328,7 @@ defmodule Pleroma.Web.ActivityPub.Builder do end @spec block(User.t(), User.t(), map()) :: {:ok, map(), keyword()} - def block(blocker, blocked, params) do + def block(blocker, blocked, params \\ %{}) do {:ok, %{ "id" => Utils.generate_activity_id(), From db65b35ca38f682b286786b23ed94ba1821dca65 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 5 Jun 2025 17:11:16 -0700 Subject: [PATCH 115/128] Fix test Returns JSON when format is not supported (Pleroma.Web.WebFinger.WebFingerControllerTest) If we want to return JSON when a badly behaving client requests text/html, we still have to accept it at the Plug --- lib/pleroma/web/router.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index f2f9d7246..dfab1b216 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -189,7 +189,7 @@ defmodule Pleroma.Web.Router do end pipeline :well_known do - plug(:accepts, ["activity+json", "json", "jrd", "jrd+json", "xml", "xrd+xml"]) + plug(:accepts, ["activity+json", "json", "jrd", "jrd+json", "xml", "xrd+xml", "html"]) end pipeline :config do From 922696376317266f6f8a3259b0a7ba91443c0663 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 5 Jun 2025 17:13:55 -0700 Subject: [PATCH 116/128] Fix test fallout from most recent merges --- changelog.d/fixtests.skip | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 changelog.d/fixtests.skip diff --git a/changelog.d/fixtests.skip b/changelog.d/fixtests.skip new file mode 100644 index 000000000..e69de29bb From a361b84fc97041f69c890f09de9227f51ac905f4 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Wed, 11 Jun 2025 23:02:42 +0300 Subject: [PATCH 117/128] Relax alsoKnownAs requirements to just being a URI --- changelog.d/relax-also-known-as.change | 1 + lib/pleroma/user.ex | 2 +- .../web/mastodon_api/controllers/search_controller.ex | 2 +- test/pleroma/user_test.exs | 9 +++++++++ 4 files changed, 12 insertions(+), 2 deletions(-) create mode 100644 changelog.d/relax-also-known-as.change diff --git a/changelog.d/relax-also-known-as.change b/changelog.d/relax-also-known-as.change new file mode 100644 index 000000000..800c3e72a --- /dev/null +++ b/changelog.d/relax-also-known-as.change @@ -0,0 +1 @@ +Relax alsoKnownAs requirements to just URI, not necessarily HTTP(S) \ No newline at end of file diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 427f7878d..84551afd5 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -150,7 +150,7 @@ defmodule Pleroma.User do field(:allow_following_move, :boolean, default: true) field(:skip_thread_containment, :boolean, default: false) field(:actor_type, :string, default: "Person") - field(:also_known_as, {:array, ObjectValidators.ObjectID}, default: []) + field(:also_known_as, {:array, ObjectValidators.BareUri}, default: []) field(:inbox, :string) field(:shared_inbox, :string) field(:accepts_chat_messages, :boolean, default: nil) diff --git a/lib/pleroma/web/mastodon_api/controllers/search_controller.ex b/lib/pleroma/web/mastodon_api/controllers/search_controller.ex index 628aa311b..d9a1ba41e 100644 --- a/lib/pleroma/web/mastodon_api/controllers/search_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/search_controller.ex @@ -190,7 +190,7 @@ defmodule Pleroma.Web.MastodonAPI.SearchController do f.() rescue error -> - Logger.error("#{__MODULE__} search error: #{inspect(error)}") + Logger.error(Exception.format(:error, error, __STACKTRACE__)) fallback end end diff --git a/test/pleroma/user_test.exs b/test/pleroma/user_test.exs index 44e2d0d65..0b4dc9197 100644 --- a/test/pleroma/user_test.exs +++ b/test/pleroma/user_test.exs @@ -2792,6 +2792,15 @@ defmodule Pleroma.UserTest do assert user_updated.also_known_as |> length() == 1 assert user2.ap_id in user_updated.also_known_as end + + test "should tolerate non-http(s) aliases" do + user = + insert(:user, %{ + also_known_as: ["at://did:plc:xgvzy7ni6ig6ievcbls5jaxe"] + }) + + assert "at://did:plc:xgvzy7ni6ig6ievcbls5jaxe" in user.also_known_as + end end describe "alias_users/1" do From 27ec46814cfb5515b16727d36bb028b5634b6b5d Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 12 Jun 2025 21:27:19 -0700 Subject: [PATCH 118/128] Revert "Public getting stripped from unlisted activity CC: Add possible tests" This reverts commit ded40182b0aa6848b55febe73ec7e41eace1e0f6. --- test/fixtures/poast_unlisted.json | 65 ------------------- .../transmogrifier/note_handling_test.exs | 31 --------- .../web/activity_pub/transmogrifier_test.exs | 37 ----------- 3 files changed, 133 deletions(-) delete mode 100644 test/fixtures/poast_unlisted.json diff --git a/test/fixtures/poast_unlisted.json b/test/fixtures/poast_unlisted.json deleted file mode 100644 index fa23153ba..000000000 --- a/test/fixtures/poast_unlisted.json +++ /dev/null @@ -1,65 +0,0 @@ -{ - "@context" : [ - "https://www.w3.org/ns/activitystreams", - "https://poa.st/schemas/litepub-0.1.jsonld", - { - "@language" : "und" - } - ], - "actor" : "https://poa.st/users/TrevorGoodchild", - "attachment" : [], - "attributedTo" : "https://poa.st/users/TrevorGoodchild", - "cc" : [ - "https://www.w3.org/ns/activitystreams#Public" - ], - "context" : "https://poa.st/contexts/c6d125f1-4e7f-43bd-aa31-33de4d90d049", - "conversation" : "https://poa.st/contexts/c6d125f1-4e7f-43bd-aa31-33de4d90d049", - "directMessage" : false, - "id" : "https://poa.st/activities/bbd3347a-4a89-4cdb-bf86-4f9eed9506e3", - "object" : { - "actor" : "https://poa.st/users/TrevorGoodchild", - "attachment" : [], - "attributedTo" : "https://poa.st/users/TrevorGoodchild", - "cc" : [ - "https://www.w3.org/ns/activitystreams#Public" - ], - "content" : "@HoroTheWhiteWolf >please let this be his zero fucks given final statement before he joins the 52%+ tranny club", - "context" : "https://poa.st/contexts/c6d125f1-4e7f-43bd-aa31-33de4d90d049", - "conversation" : "https://poa.st/contexts/c6d125f1-4e7f-43bd-aa31-33de4d90d049", - "id" : "https://poa.st/objects/7eb785d5-a556-4070-9091-f4afb226466c", - "inReplyTo" : "https://poa.st/objects/71995b41-cfb2-48ce-abce-76d570d54edc", - "published" : "2025-05-03T23:54:07.489885Z", - "repliesCount" : 2, - "sensitive" : false, - "source" : { - "content" : ">please let this be his zero fucks given final statement before he joins the 52%+ tranny club", - "mediaType" : "text/plain" - }, - "summary" : "", - "tag" : [ - { - "href" : "https://poa.st/users/HoroTheWhiteWolf", - "name" : "@HoroTheWhiteWolf", - "type" : "Mention" - } - ], - "to" : [ - "https://poa.st/users/HoroTheWhiteWolf", - "https://poa.st/users/TrevorGoodchild/followers" - ], - "type" : "Note" - }, - "published" : "2025-05-03T23:54:07.489837Z", - "tag" : [ - { - "href" : "https://poa.st/users/HoroTheWhiteWolf", - "name" : "@HoroTheWhiteWolf", - "type" : "Mention" - } - ], - "to" : [ - "https://poa.st/users/HoroTheWhiteWolf", - "https://poa.st/users/TrevorGoodchild/followers" - ], - "type" : "Create" -} diff --git a/test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs b/test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs index 13982940a..fd7a3c772 100644 --- a/test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs +++ b/test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs @@ -786,35 +786,4 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.NoteHandlingTest do assert object.data["context"] == object.data["inReplyTo"] assert modified.data["context"] == object.data["inReplyTo"] end - - test "it keeps the public address in cc in the activity when it is present" do - data = - File.read!("test/fixtures/mastodon-post-activity.json") - |> Jason.decode!() - - object = - data["object"] - |> Map.put("cc", ["https://www.w3.org/ns/activitystreams#Public"]) - |> Map.put("to", []) - - data = - data - |> Map.put("object", object) - |> Map.put("cc", ["https://www.w3.org/ns/activitystreams#Public"]) - |> Map.put("to", []) - - {:ok, %Activity{} = modified} = Transmogrifier.handle_incoming(data) - assert modified.data["cc"] == ["https://www.w3.org/ns/activitystreams#Public"] - end - - test "it tries it with the real poast_unlisted.json, ensuring that public is in the cc" do - data = - File.read!("test/fixtures/poast_unlisted.json") - |> Jason.decode!() - - _user = insert(:user, ap_id: data["actor"]) - - {:ok, %Activity{} = modified} = Transmogrifier.handle_incoming(data) - assert modified.data["cc"] == ["https://www.w3.org/ns/activitystreams#Public"] - end end diff --git a/test/pleroma/web/activity_pub/transmogrifier_test.exs b/test/pleroma/web/activity_pub/transmogrifier_test.exs index ef6e004f1..e0395d7bb 100644 --- a/test/pleroma/web/activity_pub/transmogrifier_test.exs +++ b/test/pleroma/web/activity_pub/transmogrifier_test.exs @@ -757,43 +757,6 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do refute recipient.follower_address in fixed_object["cc"] refute recipient.follower_address in fixed_object["to"] end - - test "preserves public URL in cc even when not explicitly mentioned", %{user: user} do - public_url = "https://www.w3.org/ns/activitystreams#Public" - - # Case 1: Public URL in cc but no mentions - object = %{ - "actor" => user.ap_id, - "to" => ["https://social.beepboop.ga/users/dirb"], - "cc" => [public_url], - "tag" => [] - } - - fixed_object = Transmogrifier.fix_explicit_addressing(object, user.follower_address) - assert public_url in fixed_object["cc"] - - # Case 2: Public URL in cc, with mentions but public not in to - object = %{ - "actor" => user.ap_id, - "to" => ["https://pleroma.gold/users/user1"], - "cc" => [public_url], - "tag" => [%{"type" => "Mention", "href" => "https://pleroma.gold/users/user1"}] - } - - fixed_object = Transmogrifier.fix_explicit_addressing(object, user.follower_address) - assert public_url in fixed_object["cc"] - - # Case 3: Public URL in to, it should be moved to to - object = %{ - "actor" => user.ap_id, - "to" => [public_url], - "cc" => [], - "tag" => [] - } - - fixed_object = Transmogrifier.fix_explicit_addressing(object, user.follower_address) - assert public_url in fixed_object["to"] - end end describe "fix_summary/1" do From 9f79df75082cfc563ce7816a1839800aa22ec350 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 12 Jun 2025 21:28:58 -0700 Subject: [PATCH 119/128] Add test demonstrating public getting stripped from unlisted activity CC --- .../web/activity_pub/publisher_test.exs | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/test/pleroma/web/activity_pub/publisher_test.exs b/test/pleroma/web/activity_pub/publisher_test.exs index 99ed42877..ec3201b96 100644 --- a/test/pleroma/web/activity_pub/publisher_test.exs +++ b/test/pleroma/web/activity_pub/publisher_test.exs @@ -520,4 +520,73 @@ defmodule Pleroma.Web.ActivityPub.PublisherTest do assert decoded["cc"] == [] end + + test "retains public address in cc for unlisted activities" do + user = insert(:user) + + activity = + insert(:note_activity, + user: user, + data_attrs: %{ + "cc" => [@as_public], + "to" => [user.follower_address] + } + ) + + assert @as_public in activity.data["cc"] + + # Call prepare_one without an explicit cc parameter (default in production) + prepared = + Publisher.prepare_one(%{ + inbox: "https://remote.instance/users/someone/inbox", + activity_id: activity.id + }) + + # Parse the JSON to verify the cc field in the federated message + {:ok, decoded} = Jason.decode(prepared.json) + + # The public address should be preserved in the cc field + # Currently this will fail because it's being removed + assert @as_public in decoded["cc"] + + # For verification, also test with an explicit cc parameter + # to show the cc field is completely replaced + prepared_with_cc = + Publisher.prepare_one(%{ + inbox: "https://remote.instance/users/someone/inbox", + activity_id: activity.id, + cc: ["https://example.com/specific/user"] + }) + + {:ok, decoded_with_cc} = Jason.decode(prepared_with_cc.json) + + # Verify cc is completely replaced with the provided value + assert decoded_with_cc["cc"] == ["https://example.com/specific/user"] + end + + test "public address in cc parameter is preserved" do + user = insert(:user) + + activity = + insert(:note_activity, + user: user, + data_attrs: %{ + "cc" => [@as_public, "https://example.org/users/other"], + "to" => [user.follower_address] + } + ) + + assert @as_public in activity.data["cc"] + + prepared_with_public_cc = + Publisher.prepare_one(%{ + inbox: "https://remote.instance/users/someone/inbox", + activity_id: activity.id, + cc: [@as_public] + }) + + {:ok, decoded_with_public_cc} = Jason.decode(prepared_with_public_cc.json) + + assert @as_public in decoded_with_public_cc["cc"] + end end From 23be24b92fa4f868b814b2c6927f2a6a69fa882d Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 12 Jun 2025 21:37:50 -0700 Subject: [PATCH 120/128] Fix federation issue where Public visibility information in cc field was lost when sent to remote servers, causing posts to appear with inconsistent visibility across instances --- changelog.d/preserve-public-cc.fix | 1 + lib/pleroma/web/activity_pub/publisher.ex | 15 ++++++++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 changelog.d/preserve-public-cc.fix diff --git a/changelog.d/preserve-public-cc.fix b/changelog.d/preserve-public-cc.fix new file mode 100644 index 000000000..1b20ce9ad --- /dev/null +++ b/changelog.d/preserve-public-cc.fix @@ -0,0 +1 @@ +Fix federation issue where Public visibility information in cc field was lost when sent to remote servers, causing posts to appear with inconsistent visibility across instances diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex index 0de3a0d43..762c991fd 100644 --- a/lib/pleroma/web/activity_pub/publisher.ex +++ b/lib/pleroma/web/activity_pub/publisher.ex @@ -93,7 +93,20 @@ defmodule Pleroma.Web.ActivityPub.Publisher do {:ok, data} = Transmogrifier.prepare_outgoing(activity.data) - cc = Map.get(params, :cc, []) + param_cc = Map.get(params, :cc, []) + + original_cc = Map.get(data, "cc", []) + + public_address = Pleroma.Constants.as_public() + + # Avoid overriding explicitly set cc values for specific recipients. + # e.g., this ensures unlisted posts are visible to users on other servers. + cc = + if public_address in original_cc and param_cc == [] do + [public_address] + else + param_cc + end json = data From d3adc3e05e09fdcb663ec1a3e20c1bc2d04a6ab5 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 12 Jun 2025 21:59:26 -0700 Subject: [PATCH 121/128] Split this cc test into two individual cases --- .../web/activity_pub/publisher_test.exs | 58 +++++++++++-------- 1 file changed, 34 insertions(+), 24 deletions(-) diff --git a/test/pleroma/web/activity_pub/publisher_test.exs b/test/pleroma/web/activity_pub/publisher_test.exs index ec3201b96..7bc571595 100644 --- a/test/pleroma/web/activity_pub/publisher_test.exs +++ b/test/pleroma/web/activity_pub/publisher_test.exs @@ -521,9 +521,11 @@ defmodule Pleroma.Web.ActivityPub.PublisherTest do assert decoded["cc"] == [] end - test "retains public address in cc for unlisted activities" do + test "unlisted activities retain public address in cc" do user = insert(:user) + # simulate unlistd activity by only having + # public address in cc activity = insert(:note_activity, user: user, @@ -535,58 +537,66 @@ defmodule Pleroma.Web.ActivityPub.PublisherTest do assert @as_public in activity.data["cc"] - # Call prepare_one without an explicit cc parameter (default in production) prepared = Publisher.prepare_one(%{ inbox: "https://remote.instance/users/someone/inbox", activity_id: activity.id }) - # Parse the JSON to verify the cc field in the federated message {:ok, decoded} = Jason.decode(prepared.json) - # The public address should be preserved in the cc field - # Currently this will fail because it's being removed assert @as_public in decoded["cc"] - - # For verification, also test with an explicit cc parameter - # to show the cc field is completely replaced - prepared_with_cc = - Publisher.prepare_one(%{ - inbox: "https://remote.instance/users/someone/inbox", - activity_id: activity.id, - cc: ["https://example.com/specific/user"] - }) - - {:ok, decoded_with_cc} = Jason.decode(prepared_with_cc.json) - - # Verify cc is completely replaced with the provided value - assert decoded_with_cc["cc"] == ["https://example.com/specific/user"] end test "public address in cc parameter is preserved" do user = insert(:user) + cc_with_public = [@as_public, "https://example.org/users/other"] + activity = insert(:note_activity, user: user, data_attrs: %{ - "cc" => [@as_public, "https://example.org/users/other"], + "cc" => cc_with_public, "to" => [user.follower_address] } ) assert @as_public in activity.data["cc"] - prepared_with_public_cc = + prepared = Publisher.prepare_one(%{ inbox: "https://remote.instance/users/someone/inbox", activity_id: activity.id, - cc: [@as_public] + cc: cc_with_public }) - {:ok, decoded_with_public_cc} = Jason.decode(prepared_with_public_cc.json) + {:ok, decoded} = Jason.decode(prepared.json) - assert @as_public in decoded_with_public_cc["cc"] + assert cc_with_public == decoded["cc"] + end + + test "cc parameter is preserved" do + user = insert(:user) + + activity = + insert(:note_activity, + user: user, + data_attrs: %{ + "cc" => ["https://example.com/specific/user"], + "to" => [user.follower_address] + } + ) + + prepared = + Publisher.prepare_one(%{ + inbox: "https://remote.instance/users/someone/inbox", + activity_id: activity.id, + cc: ["https://example.com/specific/user"] + }) + + {:ok, decoded} = Jason.decode(prepared.json) + + assert decoded["cc"] == ["https://example.com/specific/user"] end end From fe6d2ecc970008f99f9d948b86e5da07e80c2a29 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 12 Jun 2025 22:33:57 -0700 Subject: [PATCH 122/128] Test for unlisted but Publisher param_cc is not empty --- .../web/activity_pub/publisher_test.exs | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/test/pleroma/web/activity_pub/publisher_test.exs b/test/pleroma/web/activity_pub/publisher_test.exs index 7bc571595..b7ff0ed5f 100644 --- a/test/pleroma/web/activity_pub/publisher_test.exs +++ b/test/pleroma/web/activity_pub/publisher_test.exs @@ -546,6 +546,28 @@ defmodule Pleroma.Web.ActivityPub.PublisherTest do {:ok, decoded} = Jason.decode(prepared.json) assert @as_public in decoded["cc"] + + # maybe we also have another inbox in cc + # during Publishing + activity = + insert(:note_activity, + user: user, + data_attrs: %{ + "cc" => [@as_public], + "to" => [user.follower_address] + } + ) + + prepared = + Publisher.prepare_one(%{ + inbox: "https://remote.instance/users/someone/inbox", + activity_id: activity.id, + cc: ["https://remote.instance/users/someone_else/inbox"] + }) + + {:ok, decoded} = Jason.decode(prepared.json) + + assert decoded["cc"] == [@as_public, "https://remote.instance/users/someone_else/inbox"] end test "public address in cc parameter is preserved" do From 7c64bfaace454185c4428fec0e7247ba93fff048 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 12 Jun 2025 22:42:40 -0700 Subject: [PATCH 123/128] Include public address in cc if original activity specified it and Publisher param_cc also has values --- lib/pleroma/web/activity_pub/publisher.ex | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex index 762c991fd..f160f1e17 100644 --- a/lib/pleroma/web/activity_pub/publisher.ex +++ b/lib/pleroma/web/activity_pub/publisher.ex @@ -99,11 +99,11 @@ defmodule Pleroma.Web.ActivityPub.Publisher do public_address = Pleroma.Constants.as_public() - # Avoid overriding explicitly set cc values for specific recipients. - # e.g., this ensures unlisted posts are visible to users on other servers. + # Ensure unlisted posts don't lose the public address in the cc + # if the param_cc was set cc = - if public_address in original_cc and param_cc == [] do - [public_address] + if public_address in original_cc and public_address not in param_cc do + [public_address | param_cc] else param_cc end From 00d536d9e2c6c76d88724e5c75cc82a857519c65 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Thu, 30 Jan 2025 15:50:50 +0100 Subject: [PATCH 124/128] backports: Copy mkdir_p TOCTOU fix from elixir PR 14242 See: https://github.com/elixir-lang/elixir/pull/14242 --- changelog.d/toctou-mkdir.fix | 1 + lib/pleroma/backports.ex | 72 ++++++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+) create mode 100644 changelog.d/toctou-mkdir.fix create mode 100644 lib/pleroma/backports.ex diff --git a/changelog.d/toctou-mkdir.fix b/changelog.d/toctou-mkdir.fix new file mode 100644 index 000000000..b070db1a0 --- /dev/null +++ b/changelog.d/toctou-mkdir.fix @@ -0,0 +1 @@ +Backport [Elixir PR 14242](https://github.com/elixir-lang/elixir/pull/14242) fixing racy mkdir and lack of error handling of parent directory creation \ No newline at end of file diff --git a/lib/pleroma/backports.ex b/lib/pleroma/backports.ex new file mode 100644 index 000000000..68cb7b990 --- /dev/null +++ b/lib/pleroma/backports.ex @@ -0,0 +1,72 @@ +# Copyright 2012 Plataformatec +# Copyright 2021 The Elixir Team +# SPDX-License-Identifier: Apache-2.0 + +defmodule Pleroma.Backports do + import File, only: [dir?: 1] + + # + # To be removed when we require Elixir 1.19 + @doc """ + Tries to create the directory `path`. + + Missing parent directories are created. Returns `:ok` if successful, or + `{:error, reason}` if an error occurs. + + Typical error reasons are: + + * `:eacces` - missing search or write permissions for the parent + directories of `path` + * `:enospc` - there is no space left on the device + * `:enotdir` - a component of `path` is not a directory + + """ + @spec mkdir_p(Path.t()) :: :ok | {:error, File.posix() | :badarg} + def mkdir_p(path) do + do_mkdir_p(IO.chardata_to_string(path)) + end + + defp do_mkdir_p("/") do + :ok + end + + defp do_mkdir_p(path) do + parent = Path.dirname(path) + + if parent == path do + :ok + else + case do_mkdir_p(parent) do + :ok -> + case :file.make_dir(path) do + {:error, :eexist} -> + if dir?(path), do: :ok, else: {:error, :enotdir} + + other -> + other + end + + e -> + e + end + end + end + + @doc """ + Same as `mkdir_p/1`, but raises a `File.Error` exception in case of failure. + Otherwise `:ok`. + """ + @spec mkdir_p!(Path.t()) :: :ok + def mkdir_p!(path) do + case mkdir_p(path) do + :ok -> + :ok + + {:error, reason} -> + raise File.Error, + reason: reason, + action: "make directory (with -p)", + path: IO.chardata_to_string(path) + end + end +end From a69e417020bcbbb998b1e8c039b1cfde23f60a02 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Thu, 30 Jan 2025 16:05:49 +0100 Subject: [PATCH 125/128] File.mkdir_p -> Pleroma.Backports.mkdir_p --- lib/mix/tasks/pleroma/instance.ex | 2 +- lib/mix/tasks/pleroma/robots_txt.ex | 2 +- lib/pleroma/emoji/pack.ex | 6 +++--- lib/pleroma/frontend.ex | 4 ++-- lib/pleroma/uploaders/local.ex | 2 +- lib/pleroma/user/backup.ex | 2 +- lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex | 2 +- lib/pleroma/web/instance_document.ex | 2 +- test/mix/tasks/pleroma/frontend_test.exs | 4 ++-- test/mix/tasks/pleroma/instance_test.exs | 2 +- test/mix/tasks/pleroma/uploads_test.exs | 2 +- test/pleroma/emoji/pack_test.exs | 2 +- test/pleroma/frontend_test.exs | 4 ++-- test/pleroma/object_test.exs | 2 +- test/pleroma/safe_zip_test.exs | 10 +++++----- .../admin_api/controllers/frontend_controller_test.exs | 2 +- .../controllers/instance_document_controller_test.exs | 2 +- test/pleroma/web/plugs/frontend_static_plug_test.exs | 8 ++++---- test/pleroma/web/plugs/instance_static_test.exs | 4 ++-- 19 files changed, 32 insertions(+), 32 deletions(-) diff --git a/lib/mix/tasks/pleroma/instance.ex b/lib/mix/tasks/pleroma/instance.ex index 0dc30549c..143af5cdd 100644 --- a/lib/mix/tasks/pleroma/instance.ex +++ b/lib/mix/tasks/pleroma/instance.ex @@ -271,7 +271,7 @@ defmodule Mix.Tasks.Pleroma.Instance do [config_dir, psql_dir, static_dir, uploads_dir] |> Enum.reject(&File.exists?/1) |> Enum.each(fn dir -> - File.mkdir_p!(dir) + Pleroma.Backports.mkdir_p!(dir) File.chmod!(dir, 0o700) end) diff --git a/lib/mix/tasks/pleroma/robots_txt.ex b/lib/mix/tasks/pleroma/robots_txt.ex index 5124c7c40..e741f3cf0 100644 --- a/lib/mix/tasks/pleroma/robots_txt.ex +++ b/lib/mix/tasks/pleroma/robots_txt.ex @@ -22,7 +22,7 @@ defmodule Mix.Tasks.Pleroma.RobotsTxt do static_dir = Pleroma.Config.get([:instance, :static_dir], "instance/static/") if !File.exists?(static_dir) do - File.mkdir_p!(static_dir) + Pleroma.Backports.mkdir_p!(static_dir) end robots_txt_path = Path.join(static_dir, "robots.txt") diff --git a/lib/pleroma/emoji/pack.ex b/lib/pleroma/emoji/pack.ex index c58748d3c..99fa1994f 100644 --- a/lib/pleroma/emoji/pack.ex +++ b/lib/pleroma/emoji/pack.ex @@ -488,7 +488,7 @@ defmodule Pleroma.Emoji.Pack do with true <- String.contains?(file_path, "/"), path <- Path.dirname(file_path), false <- File.exists?(path) do - File.mkdir_p!(path) + Pleroma.Backports.mkdir_p!(path) end end @@ -536,7 +536,7 @@ defmodule Pleroma.Emoji.Pack do emoji_path = emoji_path() # Create the directory first if it does not exist. This is probably the first request made # with the API so it should be sufficient - with {:create_dir, :ok} <- {:create_dir, File.mkdir_p(emoji_path)}, + with {:create_dir, :ok} <- {:create_dir, Pleroma.Backports.mkdir_p(emoji_path)}, {:ls, {:ok, results}} <- {:ls, File.ls(emoji_path)} do {:ok, Enum.sort(results)} else @@ -561,7 +561,7 @@ defmodule Pleroma.Emoji.Pack do end defp unzip(archive, pack_info, remote_pack, local_pack) do - with :ok <- File.mkdir_p!(local_pack.path) do + with :ok <- Pleroma.Backports.mkdir_p!(local_pack.path) do files = Enum.map(remote_pack["files"], fn {_, path} -> path end) # Fallback cannot contain a pack.json file files = if pack_info[:fallback], do: files, else: ["pack.json" | files] diff --git a/lib/pleroma/frontend.ex b/lib/pleroma/frontend.ex index fe7f525ea..e651d7d9d 100644 --- a/lib/pleroma/frontend.ex +++ b/lib/pleroma/frontend.ex @@ -66,7 +66,7 @@ defmodule Pleroma.Frontend do def unzip(zip, dest) do File.rm_rf!(dest) - File.mkdir_p!(dest) + Pleroma.Backports.mkdir_p!(dest) case Pleroma.SafeZip.unzip_data(zip, dest) do {:ok, _} -> :ok @@ -90,7 +90,7 @@ defmodule Pleroma.Frontend do defp install_frontend(frontend_info, source, dest) do from = frontend_info["build_dir"] || "dist" File.rm_rf!(dest) - File.mkdir_p!(dest) + Pleroma.Backports.mkdir_p!(dest) File.cp_r!(Path.join([source, from]), dest) :ok end diff --git a/lib/pleroma/uploaders/local.ex b/lib/pleroma/uploaders/local.ex index e4a309cea..7aab05b36 100644 --- a/lib/pleroma/uploaders/local.ex +++ b/lib/pleroma/uploaders/local.ex @@ -19,7 +19,7 @@ defmodule Pleroma.Uploaders.Local do [file | folders] -> path = Path.join([upload_path()] ++ Enum.reverse(folders)) - File.mkdir_p!(path) + Pleroma.Backports.mkdir_p!(path) {path, file} end diff --git a/lib/pleroma/user/backup.ex b/lib/pleroma/user/backup.ex index 244b08adb..3f67cdf0c 100644 --- a/lib/pleroma/user/backup.ex +++ b/lib/pleroma/user/backup.ex @@ -193,7 +193,7 @@ defmodule Pleroma.User.Backup do backup = Repo.preload(backup, :user) tempfile = Path.join([backup.tempdir, backup.file_name]) - with {_, :ok} <- {:mkdir, File.mkdir_p(backup.tempdir)}, + with {_, :ok} <- {:mkdir, Pleroma.Backports.mkdir_p(backup.tempdir)}, {_, :ok} <- {:actor, actor(backup.tempdir, backup.user)}, {_, :ok} <- {:statuses, statuses(backup.tempdir, backup.user)}, {_, :ok} <- {:likes, likes(backup.tempdir, backup.user)}, diff --git a/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex b/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex index 49d17d8b9..54f0e6bc1 100644 --- a/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex @@ -87,7 +87,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.StealEmojiPolicy do Path.join(Config.get([:instance, :static_dir]), "emoji/stolen") ) - File.mkdir_p(emoji_dir_path) + Pleroma.Backports.mkdir_p(emoji_dir_path) new_emojis = foreign_emojis diff --git a/lib/pleroma/web/instance_document.ex b/lib/pleroma/web/instance_document.ex index 9da3c5008..143a0b0b8 100644 --- a/lib/pleroma/web/instance_document.ex +++ b/lib/pleroma/web/instance_document.ex @@ -46,7 +46,7 @@ defmodule Pleroma.Web.InstanceDocument do defp put_file(origin_path, destination_path) do with destination <- instance_static_dir(destination_path), - {_, :ok} <- {:mkdir_p, File.mkdir_p(Path.dirname(destination))}, + {_, :ok} <- {:mkdir_p, Pleroma.Backports.mkdir_p(Path.dirname(destination))}, {_, {:ok, _}} <- {:copy, File.copy(origin_path, destination)} do :ok else diff --git a/test/mix/tasks/pleroma/frontend_test.exs b/test/mix/tasks/pleroma/frontend_test.exs index 6d09f8e36..59ebcec92 100644 --- a/test/mix/tasks/pleroma/frontend_test.exs +++ b/test/mix/tasks/pleroma/frontend_test.exs @@ -11,7 +11,7 @@ defmodule Mix.Tasks.Pleroma.FrontendTest do @dir "test/frontend_static_test" setup do - File.mkdir_p!(@dir) + Pleroma.Backports.mkdir_p!(@dir) clear_config([:instance, :static_dir], @dir) on_exit(fn -> @@ -50,7 +50,7 @@ defmodule Mix.Tasks.Pleroma.FrontendTest do folder = Path.join([@dir, "frontends", "pleroma", "fantasy"]) previously_existing = Path.join([folder, "temp"]) - File.mkdir_p!(folder) + Pleroma.Backports.mkdir_p!(folder) File.write!(previously_existing, "yey") assert File.exists?(previously_existing) diff --git a/test/mix/tasks/pleroma/instance_test.exs b/test/mix/tasks/pleroma/instance_test.exs index b1c10e03c..5ecb6e445 100644 --- a/test/mix/tasks/pleroma/instance_test.exs +++ b/test/mix/tasks/pleroma/instance_test.exs @@ -7,7 +7,7 @@ defmodule Mix.Tasks.Pleroma.InstanceTest do use Pleroma.DataCase setup do - File.mkdir_p!(tmp_path()) + Pleroma.Backports.mkdir_p!(tmp_path()) on_exit(fn -> File.rm_rf(tmp_path()) diff --git a/test/mix/tasks/pleroma/uploads_test.exs b/test/mix/tasks/pleroma/uploads_test.exs index f3d5aa64f..0aa24807e 100644 --- a/test/mix/tasks/pleroma/uploads_test.exs +++ b/test/mix/tasks/pleroma/uploads_test.exs @@ -62,7 +62,7 @@ defmodule Mix.Tasks.Pleroma.UploadsTest do upload_dir = Config.get([Pleroma.Uploaders.Local, :uploads]) if not File.exists?(upload_dir) || File.ls!(upload_dir) == [] do - File.mkdir_p(upload_dir) + Pleroma.Backports.mkdir_p(upload_dir) Path.join([upload_dir, "file.txt"]) |> File.touch() diff --git a/test/pleroma/emoji/pack_test.exs b/test/pleroma/emoji/pack_test.exs index 6ab3e657e..b458401a7 100644 --- a/test/pleroma/emoji/pack_test.exs +++ b/test/pleroma/emoji/pack_test.exs @@ -58,7 +58,7 @@ defmodule Pleroma.Emoji.PackTest do test "skips existing emojis when adding from zip file", %{pack: pack} do # First, let's create a test pack with a "bear" emoji test_pack_path = Path.join(@emoji_path, "test_bear_pack") - File.mkdir_p(test_pack_path) + Pleroma.Backports.mkdir_p(test_pack_path) # Create a pack.json file File.write!(Path.join(test_pack_path, "pack.json"), """ diff --git a/test/pleroma/frontend_test.exs b/test/pleroma/frontend_test.exs index c89c56c8c..22e0ffb9a 100644 --- a/test/pleroma/frontend_test.exs +++ b/test/pleroma/frontend_test.exs @@ -9,7 +9,7 @@ defmodule Pleroma.FrontendTest do @dir "test/frontend_static_test" setup do - File.mkdir_p!(@dir) + Pleroma.Backports.mkdir_p!(@dir) clear_config([:instance, :static_dir], @dir) on_exit(fn -> @@ -46,7 +46,7 @@ defmodule Pleroma.FrontendTest do folder = Path.join([@dir, "frontends", "pleroma", "fantasy"]) previously_existing = Path.join([folder, "temp"]) - File.mkdir_p!(folder) + Pleroma.Backports.mkdir_p!(folder) File.write!(previously_existing, "yey") assert File.exists?(previously_existing) diff --git a/test/pleroma/object_test.exs b/test/pleroma/object_test.exs index ed5c2b6c8..13e941e4d 100644 --- a/test/pleroma/object_test.exs +++ b/test/pleroma/object_test.exs @@ -156,7 +156,7 @@ defmodule Pleroma.ObjectTest do uploads_dir = Pleroma.Config.get!([Pleroma.Uploaders.Local, :uploads]) - File.mkdir_p!(uploads_dir) + Pleroma.Backports.mkdir_p!(uploads_dir) file = %Plug.Upload{ content_type: "image/jpeg", diff --git a/test/pleroma/safe_zip_test.exs b/test/pleroma/safe_zip_test.exs index 3312d4e63..f07b25675 100644 --- a/test/pleroma/safe_zip_test.exs +++ b/test/pleroma/safe_zip_test.exs @@ -9,12 +9,12 @@ defmodule Pleroma.SafeZipTest do setup do # Ensure tmp directory exists - File.mkdir_p!(@tmp_dir) + Pleroma.Backports.mkdir_p!(@tmp_dir) on_exit(fn -> # Clean up any files created during tests File.rm_rf!(@tmp_dir) - File.mkdir_p!(@tmp_dir) + Pleroma.Backports.mkdir_p!(@tmp_dir) end) :ok @@ -89,7 +89,7 @@ defmodule Pleroma.SafeZipTest do # For this test, we'll manually check if the file exists in the archive # by extracting it and verifying it exists extract_dir = Path.join(@tmp_dir, "extract_check") - File.mkdir_p!(extract_dir) + Pleroma.Backports.mkdir_p!(extract_dir) {:ok, files} = SafeZip.unzip_file(zip_path, extract_dir) # Verify the root file was extracted @@ -145,7 +145,7 @@ defmodule Pleroma.SafeZipTest do test "can create zip with directories" do # Create a directory structure dir_path = Path.join(@tmp_dir, "test_dir") - File.mkdir_p!(dir_path) + Pleroma.Backports.mkdir_p!(dir_path) file_in_dir_path = Path.join(dir_path, "file_in_dir.txt") File.write!(file_in_dir_path, "file in directory") @@ -428,7 +428,7 @@ defmodule Pleroma.SafeZipTest do # Create a directory and a file in it dir_path = Path.join(@tmp_dir, "file_in_dir") - File.mkdir_p!(dir_path) + Pleroma.Backports.mkdir_p!(dir_path) file_in_dir_path = Path.join(dir_path, "test_file.txt") File.write!(file_in_dir_path, "file in directory content") diff --git a/test/pleroma/web/admin_api/controllers/frontend_controller_test.exs b/test/pleroma/web/admin_api/controllers/frontend_controller_test.exs index 0d1a4999e..a6b8dba46 100644 --- a/test/pleroma/web/admin_api/controllers/frontend_controller_test.exs +++ b/test/pleroma/web/admin_api/controllers/frontend_controller_test.exs @@ -13,7 +13,7 @@ defmodule Pleroma.Web.AdminAPI.FrontendControllerTest do setup do clear_config([:instance, :static_dir], @dir) - File.mkdir_p!(Pleroma.Frontend.dir()) + Pleroma.Backports.mkdir_p!(Pleroma.Frontend.dir()) on_exit(fn -> File.rm_rf(@dir) diff --git a/test/pleroma/web/admin_api/controllers/instance_document_controller_test.exs b/test/pleroma/web/admin_api/controllers/instance_document_controller_test.exs index 9511dccea..344c908fe 100644 --- a/test/pleroma/web/admin_api/controllers/instance_document_controller_test.exs +++ b/test/pleroma/web/admin_api/controllers/instance_document_controller_test.exs @@ -10,7 +10,7 @@ defmodule Pleroma.Web.AdminAPI.InstanceDocumentControllerTest do @default_instance_panel ~s(

Welcome to Pleroma!

) setup do - File.mkdir_p!(@dir) + Pleroma.Backports.mkdir_p!(@dir) on_exit(fn -> File.rm_rf(@dir) end) end diff --git a/test/pleroma/web/plugs/frontend_static_plug_test.exs b/test/pleroma/web/plugs/frontend_static_plug_test.exs index 6f4d24d9e..a7af3e74e 100644 --- a/test/pleroma/web/plugs/frontend_static_plug_test.exs +++ b/test/pleroma/web/plugs/frontend_static_plug_test.exs @@ -13,7 +13,7 @@ defmodule Pleroma.Web.Plugs.FrontendStaticPlugTest do @dir "test/tmp/instance_static" setup do - File.mkdir_p!(@dir) + Pleroma.Backports.mkdir_p!(@dir) on_exit(fn -> File.rm_rf(@dir) end) end @@ -38,7 +38,7 @@ defmodule Pleroma.Web.Plugs.FrontendStaticPlugTest do clear_config([:frontends, :primary], %{"name" => name, "ref" => ref}) path = "#{@dir}/frontends/#{name}/#{ref}" - File.mkdir_p!(path) + Pleroma.Backports.mkdir_p!(path) File.write!("#{path}/index.html", "from frontend plug") index = get(conn, "/") @@ -52,7 +52,7 @@ defmodule Pleroma.Web.Plugs.FrontendStaticPlugTest do clear_config([:frontends, :admin], %{"name" => name, "ref" => ref}) path = "#{@dir}/frontends/#{name}/#{ref}" - File.mkdir_p!(path) + Pleroma.Backports.mkdir_p!(path) File.write!("#{path}/index.html", "from frontend plug") index = get(conn, "/pleroma/admin/") @@ -67,7 +67,7 @@ defmodule Pleroma.Web.Plugs.FrontendStaticPlugTest do clear_config([:frontends, :primary], %{"name" => name, "ref" => ref}) path = "#{@dir}/frontends/#{name}/#{ref}" - File.mkdir_p!("#{path}/proxy/rr/ss") + Pleroma.Backports.mkdir_p!("#{path}/proxy/rr/ss") File.write!("#{path}/proxy/rr/ss/Ek7w8WPVcAApOvN.jpg:large", "FB image") ConfigMock diff --git a/test/pleroma/web/plugs/instance_static_test.exs b/test/pleroma/web/plugs/instance_static_test.exs index 33b74dcf0..b5a5a3334 100644 --- a/test/pleroma/web/plugs/instance_static_test.exs +++ b/test/pleroma/web/plugs/instance_static_test.exs @@ -8,7 +8,7 @@ defmodule Pleroma.Web.Plugs.InstanceStaticTest do @dir "test/tmp/instance_static" setup do - File.mkdir_p!(@dir) + Pleroma.Backports.mkdir_p!(@dir) on_exit(fn -> File.rm_rf(@dir) end) end @@ -34,7 +34,7 @@ defmodule Pleroma.Web.Plugs.InstanceStaticTest do refute html_response(bundled_index, 200) == "from frontend plug" path = "#{@dir}/frontends/#{name}/#{ref}" - File.mkdir_p!(path) + Pleroma.Backports.mkdir_p!(path) File.write!("#{path}/index.html", "from frontend plug") index = get(conn, "/") From 7ecfb953316d718235042e60a58b464ccdd764f0 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Fri, 13 Jun 2025 22:47:32 +0300 Subject: [PATCH 126/128] Handle the Dislike activity by transforming into a thumbs-down emote --- changelog.d/dislike-activity.add | 1 + lib/pleroma/constants.ex | 2 + .../web/activity_pub/transmogrifier.ex | 18 +++++ test/fixtures/friendica-dislike-undo.json | 76 +++++++++++++++++++ test/fixtures/friendica-dislike.json | 56 ++++++++++++++ .../transmogrifier/like_handling_test.exs | 36 +++++++++ 6 files changed, 189 insertions(+) create mode 100644 changelog.d/dislike-activity.add create mode 100644 test/fixtures/friendica-dislike-undo.json create mode 100644 test/fixtures/friendica-dislike.json diff --git a/changelog.d/dislike-activity.add b/changelog.d/dislike-activity.add new file mode 100644 index 000000000..1fcbda78b --- /dev/null +++ b/changelog.d/dislike-activity.add @@ -0,0 +1 @@ +Support Dislike activity, as sent by Mitra and Friendica, by changing it into a thumbs-down EmojiReact \ No newline at end of file diff --git a/lib/pleroma/constants.ex b/lib/pleroma/constants.ex index 3762c0035..92ca11494 100644 --- a/lib/pleroma/constants.ex +++ b/lib/pleroma/constants.ex @@ -100,6 +100,7 @@ defmodule Pleroma.Constants do "Add", "Remove", "Like", + "Dislike", "Announce", "Undo", "Flag", @@ -115,6 +116,7 @@ defmodule Pleroma.Constants do "Flag", "Follow", "Like", + "Dislike", "EmojiReact", "Announce" ] diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 6517f5eff..8819e1596 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -664,6 +664,24 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end end + # Rewrite dislikes into the thumbs down emoji + defp handle_incoming_normalized(%{"type" => "Dislike"} = data, options) do + data + |> Map.put("type", "EmojiReact") + |> Map.put("content", "👎") + |> handle_incoming_normalized(options) + end + + defp handle_incoming_normalized( + %{"type" => "Undo", "object" => %{"type" => "Dislike"}} = data, + options + ) do + data + |> put_in(["object", "type"], "EmojiReact") + |> put_in(["object", "content"], "👎") + |> handle_incoming_normalized(options) + end + defp handle_incoming_normalized(_, _), do: :error @spec get_obj_helper(String.t(), Keyword.t()) :: {:ok, Object.t()} | nil diff --git a/test/fixtures/friendica-dislike-undo.json b/test/fixtures/friendica-dislike-undo.json new file mode 100644 index 000000000..b258e00be --- /dev/null +++ b/test/fixtures/friendica-dislike-undo.json @@ -0,0 +1,76 @@ +{ + "@context": [ + "https://www.w3.org/ns/activitystreams", + "https://w3id.org/security/v1", + { + "Hashtag": "as:Hashtag", + "PropertyValue": "schema:PropertyValue", + "conversation": "ostatus:conversation", + "dfrn": "http://purl.org/macgirvin/dfrn/1.0/", + "diaspora": "https://diasporafoundation.org/ns/", + "directMessage": "litepub:directMessage", + "discoverable": "toot:discoverable", + "featured": { + "@id": "toot:featured", + "@type": "@id" + }, + "litepub": "http://litepub.social/ns#", + "manuallyApprovesFollowers": "as:manuallyApprovesFollowers", + "ostatus": "http://ostatus.org#", + "quoteUrl": "as:quoteUrl", + "schema": "http://schema.org#", + "sensitive": "as:sensitive", + "toot": "http://joinmastodon.org/ns#", + "value": "schema:value", + "vcard": "http://www.w3.org/2006/vcard/ns#" + } + ], + "actor": "https://my-place.social/profile/vaartis", + "cc": [ + "https://my-place.social/followers/vaartis" + ], + "id": "https://my-place.social/objects/e599373b-1368-4b20-cd24-837166957182/Undo", + "instrument": { + "id": "https://my-place.social/friendica", + "name": "Friendica 'Interrupted Fern' 2024.12-1576", + "type": "Application", + "url": "https://my-place.social" + }, + "object": { + "actor": "https://my-place.social/profile/vaartis", + "cc": [ + "https://my-place.social/followers/vaartis" + ], + "diaspora:guid": "e599373b-1968-4b20-cd24-80d340160302", + "diaspora:like": "{\"author\":\"vaartis@my-place.social\",\"guid\":\"e599373b-1968-4b20-cd24-80d340160302\",\"parent_guid\":\"cd36feba-c31f3ed3fd5c064a-17c31593\",\"parent_type\":\"Post\",\"positive\":\"false\",\"author_signature\":\"xR2zLJNfc9Nhx1n8LLMWM1kde12my4cqamIsrH\\/UntKzuDwO4DuHBL0fkFhgC\\/dylxm4HqsHD45MQbtwQCVGq6WhC96TrbMuYEK61HIO23dTr3m+qJVtfdH4wyhUNHgiiYPhZpkLDfnR1JiRWmFTlmZC8q8JEkOB5IQsrWia2eOR6IsqDcdKO\\/Urgns9\\/BdQi8KnchBKSEFc1iUtcOEruvhunKGyW5zI\\/Rltfdz3xGH8tlw+YlMXeWXPnqgOJ9GzNA0lwG4U421L6yylYagW7oxIznnBLB4bO46vYZbgXZV1hiI9ZyveHOinLMY1QkmTj5CNvnx3\\/VJwLovd0v+0Nr2vu\\/3ftbpBXc6L1bsNjlRqtsfwJlcgl+tH1DC4W8tKf+Y3tdtzVw0CHXCuacxHLyq5wZd\\/5YfYR9SJQ\\/jInU4PHA5+hIE3PGqNUp5QfFE0umq56H7MQKsIPgM5mMV4fPAA8OpltuMVDvQYUxalrnvoTf00k90x1wCTK71\\/jQGh7r7PmGvSdfPr+65eVTjITD8\\/lTGIb8850v1fl3\\/i2R8Dv17jQIRyX5o9MXPSO6jHo4Swma5RzPA\\/0bRj6qRTyPkM1L9qEIr+2H2I7KKhT2ZE5GhAU7yI9A3VLBWzpTrUPMGbfpd1OjVTEqXAdMjpLDYI3Mh5zQ58p8xCzt+W+t0=\"}", + "id": "https://my-place.social/objects/e599373b-1368-4b20-cd24-837166957182", + "instrument": { + "id": "https://my-place.social/friendica", + "name": "Friendica 'Interrupted Fern' 2024.12-1576", + "type": "Application", + "url": "https://my-place.social" + }, + "object": "https://pl.kotobank.ch/objects/301bce65-8a1b-4c49-a65c-fe2ce861a213", + "published": "2025-06-12T18:47:41Z", + "to": [ + "https://pl.kotobank.ch/users/vaartis", + "https://mitra.social/users/silverpill", + "https://www.w3.org/ns/activitystreams#Public" + ], + "type": "Dislike" + }, + "published": "2025-06-12T18:41:25Z", + "signature": { + "created": "2025-06-12T18:44:16Z", + "creator": "https://my-place.social/profile/vaartis#main-key", + "nonce": "2d67847d4bd4b1b83a30d61eac6cdc7ad6b980df06a8b9b97217e1d8f7b6cf20", + "signatureValue": "LnoRMZuQGDvTICkShGBq28ynaj2lF1bViJFGS6n4gKn3IbxPWATHxao43gxWRc+HCTrHNg7quzgaW4+PYM7UVUz3jO+bjNKsN845nijOVdyFrPOXbuaij3KQh2OoHhFJWoV/ZQQTFF0kRK1qT4BwG+P8NqOOKAMv+Cw7ruQH+f2w7uDgcNIbCD1gLcwb6cw7WVe5qu8yMkKqp2kBdqW3RCsI85RmmFgwehDgH5nrX7ER1qbeLWrqy7echwD9/fO3rqAu13xDNyiGZHDT7JB3RUt0AyMm0XCfjbwSQ0n+MkYXgE4asvFz81+iiPCLt+6gePWAFc5odF1FxdySBpSuUOs4p92NzP9OhQ0c0qrqrzYI7aYklY7oMfxjkva+X+0bm3up+2IRJdnZa/pXlmwdcqTpyMr1sgzaexMUNBp3dq7zA51eEaakLDX3i2onXJowfmze3+6XgPAFHYamR+pRNtuEoY4uyYEK3fj5GgwJ4RtFJMYVoEs/Q8h3OgYRcK1FE9UlDjSqbQ7QIRn2Ib4wjgmkeM0vrHIwh/1CtqA/M/6WuYFzCaJBc8O9ykpK9ZMbw64ToQXKf2SqhZsDoyTWRWTO1PXOk1XCAAElUh8/WCyeghvgqLXn0LHov4lmBsHA5iMUcLqBKD3GJIHd+ExrOFxMZs4mBLLGyz0p5joJ3NY=", + "type": "RsaSignature2017" + }, + "to": [ + "https://pl.kotobank.ch/users/vaartis", + "https://mitra.social/users/silverpill", + "https://www.w3.org/ns/activitystreams#Public" + ], + "type": "Undo" +} diff --git a/test/fixtures/friendica-dislike.json b/test/fixtures/friendica-dislike.json new file mode 100644 index 000000000..c75939073 --- /dev/null +++ b/test/fixtures/friendica-dislike.json @@ -0,0 +1,56 @@ +{ + "@context": [ + "https://www.w3.org/ns/activitystreams", + "https://w3id.org/security/v1", + { + "Hashtag": "as:Hashtag", + "PropertyValue": "schema:PropertyValue", + "conversation": "ostatus:conversation", + "dfrn": "http://purl.org/macgirvin/dfrn/1.0/", + "diaspora": "https://diasporafoundation.org/ns/", + "directMessage": "litepub:directMessage", + "discoverable": "toot:discoverable", + "featured": { + "@id": "toot:featured", + "@type": "@id" + }, + "litepub": "http://litepub.social/ns#", + "manuallyApprovesFollowers": "as:manuallyApprovesFollowers", + "ostatus": "http://ostatus.org#", + "quoteUrl": "as:quoteUrl", + "schema": "http://schema.org#", + "sensitive": "as:sensitive", + "toot": "http://joinmastodon.org/ns#", + "value": "schema:value", + "vcard": "http://www.w3.org/2006/vcard/ns#" + } + ], + "actor": "https://my-place.social/profile/vaartis", + "cc": [ + "https://my-place.social/followers/vaartis" + ], + "diaspora:guid": "e599373b-1968-4b20-cd24-80d340160302", + "diaspora:like": "{\"author\":\"vaartis@my-place.social\",\"guid\":\"e599373b-1968-4b20-cd24-80d340160302\",\"parent_guid\":\"cd36feba-c31f3ed3fd5c064a-17c31593\",\"parent_type\":\"Post\",\"positive\":\"false\",\"author_signature\":\"xR2zLJNfc9Nhx1n8LLMWM1kde12my4cqamIsrH\\/UntKzuDwO4DuHBL0fkFhgC\\/dylxm4HqsHD45MQbtwQCVGq6WhC96TrbMuYEK61HIO23dTr3m+qJVtfdH4wyhUNHgiiYPhZpkLDfnR1JiRWmFTlmZC8q8JEkOB5IQsrWia2eOR6IsqDcdKO\\/Urgns9\\/BdQi8KnchBKSEFc1iUtcOEruvhunKGyW5zI\\/Rltfdz3xGH8tlw+YlMXeWXPnqgOJ9GzNA0lwG4U421L6yylYagW7oxIznnBLB4bO46vYZbgXZV1hiI9ZyveHOinLMY1QkmTj5CNvnx3\\/VJwLovd0v+0Nr2vu\\/3ftbpBXc6L1bsNjlRqtsfwJlcgl+tH1DC4W8tKf+Y3tdtzVw0CHXCuacxHLyq5wZd\\/5YfYR9SJQ\\/jInU4PHA5+hIE3PGqNUp5QfFE0umq56H7MQKsIPgM5mMV4fPAA8OpltuMVDvQYUxalrnvoTf00k90x1wCTK71\\/jQGh7r7PmGvSdfPr+65eVTjITD8\\/lTGIb8850v1fl3\\/i2R8Dv17jQIRyX5o9MXPSO6jHo4Swma5RzPA\\/0bRj6qRTyPkM1L9qEIr+2H2I7KKhT2ZE5GhAU7yI9A3VLBWzpTrUPMGbfpd1OjVTEqXAdMjpLDYI3Mh5zQ58p8xCzt+W+t0=\"}", + "id": "https://my-place.social/objects/e599373b-1368-4b20-cd24-837166957182", + "instrument": { + "id": "https://my-place.social/friendica", + "name": "Friendica 'Interrupted Fern' 2024.12-1576", + "type": "Application", + "url": "https://my-place.social" + }, + "object": "https://pl.kotobank.ch/objects/301bce65-8a1b-4c49-a65c-fe2ce861a213", + "published": "2025-06-12T18:47:41Z", + "signature": { + "created": "2025-06-12T18:47:42Z", + "creator": "https://my-place.social/profile/vaartis#main-key", + "nonce": "84e496f80b09d7a299c5cc89e8cadd13abf621b3a0a321684fa74278b68a6dd8", + "signatureValue": "qe2WxY+j7daIYLRadCctgal6A1s9XgoiMfM/8KjJm15w0sSizYYqruyQ5gS44e+cj5GHc9v5gP2ieod5v7eHAPzlcDI4bfkcyHVapAXTqU67ZebW+v6Q+21IMDgqrkYCv5TbV7LTxltW59dlqovpHE4TEe/M7xLKWJ3vVchRUcWqH9kDmak0nacoqYVAb5E9jYnQhUWPTCfPV82qQpeWQPOZ4iIvPw6rDkSSY5jL6bCogBZblHGpUjXfe/FPlacaCWiTQdoga3yOBXB1RYPw9nh5FI5Xkv/oi+52WmJrECinlD6AL8/BpiYvKz236zy7p/TR4BXlCx9RR/msjOnSabkQ4kmYFrRr80UDCGF+CdkdzLl8K9rSE3PbF1+nEqD7X0GOWn/DdtixqXJw6IR4bh32YW2SlcrSRBvI1p82Mv68BeqRaYqL6FAhKFwLhX5JpXngZ3k0g7rWWxc498voPWnFZDyCTRNxO9VIIUavDDEQ0BdFk6WDb8zx9tsAg8JoK57eVDcFly7tfVQffYiHpve06d8ag1DtzipqguRsURmuqpGNMq28XBTxwtrP2LnXXHKxoYN/YQ9cDnCKclbx7/uKmOVMLkLZlM0wAVoZpm5z2fG4voKqFiGZ1PoiFY2sN4URMArJtygV3PlTX4ASAQrak0ksvEo9msrBUD0Su9c=", + "type": "RsaSignature2017" + }, + "to": [ + "https://pl.kotobank.ch/users/vaartis", + "https://mitra.social/users/silverpill", + "https://www.w3.org/ns/activitystreams#Public" + ], + "type": "Dislike" +} diff --git a/test/pleroma/web/activity_pub/transmogrifier/like_handling_test.exs b/test/pleroma/web/activity_pub/transmogrifier/like_handling_test.exs index fc04c1391..27f8522ce 100644 --- a/test/pleroma/web/activity_pub/transmogrifier/like_handling_test.exs +++ b/test/pleroma/web/activity_pub/transmogrifier/like_handling_test.exs @@ -143,4 +143,40 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.LikeHandlingTest do assert {:ok, activity} = Transmogrifier.handle_incoming(data) assert activity.data["type"] == "Like" end + + test "it changes incoming dislikes into emoji reactions" do + user = insert(:user) + + {:ok, activity} = CommonAPI.post(user, %{status: "hello"}) + + data = + File.read!("test/fixtures/friendica-dislike.json") + |> Jason.decode!() + |> Map.put("object", activity.data["object"]) + + _actor = insert(:user, ap_id: data["actor"], local: false) + + {:ok, %Activity{data: data, local: false} = activity} = Transmogrifier.handle_incoming(data) + + refute Enum.empty?(activity.recipients) + + assert data["actor"] == "https://my-place.social/profile/vaartis" + assert data["type"] == "EmojiReact" + assert data["content"] == "👎" + assert data["id"] == "https://my-place.social/objects/e599373b-1368-4b20-cd24-837166957182" + assert data["object"] == activity.data["object"] + + data = + File.read!("test/fixtures/friendica-dislike-undo.json") + |> Jason.decode!() + |> put_in(["object", "object"], activity.data["object"]) + + {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data) + + assert data["actor"] == "https://my-place.social/profile/vaartis" + assert data["type"] == "Undo" + + assert data["object"] == + "https://my-place.social/objects/e599373b-1368-4b20-cd24-837166957182" + end end From 0151d99202749b5ccfe01beda3704e40b0f52548 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Wed, 18 Jun 2025 17:36:08 +0300 Subject: [PATCH 127/128] Use manually created variables for CI instead of CI_JOB_TOKEN For protected branches, it seems now just CI_JOB_TOKEN is not enough. https://gitlab.com/gitlab-org/gitlab-foss/-/issues/36898#note_38415655 According to this, the CI_JOB_TOKEN is based on whoever created the job and creating a pipeline on a protected branch requires special permissions. Somehow this still did not work for other people who merged, even though they had access to the docs repo. --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 29ee24a05..bfd9bf414 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -208,7 +208,7 @@ docs-deploy: before_script: - apk add curl script: - - curl --fail-with-body -X POST -F"token=$CI_JOB_TOKEN" -F'ref=master' -F"variables[BRANCH]=$CI_COMMIT_REF_NAME" https://git.pleroma.social/api/v4/projects/673/trigger/pipeline + - curl --fail-with-body -X POST -F"token=$DOCS_PIPELINE_TRIGGER" -F'ref=master' -F"variables[BRANCH]=$CI_COMMIT_REF_NAME" https://git.pleroma.social/api/v4/projects/673/trigger/pipeline review_app: image: alpine:3.9 stage: deploy @@ -249,7 +249,7 @@ spec-deploy: before_script: - apk add curl script: - - curl --fail-with-body -X POST -F"token=$CI_JOB_TOKEN" -F'ref=master' -F"variables[BRANCH]=$CI_COMMIT_REF_NAME" -F"variables[JOB_REF]=$CI_JOB_ID" https://git.pleroma.social/api/v4/projects/1130/trigger/pipeline + - curl --fail-with-body -X POST -F"token=$API_DOCS_PIPELINE_TRIGGER" -F'ref=master' -F"variables[BRANCH]=$CI_COMMIT_REF_NAME" -F"variables[JOB_REF]=$CI_JOB_ID" https://git.pleroma.social/api/v4/projects/1130/trigger/pipeline stop_review_app: From 9d6f201e5eb37c74490fc47b9b9c98575c6803e6 Mon Sep 17 00:00:00 2001 From: Pleroma User <66706-pleromian@users.noreply.git.pleroma.social> Date: Fri, 20 Jun 2025 21:22:27 +0000 Subject: [PATCH 128/128] Add tos setting --- changelog.d/tos-setting.add | 1 + config/config.exs | 1 + config/description.exs | 7 +++++++ 3 files changed, 9 insertions(+) create mode 100644 changelog.d/tos-setting.add diff --git a/changelog.d/tos-setting.add b/changelog.d/tos-setting.add new file mode 100644 index 000000000..db9b0d5f2 --- /dev/null +++ b/changelog.d/tos-setting.add @@ -0,0 +1 @@ +Allow Terms of Service panel behaviour to be configurable diff --git a/config/config.exs b/config/config.exs index a231c5ba0..31d7258ee 100644 --- a/config/config.exs +++ b/config/config.exs @@ -307,6 +307,7 @@ config :pleroma, :frontend_configurations, collapseMessageWithSubject: false, disableChat: false, greentext: false, + embeddedToS: true, hideFilteredStatuses: false, hideMutedPosts: false, hidePostStats: false, diff --git a/config/description.exs b/config/description.exs index 2f7dc30a0..e20fa4b28 100644 --- a/config/description.exs +++ b/config/description.exs @@ -1261,6 +1261,7 @@ config :pleroma, :config_description, [ background: "/static/aurora_borealis.jpg", collapseMessageWithSubject: false, greentext: false, + embeddedToS: true, hideFilteredStatuses: false, hideMutedPosts: false, hidePostStats: false, @@ -1312,6 +1313,12 @@ config :pleroma, :config_description, [ type: :boolean, description: "Enables green text on lines prefixed with the > character" }, + %{ + key: :embeddedToS, + label: "Embedded ToS panel", + type: :boolean, + description: "Hide Terms of Service panel decorations on About and Registration pages" + }, %{ key: :hideFilteredStatuses, label: "Hide Filtered Statuses",