From 7575b7abc2b1559dbb2a780ce55e572969d8184f Mon Sep 17 00:00:00 2001 From: Phantasm Date: Thu, 14 May 2026 12:20:27 +0200 Subject: [PATCH 01/14] Woodpecker CI: Do not trigger on Weblate MRs --- .woodpecker/changelog.yaml | 1 + .woodpecker/lint.yaml | 1 + .woodpecker/unit-testing-elixir-1.15.yaml | 1 + .woodpecker/unit-testing-elixir-1.18.yaml | 1 + changelog.d/weblate-ci.skip | 0 5 files changed, 4 insertions(+) create mode 100644 changelog.d/weblate-ci.skip diff --git a/.woodpecker/changelog.yaml b/.woodpecker/changelog.yaml index 64062f17e..a3732c240 100644 --- a/.woodpecker/changelog.yaml +++ b/.woodpecker/changelog.yaml @@ -1,5 +1,6 @@ when: - event: pull_request + evaluate: 'CI_COMMIT_SOURCE_BRANCH != "weblate"' labels: platform: linux/amd64 diff --git a/.woodpecker/lint.yaml b/.woodpecker/lint.yaml index 0ab7441a8..8a5718c5d 100644 --- a/.woodpecker/lint.yaml +++ b/.woodpecker/lint.yaml @@ -1,6 +1,7 @@ when: - event: pull_request path: [ "**/*.ex", "**/*.eex", "**/*.exs", "mix.lock", ".woodpecker/**" ] + evaluate: 'CI_COMMIT_SOURCE_BRANCH != "weblate"' - event: push branch: ${CI_REPO_DEFAULT_BRANCH} path: [ "**/*.ex", "**/*.eex", "**/*.exs", "mix.lock", ".woodpecker/**" ] diff --git a/.woodpecker/unit-testing-elixir-1.15.yaml b/.woodpecker/unit-testing-elixir-1.15.yaml index a4a8fc266..a5bcc0ef3 100644 --- a/.woodpecker/unit-testing-elixir-1.15.yaml +++ b/.woodpecker/unit-testing-elixir-1.15.yaml @@ -1,6 +1,7 @@ when: - event: pull_request path: [ "**/*.ex", "**/*.eex", "**/*.exs", "mix.lock", ".woodpecker/**" ] + evaluate: 'CI_COMMIT_SOURCE_BRANCH != "weblate"' - event: push branch: ${CI_REPO_DEFAULT_BRANCH} path: [ "**/*.ex", "**/*.eex", "**/*.exs", "mix.lock", ".woodpecker/**" ] diff --git a/.woodpecker/unit-testing-elixir-1.18.yaml b/.woodpecker/unit-testing-elixir-1.18.yaml index 9ad9eebc9..6e576fad9 100644 --- a/.woodpecker/unit-testing-elixir-1.18.yaml +++ b/.woodpecker/unit-testing-elixir-1.18.yaml @@ -1,6 +1,7 @@ when: - event: pull_request path: [ "**/*.ex", "**/*.eex", "**/*.exs", "mix.lock", ".woodpecker/**" ] + evaluate: 'CI_COMMIT_SOURCE_BRANCH != "weblate"' - event: push branch: ${CI_REPO_DEFAULT_BRANCH} path: [ "**/*.ex", "**/*.eex", "**/*.exs", "mix.lock", ".woodpecker/**" ] diff --git a/changelog.d/weblate-ci.skip b/changelog.d/weblate-ci.skip new file mode 100644 index 000000000..e69de29bb From 1a13ec539baca66d23db899d92b42c181aaf3149 Mon Sep 17 00:00:00 2001 From: Henry Jameson Date: Mon, 18 May 2026 21:05:46 +0300 Subject: [PATCH 02/14] reorganize mfm attributes in order they're listed in misskey repo, add missing ones --- priv/scrubbers/default.ex | 49 ++++++++++++++++++++++----------------- 1 file changed, 28 insertions(+), 21 deletions(-) diff --git a/priv/scrubbers/default.ex b/priv/scrubbers/default.ex index 342ef9944..d5c7340d6 100644 --- a/priv/scrubbers/default.ex +++ b/priv/scrubbers/default.ex @@ -83,48 +83,55 @@ defmodule Pleroma.HTML.Scrubber.Default do "quote-inline", "invisible", "ellipsis", - "mfm-center", - "mfm-flip", - "mfm-font", - "mfm-blur", - "mfm-rotate", - "mfm-x2", - "mfm-x3", - "mfm-x4", - "mfm-position", - "mfm-scale", - "mfm-fg", - "mfm-bg", + "mfm-tada", "mfm-jelly", "mfm-twitch", "mfm-shake", "mfm-spin", "mfm-jump", "mfm-bounce", + "mfm-flip", + "mfm-x2", + "mfm-x3", + "mfm-x4", + "mfm-scale", + "mfm-position", + "mfm-fg", + "mfm-bg", + "mfm-border", + "mfm-font", + "mfm-blur", "mfm-rainbow", - "mfm-tada", - "mfm-sparkle" + "mfm-sparkle", + "mfm-rotate", + "mfm-ruby", + "mfm-unixtime", ]) Meta.allow_tag_with_this_attribute_values(:p, "class", ["quote-inline"]) Meta.allow_tag_with_these_attributes(:span, [ "lang", - "data-mfm-h", - "data-mfm-v", + "data-mfm-speed", + "data-mfm-delay", + "data-mfm-left", + "data-mfm-alternate", "data-mfm-x", "data-mfm-y", - "data-mfm-alternate", - "data-mfm-speed", - "data-mfm-deg", - "data-mfm-left", + "data-mfm-h", + "data-mfm-v", + "data-mfm-color", + "data-mfm-width", + "data-mfm-style", + "data-mfm-radius", + "data-mfm-noclip", "data-mfm-serif", "data-mfm-monospace", "data-mfm-cursive", "data-mfm-fantasy", "data-mfm-emoji", "data-mfm-math", - "data-mfm-color" + "data-mfm-deg", ]) Meta.allow_tag_with_this_attribute_values(:code, "class", ["inline"]) From c428cf43e8683f5428341845629013504d0057bf Mon Sep 17 00:00:00 2001 From: Henry Jameson Date: Mon, 18 May 2026 21:07:12 +0300 Subject: [PATCH 03/14] "changelog" --- changelog.d/mfm-extend.skip | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 changelog.d/mfm-extend.skip diff --git a/changelog.d/mfm-extend.skip b/changelog.d/mfm-extend.skip new file mode 100644 index 000000000..e69de29bb From 46a28086900600d187348b796c486d11575f22da Mon Sep 17 00:00:00 2001 From: Henry Jameson Date: Mon, 18 May 2026 21:08:24 +0300 Subject: [PATCH 04/14] extraneous commas --- priv/scrubbers/default.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/priv/scrubbers/default.ex b/priv/scrubbers/default.ex index d5c7340d6..8963d99e8 100644 --- a/priv/scrubbers/default.ex +++ b/priv/scrubbers/default.ex @@ -105,7 +105,7 @@ defmodule Pleroma.HTML.Scrubber.Default do "mfm-sparkle", "mfm-rotate", "mfm-ruby", - "mfm-unixtime", + "mfm-unixtime" ]) Meta.allow_tag_with_this_attribute_values(:p, "class", ["quote-inline"]) @@ -131,7 +131,7 @@ defmodule Pleroma.HTML.Scrubber.Default do "data-mfm-fantasy", "data-mfm-emoji", "data-mfm-math", - "data-mfm-deg", + "data-mfm-deg" ]) Meta.allow_tag_with_this_attribute_values(:code, "class", ["inline"]) From 9dd02ecd5074cc0c164cbae86253d8eea921c787 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Fri, 22 May 2026 16:30:14 +0400 Subject: [PATCH 05/14] Fix WebSocket protocol token handshakes --- changelog.d/mastodon-websocket-protocol.fix | 1 + lib/pleroma/web/endpoint.ex | 7 +- .../web/mastodon_api/websocket_handler.ex | 15 +-- .../web/mastodon_api/websocket_plug.ex | 105 ++++++++++++++++++ .../integration/mastodon_websocket_test.exs | 100 ++++++++++++++--- 5 files changed, 200 insertions(+), 28 deletions(-) create mode 100644 changelog.d/mastodon-websocket-protocol.fix create mode 100644 lib/pleroma/web/mastodon_api/websocket_plug.ex diff --git a/changelog.d/mastodon-websocket-protocol.fix b/changelog.d/mastodon-websocket-protocol.fix new file mode 100644 index 000000000..66dab16ed --- /dev/null +++ b/changelog.d/mastodon-websocket-protocol.fix @@ -0,0 +1 @@ +Echo Mastodon-style `Sec-WebSocket-Protocol` tokens in streaming WebSocket handshakes. diff --git a/lib/pleroma/web/endpoint.ex b/lib/pleroma/web/endpoint.ex index 36e78e1e2..07b33f866 100644 --- a/lib/pleroma/web/endpoint.ex +++ b/lib/pleroma/web/endpoint.ex @@ -9,8 +9,8 @@ defmodule Pleroma.Web.Endpoint do alias Pleroma.Config - socket("/api/v1/streaming", Pleroma.Web.MastodonAPI.WebsocketHandler, - longpoll: false, + plug(Pleroma.Web.MastodonAPI.WebsocketPlug, + path: "/api/v1/streaming", websocket: [ path: "/", compress: false, @@ -169,8 +169,7 @@ defmodule Pleroma.Web.Endpoint do else: "pleroma_key" extra = - Config.get([__MODULE__, :extra_cookie_attrs]) - |> Enum.join(";") + Enum.join(Config.get([__MODULE__, :extra_cookie_attrs]), ";") # The session will be stored in the cookie and signed, # this means its contents can be read but not tampered with. diff --git a/lib/pleroma/web/mastodon_api/websocket_handler.ex b/lib/pleroma/web/mastodon_api/websocket_handler.ex index 2b698bd5d..3dc862a5a 100644 --- a/lib/pleroma/web/mastodon_api/websocket_handler.ex +++ b/lib/pleroma/web/mastodon_api/websocket_handler.ex @@ -67,9 +67,10 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do @impl Phoenix.Socket.Transport def handle_in({text, [opcode: :text]}, state) do - with {:ok, %{} = event} <- Jason.decode(text) do - handle_client_event(event, state) - else + case Jason.decode(text) do + {:ok, %{} = event} -> + handle_client_event(event, state) + _ -> Logger.error("#{__MODULE__} received non-JSON event: #{inspect(text)}") {:ok, state} @@ -85,11 +86,11 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do def handle_info({:render_with_user, view, template, item, topic}, state) do user = %User{} = User.get_cached_by_ap_id(state.user.ap_id) - unless Streamer.filtered_by_user?(user, item) do + if Streamer.filtered_by_user?(user, item) do + {:ok, state} + else message = view.render(template, item, user, topic) {:push, {:text, message}, %{state | user: user}} - else - {:ok, state} end end @@ -253,7 +254,7 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do defp find_sec_websocket_protocol(sec_headers) do Enum.find_value(sec_headers, fn - {"sec-websocket-protocol", token} -> token + {"sec-websocket-protocol", protocols} -> protocols |> Plug.Conn.Utils.list() |> List.first() _ -> nil end) end diff --git a/lib/pleroma/web/mastodon_api/websocket_plug.ex b/lib/pleroma/web/mastodon_api/websocket_plug.ex new file mode 100644 index 000000000..58ee913b4 --- /dev/null +++ b/lib/pleroma/web/mastodon_api/websocket_plug.ex @@ -0,0 +1,105 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.MastodonAPI.WebsocketPlug do + @moduledoc """ + A Phoenix 1.8 compatible WebSocket transport for Mastodon streaming. + + It mirrors Phoenix.Transports.WebSocket, but echoes a successfully authenticated + Mastodon-style Sec-WebSocket-Protocol token so browser clients accept the handshake. + """ + + @behaviour Plug + + import Plug.Conn + + alias Phoenix.Socket.Transport + alias Pleroma.Web.Endpoint + alias Pleroma.Web.MastodonAPI.WebsocketHandler + + @connect_info_opts [:check_csrf] + + @impl Plug + def init(opts) do + path = String.split(Keyword.fetch!(opts, :path), "/", trim: true) + websocket = Keyword.fetch!(opts, :websocket) + config = Transport.load_config(websocket, Phoenix.Transports.WebSocket) + + {path, config} + end + + @impl Plug + def call(%{method: "GET", path_info: path} = conn, {path, opts}) do + conn + |> fetch_query_params() + |> Transport.code_reload(Endpoint, opts) + |> Transport.transport_log(opts[:transport_log]) + |> Transport.check_origin(WebsocketHandler, Endpoint, opts) + |> connect(opts) + end + + def call(%{path_info: path} = conn, {path, _opts}) do + conn + |> send_resp(400, "") + |> halt() + end + + def call(conn, _opts), do: conn + + defp connect(%{halted: true} = conn, _opts), do: conn + + defp connect(%{params: params} = conn, opts) do + keys = Keyword.get(opts, :connect_info, []) + + connect_info = + Transport.connect_info(conn, Endpoint, keys, Keyword.take(opts, @connect_info_opts)) + + config = %{ + endpoint: Endpoint, + transport: :websocket, + options: opts, + params: params, + connect_info: connect_info + } + + case WebsocketHandler.connect(config) do + {:ok, arg} -> + try do + conn + |> echo_sec_websocket_protocol() + |> WebSockAdapter.upgrade(WebsocketHandler, arg, opts) + |> halt() + rescue + e in WebSockAdapter.UpgradeError -> + conn + |> send_resp(400, e.message) + |> halt() + end + + :error -> + conn + |> send_resp(403, "") + |> halt() + + {:error, reason} -> + {m, f, args} = opts[:error_handler] + + halt(apply(m, f, [conn, reason | args])) + end + end + + defp echo_sec_websocket_protocol(conn) do + case get_req_header(conn, "sec-websocket-protocol") do + [protocols | _] -> + case Plug.Conn.Utils.list(protocols) do + [protocol | _] -> put_resp_header(conn, "sec-websocket-protocol", protocol) + nil -> conn + [] -> conn + end + + [] -> + conn + end + end +end diff --git a/test/pleroma/integration/mastodon_websocket_test.exs b/test/pleroma/integration/mastodon_websocket_test.exs index 078bb643c..de88e5002 100644 --- a/test/pleroma/integration/mastodon_websocket_test.exs +++ b/test/pleroma/integration/mastodon_websocket_test.exs @@ -11,6 +11,7 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do alias Pleroma.Integration.WebsocketClient alias Pleroma.Web.CommonAPI + alias Pleroma.Web.MastodonAPI.StatusView alias Pleroma.Web.OAuth @moduletag needs_streamer: true, capture_log: true @@ -31,6 +32,48 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do WebsocketClient.start_link(self(), path, headers) end + defp raw_websocket_handshake(qs, headers) do + uri = URI.parse(@path <> qs) + port = uri.port || 80 + path = uri.path <> if(uri.query, do: "?" <> uri.query, else: "") + + default_headers = [ + {"host", "#{uri.host}:#{port}"}, + {"upgrade", "websocket"}, + {"connection", "Upgrade"}, + {"sec-websocket-key", Base.encode64(:crypto.strong_rand_bytes(16))}, + {"sec-websocket-version", "13"} + ] + + request = [ + "GET #{path} HTTP/1.1\r\n", + Enum.map(default_headers ++ headers, fn {name, value} -> "#{name}: #{value}\r\n" end), + "\r\n" + ] + + with {:ok, socket} <- + :gen_tcp.connect(String.to_charlist(uri.host), port, [:binary, active: false], 1_000), + :ok <- :gen_tcp.send(socket, request), + {:ok, response} <- :gen_tcp.recv(socket, 0, 1_000) do + :gen_tcp.close(socket) + {:ok, parse_http_response(response)} + end + end + + defp parse_http_response(response) do + [headers | _] = String.split(response, "\r\n\r\n", parts: 2) + [status_line | header_lines] = String.split(headers, "\r\n") + [_, status | _] = String.split(status_line, " ") + + headers = + Enum.map(header_lines, fn line -> + [name, value] = String.split(line, ":", parts: 2) + {String.downcase(name), String.trim(value)} + end) + + %{status: String.to_integer(status), headers: headers} + end + defp decode_json(json) do with {:ok, %{"event" => event, "payload" => payload_text}} <- Jason.decode(json), {:ok, payload} <- Jason.decode(payload_text) do @@ -85,9 +128,7 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do assert json["payload"] assert {:ok, json} = Jason.decode(json["payload"]) - view_json = - Pleroma.Web.MastodonAPI.StatusView.render("show.json", activity: activity, for: nil) - |> atom_key_to_string() + view_json = atom_key_to_string(StatusView.render("show.json", activity: activity, for: nil)) assert json == view_json end @@ -114,10 +155,7 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do assert json["payload"] assert {:ok, json} = Jason.decode(json["payload"]) - view_json = - Pleroma.Web.MastodonAPI.StatusView.render("show.json", activity: activity, for: nil) - |> Jason.encode!() - |> Jason.decode!() + view_json = atom_key_to_string(StatusView.render("show.json", activity: activity, for: nil)) assert json == view_json end @@ -279,6 +317,34 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do end) end + test "echoes the Sec-WebSocket-Protocol token in the handshake", %{token: token} do + assert {:ok, %{status: 101, headers: headers}} = + raw_websocket_handshake("?stream=user", [ + {"sec-websocket-protocol", token.token} + ]) + + assert {"sec-websocket-protocol", token.token} in headers + end + + test "echoes the selected Sec-WebSocket-Protocol token", %{token: token} do + assert {:ok, %{status: 101, headers: headers}} = + raw_websocket_handshake("?stream=user", [ + {"sec-websocket-protocol", "#{token.token}, phoenix"} + ]) + + assert {"sec-websocket-protocol", token.token} in headers + end + + test "does not echo an invalid Sec-WebSocket-Protocol token", %{token: token} do + assert {:ok, %{status: 401, headers: headers}} = + raw_websocket_handshake("?stream=user", [ + {"sec-websocket-protocol", "invalid"} + ]) + + refute {"sec-websocket-protocol", token.token} in headers + refute List.keymember?(headers, "sec-websocket-protocol", 0) + end + test "prefers sec-websocket-protocol token over query access_token", %{ token: token, user: user @@ -450,12 +516,12 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do assert {:ok, json} = Jason.decode(json["payload"]) view_json = - Pleroma.Web.MastodonAPI.StatusView.render("show.json", - activity: activity, - for: reading_user + atom_key_to_string( + StatusView.render("show.json", + activity: activity, + for: reading_user + ) ) - |> Jason.encode!() - |> Jason.decode!() assert json == view_json end @@ -478,12 +544,12 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do activity = Pleroma.Activity.normalize(activity) view_json = - Pleroma.Web.MastodonAPI.StatusView.render("show.json", - activity: activity, - for: reading_user + atom_key_to_string( + StatusView.render("show.json", + activity: activity, + for: reading_user + ) ) - |> Jason.encode!() - |> Jason.decode!() assert {:ok, %{"event" => "status.update", "payload" => ^view_json}} = decode_json(raw_json) end From cda6738309117a8ea95f8367175dbed3587e697b Mon Sep 17 00:00:00 2001 From: hj Date: Fri, 22 May 2026 20:47:52 +0000 Subject: [PATCH 06/14] Update priv/scrubbers/default.ex --- priv/scrubbers/default.ex | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/priv/scrubbers/default.ex b/priv/scrubbers/default.ex index 8963d99e8..cc5eba027 100644 --- a/priv/scrubbers/default.ex +++ b/priv/scrubbers/default.ex @@ -105,7 +105,9 @@ defmodule Pleroma.HTML.Scrubber.Default do "mfm-sparkle", "mfm-rotate", "mfm-ruby", - "mfm-unixtime" + "mfm-unixtime", + # Exists in Akkoma but not Misskey? + "mfm-center" ]) Meta.allow_tag_with_this_attribute_values(:p, "class", ["quote-inline"]) From 94a28d128605bc4e3328293a01585471c193f82b Mon Sep 17 00:00:00 2001 From: Phantasm Date: Sat, 16 May 2026 00:31:02 +0200 Subject: [PATCH 07/14] RichMedia Backfill: Add cachex positive test --- test/pleroma/web/rich_media/backfill_test.exs | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/test/pleroma/web/rich_media/backfill_test.exs b/test/pleroma/web/rich_media/backfill_test.exs index 6d221fcf5..8b4834bf7 100644 --- a/test/pleroma/web/rich_media/backfill_test.exs +++ b/test/pleroma/web/rich_media/backfill_test.exs @@ -23,4 +23,26 @@ defmodule Pleroma.Web.RichMedia.BackfillTest do Backfill.run(%{"url" => url}) end + + test "sets a warm_cache entry" do + url = "https://good.example.com/" + url_hash = Card.url_to_hash(url) + + Tesla.Mock.mock(fn %{url: ^url} -> + {:ok, + %Tesla.Env{ + status: 200, + body: "" + }} + end) + + Pleroma.CachexMock + |> expect(:put, fn :rich_media_cache, + ^url_hash, + %Pleroma.Web.RichMedia.Card{url_hash: ^url_hash} -> + {:ok, true} + end) + + Backfill.run(%{"url" => url}) + end end From 678fe8a0642771e24d7b4d9b1df78578024351fe Mon Sep 17 00:00:00 2001 From: Phantasm Date: Wed, 20 May 2026 20:16:19 +0200 Subject: [PATCH 08/14] RichMedia: Add support for disabling wss streaming out on backfill --- .../web/mastodon_api/views/status_view.ex | 9 +-- lib/pleroma/web/rich_media/backfill.ex | 14 +++-- lib/pleroma/web/rich_media/card.ex | 16 ++++-- test/pleroma/web/rich_media/backfill_test.exs | 56 ++++++++++++++++++- 4 files changed, 79 insertions(+), 16 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 09af60865..80d53adff 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -28,9 +28,9 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do # This is a naive way to do this, just spawning a process per activity # to fetch the preview. However it should be fine considering # pagination is restricted to 40 activities at a time - defp fetch_rich_media_for_activities(activities) do + defp fetch_rich_media_for_activities(activities, opts) do Enum.each(activities, fn activity -> - Card.get_by_activity(activity) + Card.get_by_activity(activity, opts) end) end @@ -113,7 +113,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do activities = Enum.filter(opts.activities, & &1) # Start prefetching rich media before doing anything else - fetch_rich_media_for_activities(activities) + fetch_rich_media_for_activities(activities, opts) + replied_to_activities = get_replied_to_activities(activities) quoted_activities = get_quoted_activities(activities) @@ -362,7 +363,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do summary = object.data["summary"] || "" card = - case Card.get_by_activity(activity) do + case Card.get_by_activity(activity, opts) do %Card{} = result -> render("card.json", result) _ -> nil end diff --git a/lib/pleroma/web/rich_media/backfill.ex b/lib/pleroma/web/rich_media/backfill.ex index 1cd90629f..607c5238a 100644 --- a/lib/pleroma/web/rich_media/backfill.ex +++ b/lib/pleroma/web/rich_media/backfill.ex @@ -11,6 +11,8 @@ defmodule Pleroma.Web.RichMedia.Backfill do require Logger + @callback run(map()) :: :ok | Parser.parse_errors() | Helpers.get_errors() + @cachex Pleroma.Config.get([:cachex, :provider], Cachex) @stream_out_impl Pleroma.Config.get( [__MODULE__, :stream_out], @@ -26,11 +28,7 @@ defmodule Pleroma.Web.RichMedia.Backfill do {:ok, card} = Card.create(url, fields) maybe_schedule_expiration(url, fields) - - with %{"activity_id" => activity_id} <- args, - false <- is_nil(activity_id) do - stream_update(args) - end + maybe_update_stream(args) warm_cache(url_hash, card) :ok @@ -55,12 +53,16 @@ defmodule Pleroma.Web.RichMedia.Backfill do end end - defp stream_update(%{"activity_id" => activity_id}) do + defp maybe_update_stream(%{"activity_id" => activity_id, "stream" => true}) when is_binary(activity_id) do Pleroma.Activity.get_by_id(activity_id) |> Pleroma.Activity.normalize() |> @stream_out_impl.stream_out() end + # Streamer.stream_out returns noop when unsupported activity type is requested to be streamed. + # Do the same here for unwanted streaming + defp maybe_update_stream(_), do: :noop + defp warm_cache(key, val), do: @cachex.put(:rich_media_cache, key, val) defp negative_cache(key, ttl \\ :timer.minutes(15)), diff --git a/lib/pleroma/web/rich_media/card.ex b/lib/pleroma/web/rich_media/card.ex index 6b4bb9555..86a4f577a 100644 --- a/lib/pleroma/web/rich_media/card.ex +++ b/lib/pleroma/web/rich_media/card.ex @@ -90,8 +90,12 @@ defmodule Pleroma.Web.RichMedia.Card do nil -> activity_id = Keyword.get(opts, :activity_id, nil) + # Nested opts, first layer comes from get_by_activity/2 as Keyword, second from API views/Federation as Map. + # Provide default Map when called directly. + opts = Keyword.get(opts, :opts, %{}) + stream = Map.get(opts, :stream, true) - RichMediaWorker.new(%{"op" => "backfill", "url" => url, "activity_id" => activity_id}) + RichMediaWorker.new(%{"op" => "backfill", "url" => url, "activity_id" => activity_id, "stream" => stream}) |> Oban.insert() nil @@ -112,9 +116,11 @@ defmodule Pleroma.Web.RichMedia.Card do end end - @spec get_by_activity(Activity.t()) :: t() | nil | :error + @spec get_by_activity(Activity.t(), %{}) :: t() | nil | :error + def get_by_activity(activity, opts \\ %{}) + # Fake/Draft activity - def get_by_activity(%Activity{id: "pleroma:fakeid"} = activity) do + def get_by_activity(%Activity{id: "pleroma:fakeid"} = activity, _opts) do with {_, true} <- {:config, @config_impl.get([:rich_media, :enabled])}, %Object{} = object <- Object.normalize(activity, fetch: false), url when not is_nil(url) <- HTML.extract_first_external_url_from_object(object) do @@ -138,13 +144,13 @@ defmodule Pleroma.Web.RichMedia.Card do end end - def get_by_activity(activity) do + def get_by_activity(activity, opts) do with %Object{} = object <- Object.normalize(activity, fetch: false), {_, nil} <- {:cached, get_cached_url(object, activity.id)} do nil else {:cached, url} -> - get_or_backfill_by_url(url, activity_id: activity.id) + get_or_backfill_by_url(url, activity_id: activity.id, opts: opts) _ -> :error diff --git a/test/pleroma/web/rich_media/backfill_test.exs b/test/pleroma/web/rich_media/backfill_test.exs index 8b4834bf7..f7e74c02a 100644 --- a/test/pleroma/web/rich_media/backfill_test.exs +++ b/test/pleroma/web/rich_media/backfill_test.exs @@ -5,12 +5,20 @@ defmodule Pleroma.Web.RichMedia.BackfillTest do use Pleroma.DataCase + alias Pleroma.Web.CommonAPI alias Pleroma.Web.RichMedia.Backfill alias Pleroma.Web.RichMedia.Card import Mox + import Pleroma.Factory - setup_all do: clear_config([:rich_media, :enabled], true) + setup do + clear_config([:rich_media, :enabled], true) + + Mox.stub_with(Pleroma.CachexMock, Pleroma.NullCache) + + :ok + end test "sets a negative cache entry for an error" do url = "https://bad.example.com/" @@ -45,4 +53,50 @@ defmodule Pleroma.Web.RichMedia.BackfillTest do Backfill.run(%{"url" => url}) end + + test "streams out update when stream == true" do + url = "https://example.com" + user = insert(:user) + + Tesla.Mock.mock(fn %{url: ^url} -> + {:ok, + %Tesla.Env{ + status: 200, + body: "" + }} + end) + + {:ok, activity} = CommonAPI.post(user, %{status: "#cofe #{url}"}) + + Pleroma.CachexMock + |> expect(:put, fn :rich_media_cache, _, _ -> {:ok, true} end) + + Pleroma.Web.ActivityPub.ActivityPubMock + |> expect(:stream_out, fn _ -> :ok end) + + Backfill.run(%{"url" => url, "activity_id" => "#{activity.data["id"]}", "stream" => true}) + end + + test "does not stream out update when stream == false" do + url = "https://example.com" + user = insert(:user) + + Tesla.Mock.mock(fn %{url: ^url} -> + {:ok, + %Tesla.Env{ + status: 200, + body: "" + }} + end) + + {:ok, activity} = CommonAPI.post(user, %{status: "#cofe #{url}"}) + + Pleroma.CachexMock + |> expect(:put, fn :rich_media_cache, _, _ -> {:ok, true} end) + + Pleroma.Web.ActivityPub.ActivityPubMock + |> deny(:stream_out, 1) + + Backfill.run(%{"url" => url, "activity_id" => "#{activity.data["id"]}", "stream" => false}) + end end From 5f55c9653c64ced6e1461224a942a303e82627d8 Mon Sep 17 00:00:00 2001 From: Phantasm Date: Wed, 20 May 2026 20:18:47 +0200 Subject: [PATCH 09/14] RichMedia: Disable websockets backfill streaming in StatusView --- lib/pleroma/web/mastodon_api/views/status_view.ex | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 80d53adff..cc807f603 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -28,7 +28,10 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do # This is a naive way to do this, just spawning a process per activity # to fetch the preview. However it should be fine considering # pagination is restricted to 40 activities at a time + # Force disable Websockets streaming for backfill jobs, + # otherwise old posts can show up on timelines. defp fetch_rich_media_for_activities(activities, opts) do + opts = Map.put(opts, :stream, false) Enum.each(activities, fn activity -> Card.get_by_activity(activity, opts) end) @@ -362,8 +365,10 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do summary = object.data["summary"] || "" + # Force disable Websockets streaming for backfill jobs which the below call will create, + # otherwise old posts can show up on timelines. card = - case Card.get_by_activity(activity, opts) do + case Card.get_by_activity(activity, Map.put(opts, :stream, false)) do %Card{} = result -> render("card.json", result) _ -> nil end From ff7927e219f84ada9235353457f2c2497d336964 Mon Sep 17 00:00:00 2001 From: Phantasm Date: Fri, 22 May 2026 14:53:28 +0200 Subject: [PATCH 10/14] changelog --- changelog.d/wss-necroposts.fix | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/wss-necroposts.fix diff --git a/changelog.d/wss-necroposts.fix b/changelog.d/wss-necroposts.fix new file mode 100644 index 000000000..743f9087c --- /dev/null +++ b/changelog.d/wss-necroposts.fix @@ -0,0 +1 @@ +RichMedia: Fix backfill causing old posts to show up on timelines by disabling it in MastoAPI StatusView From 6ee40cb2ebe7a6666bedea3773710a3b555791ad Mon Sep 17 00:00:00 2001 From: Phantasm Date: Sat, 23 May 2026 19:43:44 +0200 Subject: [PATCH 11/14] RichMedia: Add StatusView backfill streaming tests --- test/pleroma/web/rich_media/backfill_test.exs | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/test/pleroma/web/rich_media/backfill_test.exs b/test/pleroma/web/rich_media/backfill_test.exs index f7e74c02a..a4b2d34fd 100644 --- a/test/pleroma/web/rich_media/backfill_test.exs +++ b/test/pleroma/web/rich_media/backfill_test.exs @@ -8,6 +8,7 @@ defmodule Pleroma.Web.RichMedia.BackfillTest do alias Pleroma.Web.CommonAPI alias Pleroma.Web.RichMedia.Backfill alias Pleroma.Web.RichMedia.Card + alias Pleroma.Tests.ObanHelpers import Mox import Pleroma.Factory @@ -15,6 +16,7 @@ defmodule Pleroma.Web.RichMedia.BackfillTest do setup do clear_config([:rich_media, :enabled], true) + Mox.stub_with(Pleroma.UnstubbedConfigMock, Pleroma.Test.StaticConfig) Mox.stub_with(Pleroma.CachexMock, Pleroma.NullCache) :ok @@ -99,4 +101,64 @@ defmodule Pleroma.Web.RichMedia.BackfillTest do Backfill.run(%{"url" => url, "activity_id" => "#{activity.data["id"]}", "stream" => false}) end + + # NOTE: Below two MastoAPI tests cover almost the same code paths. + # index.json will always prefetch rich media, while show.json will try to get the card and + # fetch it when it isn't cached (both use Card.get_by_activity in the end). + # So if index.json doesn't fetch the rich media, show.json will when it renders the post, + # hence why index.json test will only call ActivityPub.stream_out twice, + # if streaming is re-enabled for in both. + test "does not stream out in MastoAPI StatusView index" do + url = "https://example.com" + user = insert(:user) + + Tesla.Mock.mock(fn %{url: ^url} -> + {:ok, + %Tesla.Env{ + status: 200, + body: "" + }} + end) + + # CommonAPI federation processing will stream out once as a new post + Pleroma.Web.ActivityPub.ActivityPubMock + |> expect(:stream_out, 1, fn _ -> :ok end) + + {:ok, activity} = CommonAPI.post(user, %{status: "#cofe #{url}"}) + ObanHelpers.perform_all() + + # Clear cache to force backfill below + Pleroma.Activity.HTML.invalidate_cache_for(activity.id) + Pleroma.Web.RichMedia.Card.delete(url) + + Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{activities: [activity], as: :activity}) + ObanHelpers.perform_all() + end + + test "does not stream out in MastoAPI StatusView show" do + url = "https://example.com" + user = insert(:user) + + Tesla.Mock.mock(fn %{url: ^url} -> + {:ok, + %Tesla.Env{ + status: 200, + body: "" + }} + end) + + # CommonAPI federation processing will stream out once as a new post + Pleroma.Web.ActivityPub.ActivityPubMock + |> expect(:stream_out, 1, fn _ -> :ok end) + + {:ok, activity} = CommonAPI.post(user, %{status: "#cofe #{url}"}) + ObanHelpers.perform_all() + + # Clear cache to force backfill below + Pleroma.Activity.HTML.invalidate_cache_for(activity.id) + Pleroma.Web.RichMedia.Card.delete(url) + + Pleroma.Web.MastodonAPI.StatusView.render("show.json", activity: activity) + ObanHelpers.perform_all() + end end From 0c0d6873306244ad98273a6ae661a4414cee875b Mon Sep 17 00:00:00 2001 From: Phantasm Date: Sat, 23 May 2026 19:49:22 +0200 Subject: [PATCH 12/14] credo, lint --- lib/pleroma/web/mastodon_api/views/status_view.ex | 1 + lib/pleroma/web/rich_media/backfill.ex | 3 ++- lib/pleroma/web/rich_media/card.ex | 11 +++++++++-- test/pleroma/web/rich_media/backfill_test.exs | 10 +++++++--- 4 files changed, 19 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index cc807f603..6949c5f2d 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -32,6 +32,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do # otherwise old posts can show up on timelines. defp fetch_rich_media_for_activities(activities, opts) do opts = Map.put(opts, :stream, false) + Enum.each(activities, fn activity -> Card.get_by_activity(activity, opts) end) diff --git a/lib/pleroma/web/rich_media/backfill.ex b/lib/pleroma/web/rich_media/backfill.ex index 607c5238a..a100b39ab 100644 --- a/lib/pleroma/web/rich_media/backfill.ex +++ b/lib/pleroma/web/rich_media/backfill.ex @@ -53,7 +53,8 @@ defmodule Pleroma.Web.RichMedia.Backfill do end end - defp maybe_update_stream(%{"activity_id" => activity_id, "stream" => true}) when is_binary(activity_id) do + defp maybe_update_stream(%{"activity_id" => activity_id, "stream" => true}) + when is_binary(activity_id) do Pleroma.Activity.get_by_id(activity_id) |> Pleroma.Activity.normalize() |> @stream_out_impl.stream_out() diff --git a/lib/pleroma/web/rich_media/card.ex b/lib/pleroma/web/rich_media/card.ex index 86a4f577a..1e9e66ec1 100644 --- a/lib/pleroma/web/rich_media/card.ex +++ b/lib/pleroma/web/rich_media/card.ex @@ -90,12 +90,19 @@ defmodule Pleroma.Web.RichMedia.Card do nil -> activity_id = Keyword.get(opts, :activity_id, nil) - # Nested opts, first layer comes from get_by_activity/2 as Keyword, second from API views/Federation as Map. + + # Nested opts, first layer comes from get_by_activity/2 as Keyword, + # second from API views/Federation as Map. # Provide default Map when called directly. opts = Keyword.get(opts, :opts, %{}) stream = Map.get(opts, :stream, true) - RichMediaWorker.new(%{"op" => "backfill", "url" => url, "activity_id" => activity_id, "stream" => stream}) + RichMediaWorker.new(%{ + "op" => "backfill", + "url" => url, + "activity_id" => activity_id, + "stream" => stream + }) |> Oban.insert() nil diff --git a/test/pleroma/web/rich_media/backfill_test.exs b/test/pleroma/web/rich_media/backfill_test.exs index a4b2d34fd..a1949533a 100644 --- a/test/pleroma/web/rich_media/backfill_test.exs +++ b/test/pleroma/web/rich_media/backfill_test.exs @@ -5,15 +5,15 @@ defmodule Pleroma.Web.RichMedia.BackfillTest do use Pleroma.DataCase + alias Pleroma.Tests.ObanHelpers alias Pleroma.Web.CommonAPI alias Pleroma.Web.RichMedia.Backfill alias Pleroma.Web.RichMedia.Card - alias Pleroma.Tests.ObanHelpers import Mox import Pleroma.Factory - setup do + setup do clear_config([:rich_media, :enabled], true) Mox.stub_with(Pleroma.UnstubbedConfigMock, Pleroma.Test.StaticConfig) @@ -131,7 +131,11 @@ defmodule Pleroma.Web.RichMedia.BackfillTest do Pleroma.Activity.HTML.invalidate_cache_for(activity.id) Pleroma.Web.RichMedia.Card.delete(url) - Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{activities: [activity], as: :activity}) + Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{ + activities: [activity], + as: :activity + }) + ObanHelpers.perform_all() end From 4c9fc6287e547d908ee0723996fc978653d4652d Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Mon, 25 May 2026 09:26:07 +0400 Subject: [PATCH 13/14] Remove unreachable WebSocket protocol match --- lib/pleroma/web/mastodon_api/websocket_plug.ex | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/pleroma/web/mastodon_api/websocket_plug.ex b/lib/pleroma/web/mastodon_api/websocket_plug.ex index 58ee913b4..c5fb5b748 100644 --- a/lib/pleroma/web/mastodon_api/websocket_plug.ex +++ b/lib/pleroma/web/mastodon_api/websocket_plug.ex @@ -94,7 +94,6 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketPlug do [protocols | _] -> case Plug.Conn.Utils.list(protocols) do [protocol | _] -> put_resp_header(conn, "sec-websocket-protocol", protocol) - nil -> conn [] -> conn end From cdb0f103a8f7f141185e2bff6d686b047f368fa2 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Mon, 25 May 2026 14:08:07 +0400 Subject: [PATCH 14/14] Tighten rich media backfill stream test --- test/pleroma/web/rich_media/backfill_test.exs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test/pleroma/web/rich_media/backfill_test.exs b/test/pleroma/web/rich_media/backfill_test.exs index a1949533a..071f9b48a 100644 --- a/test/pleroma/web/rich_media/backfill_test.exs +++ b/test/pleroma/web/rich_media/backfill_test.exs @@ -74,9 +74,12 @@ defmodule Pleroma.Web.RichMedia.BackfillTest do |> expect(:put, fn :rich_media_cache, _, _ -> {:ok, true} end) Pleroma.Web.ActivityPub.ActivityPubMock - |> expect(:stream_out, fn _ -> :ok end) + |> expect(:stream_out, fn %Pleroma.Activity{id: id} -> + assert id == activity.id + :ok + end) - Backfill.run(%{"url" => url, "activity_id" => "#{activity.data["id"]}", "stream" => true}) + Backfill.run(%{"url" => url, "activity_id" => activity.id, "stream" => true}) end test "does not stream out update when stream == false" do