From 94a28d128605bc4e3328293a01585471c193f82b Mon Sep 17 00:00:00 2001 From: Phantasm Date: Sat, 16 May 2026 00:31:02 +0200 Subject: [PATCH 1/7] RichMedia Backfill: Add cachex positive test --- test/pleroma/web/rich_media/backfill_test.exs | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/test/pleroma/web/rich_media/backfill_test.exs b/test/pleroma/web/rich_media/backfill_test.exs index 6d221fcf5..8b4834bf7 100644 --- a/test/pleroma/web/rich_media/backfill_test.exs +++ b/test/pleroma/web/rich_media/backfill_test.exs @@ -23,4 +23,26 @@ defmodule Pleroma.Web.RichMedia.BackfillTest do Backfill.run(%{"url" => url}) end + + test "sets a warm_cache entry" do + url = "https://good.example.com/" + url_hash = Card.url_to_hash(url) + + Tesla.Mock.mock(fn %{url: ^url} -> + {:ok, + %Tesla.Env{ + status: 200, + body: "" + }} + end) + + Pleroma.CachexMock + |> expect(:put, fn :rich_media_cache, + ^url_hash, + %Pleroma.Web.RichMedia.Card{url_hash: ^url_hash} -> + {:ok, true} + end) + + Backfill.run(%{"url" => url}) + end end From 678fe8a0642771e24d7b4d9b1df78578024351fe Mon Sep 17 00:00:00 2001 From: Phantasm Date: Wed, 20 May 2026 20:16:19 +0200 Subject: [PATCH 2/7] RichMedia: Add support for disabling wss streaming out on backfill --- .../web/mastodon_api/views/status_view.ex | 9 +-- lib/pleroma/web/rich_media/backfill.ex | 14 +++-- lib/pleroma/web/rich_media/card.ex | 16 ++++-- test/pleroma/web/rich_media/backfill_test.exs | 56 ++++++++++++++++++- 4 files changed, 79 insertions(+), 16 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 09af60865..80d53adff 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -28,9 +28,9 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do # This is a naive way to do this, just spawning a process per activity # to fetch the preview. However it should be fine considering # pagination is restricted to 40 activities at a time - defp fetch_rich_media_for_activities(activities) do + defp fetch_rich_media_for_activities(activities, opts) do Enum.each(activities, fn activity -> - Card.get_by_activity(activity) + Card.get_by_activity(activity, opts) end) end @@ -113,7 +113,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do activities = Enum.filter(opts.activities, & &1) # Start prefetching rich media before doing anything else - fetch_rich_media_for_activities(activities) + fetch_rich_media_for_activities(activities, opts) + replied_to_activities = get_replied_to_activities(activities) quoted_activities = get_quoted_activities(activities) @@ -362,7 +363,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do summary = object.data["summary"] || "" card = - case Card.get_by_activity(activity) do + case Card.get_by_activity(activity, opts) do %Card{} = result -> render("card.json", result) _ -> nil end diff --git a/lib/pleroma/web/rich_media/backfill.ex b/lib/pleroma/web/rich_media/backfill.ex index 1cd90629f..607c5238a 100644 --- a/lib/pleroma/web/rich_media/backfill.ex +++ b/lib/pleroma/web/rich_media/backfill.ex @@ -11,6 +11,8 @@ defmodule Pleroma.Web.RichMedia.Backfill do require Logger + @callback run(map()) :: :ok | Parser.parse_errors() | Helpers.get_errors() + @cachex Pleroma.Config.get([:cachex, :provider], Cachex) @stream_out_impl Pleroma.Config.get( [__MODULE__, :stream_out], @@ -26,11 +28,7 @@ defmodule Pleroma.Web.RichMedia.Backfill do {:ok, card} = Card.create(url, fields) maybe_schedule_expiration(url, fields) - - with %{"activity_id" => activity_id} <- args, - false <- is_nil(activity_id) do - stream_update(args) - end + maybe_update_stream(args) warm_cache(url_hash, card) :ok @@ -55,12 +53,16 @@ defmodule Pleroma.Web.RichMedia.Backfill do end end - defp stream_update(%{"activity_id" => activity_id}) do + defp maybe_update_stream(%{"activity_id" => activity_id, "stream" => true}) when is_binary(activity_id) do Pleroma.Activity.get_by_id(activity_id) |> Pleroma.Activity.normalize() |> @stream_out_impl.stream_out() end + # Streamer.stream_out returns noop when unsupported activity type is requested to be streamed. + # Do the same here for unwanted streaming + defp maybe_update_stream(_), do: :noop + defp warm_cache(key, val), do: @cachex.put(:rich_media_cache, key, val) defp negative_cache(key, ttl \\ :timer.minutes(15)), diff --git a/lib/pleroma/web/rich_media/card.ex b/lib/pleroma/web/rich_media/card.ex index 6b4bb9555..86a4f577a 100644 --- a/lib/pleroma/web/rich_media/card.ex +++ b/lib/pleroma/web/rich_media/card.ex @@ -90,8 +90,12 @@ defmodule Pleroma.Web.RichMedia.Card do nil -> activity_id = Keyword.get(opts, :activity_id, nil) + # Nested opts, first layer comes from get_by_activity/2 as Keyword, second from API views/Federation as Map. + # Provide default Map when called directly. + opts = Keyword.get(opts, :opts, %{}) + stream = Map.get(opts, :stream, true) - RichMediaWorker.new(%{"op" => "backfill", "url" => url, "activity_id" => activity_id}) + RichMediaWorker.new(%{"op" => "backfill", "url" => url, "activity_id" => activity_id, "stream" => stream}) |> Oban.insert() nil @@ -112,9 +116,11 @@ defmodule Pleroma.Web.RichMedia.Card do end end - @spec get_by_activity(Activity.t()) :: t() | nil | :error + @spec get_by_activity(Activity.t(), %{}) :: t() | nil | :error + def get_by_activity(activity, opts \\ %{}) + # Fake/Draft activity - def get_by_activity(%Activity{id: "pleroma:fakeid"} = activity) do + def get_by_activity(%Activity{id: "pleroma:fakeid"} = activity, _opts) do with {_, true} <- {:config, @config_impl.get([:rich_media, :enabled])}, %Object{} = object <- Object.normalize(activity, fetch: false), url when not is_nil(url) <- HTML.extract_first_external_url_from_object(object) do @@ -138,13 +144,13 @@ defmodule Pleroma.Web.RichMedia.Card do end end - def get_by_activity(activity) do + def get_by_activity(activity, opts) do with %Object{} = object <- Object.normalize(activity, fetch: false), {_, nil} <- {:cached, get_cached_url(object, activity.id)} do nil else {:cached, url} -> - get_or_backfill_by_url(url, activity_id: activity.id) + get_or_backfill_by_url(url, activity_id: activity.id, opts: opts) _ -> :error diff --git a/test/pleroma/web/rich_media/backfill_test.exs b/test/pleroma/web/rich_media/backfill_test.exs index 8b4834bf7..f7e74c02a 100644 --- a/test/pleroma/web/rich_media/backfill_test.exs +++ b/test/pleroma/web/rich_media/backfill_test.exs @@ -5,12 +5,20 @@ defmodule Pleroma.Web.RichMedia.BackfillTest do use Pleroma.DataCase + alias Pleroma.Web.CommonAPI alias Pleroma.Web.RichMedia.Backfill alias Pleroma.Web.RichMedia.Card import Mox + import Pleroma.Factory - setup_all do: clear_config([:rich_media, :enabled], true) + setup do + clear_config([:rich_media, :enabled], true) + + Mox.stub_with(Pleroma.CachexMock, Pleroma.NullCache) + + :ok + end test "sets a negative cache entry for an error" do url = "https://bad.example.com/" @@ -45,4 +53,50 @@ defmodule Pleroma.Web.RichMedia.BackfillTest do Backfill.run(%{"url" => url}) end + + test "streams out update when stream == true" do + url = "https://example.com" + user = insert(:user) + + Tesla.Mock.mock(fn %{url: ^url} -> + {:ok, + %Tesla.Env{ + status: 200, + body: "" + }} + end) + + {:ok, activity} = CommonAPI.post(user, %{status: "#cofe #{url}"}) + + Pleroma.CachexMock + |> expect(:put, fn :rich_media_cache, _, _ -> {:ok, true} end) + + Pleroma.Web.ActivityPub.ActivityPubMock + |> expect(:stream_out, fn _ -> :ok end) + + Backfill.run(%{"url" => url, "activity_id" => "#{activity.data["id"]}", "stream" => true}) + end + + test "does not stream out update when stream == false" do + url = "https://example.com" + user = insert(:user) + + Tesla.Mock.mock(fn %{url: ^url} -> + {:ok, + %Tesla.Env{ + status: 200, + body: "" + }} + end) + + {:ok, activity} = CommonAPI.post(user, %{status: "#cofe #{url}"}) + + Pleroma.CachexMock + |> expect(:put, fn :rich_media_cache, _, _ -> {:ok, true} end) + + Pleroma.Web.ActivityPub.ActivityPubMock + |> deny(:stream_out, 1) + + Backfill.run(%{"url" => url, "activity_id" => "#{activity.data["id"]}", "stream" => false}) + end end From 5f55c9653c64ced6e1461224a942a303e82627d8 Mon Sep 17 00:00:00 2001 From: Phantasm Date: Wed, 20 May 2026 20:18:47 +0200 Subject: [PATCH 3/7] RichMedia: Disable websockets backfill streaming in StatusView --- lib/pleroma/web/mastodon_api/views/status_view.ex | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 80d53adff..cc807f603 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -28,7 +28,10 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do # This is a naive way to do this, just spawning a process per activity # to fetch the preview. However it should be fine considering # pagination is restricted to 40 activities at a time + # Force disable Websockets streaming for backfill jobs, + # otherwise old posts can show up on timelines. defp fetch_rich_media_for_activities(activities, opts) do + opts = Map.put(opts, :stream, false) Enum.each(activities, fn activity -> Card.get_by_activity(activity, opts) end) @@ -362,8 +365,10 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do summary = object.data["summary"] || "" + # Force disable Websockets streaming for backfill jobs which the below call will create, + # otherwise old posts can show up on timelines. card = - case Card.get_by_activity(activity, opts) do + case Card.get_by_activity(activity, Map.put(opts, :stream, false)) do %Card{} = result -> render("card.json", result) _ -> nil end From ff7927e219f84ada9235353457f2c2497d336964 Mon Sep 17 00:00:00 2001 From: Phantasm Date: Fri, 22 May 2026 14:53:28 +0200 Subject: [PATCH 4/7] changelog --- changelog.d/wss-necroposts.fix | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/wss-necroposts.fix diff --git a/changelog.d/wss-necroposts.fix b/changelog.d/wss-necroposts.fix new file mode 100644 index 000000000..743f9087c --- /dev/null +++ b/changelog.d/wss-necroposts.fix @@ -0,0 +1 @@ +RichMedia: Fix backfill causing old posts to show up on timelines by disabling it in MastoAPI StatusView From 6ee40cb2ebe7a6666bedea3773710a3b555791ad Mon Sep 17 00:00:00 2001 From: Phantasm Date: Sat, 23 May 2026 19:43:44 +0200 Subject: [PATCH 5/7] RichMedia: Add StatusView backfill streaming tests --- test/pleroma/web/rich_media/backfill_test.exs | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/test/pleroma/web/rich_media/backfill_test.exs b/test/pleroma/web/rich_media/backfill_test.exs index f7e74c02a..a4b2d34fd 100644 --- a/test/pleroma/web/rich_media/backfill_test.exs +++ b/test/pleroma/web/rich_media/backfill_test.exs @@ -8,6 +8,7 @@ defmodule Pleroma.Web.RichMedia.BackfillTest do alias Pleroma.Web.CommonAPI alias Pleroma.Web.RichMedia.Backfill alias Pleroma.Web.RichMedia.Card + alias Pleroma.Tests.ObanHelpers import Mox import Pleroma.Factory @@ -15,6 +16,7 @@ defmodule Pleroma.Web.RichMedia.BackfillTest do setup do clear_config([:rich_media, :enabled], true) + Mox.stub_with(Pleroma.UnstubbedConfigMock, Pleroma.Test.StaticConfig) Mox.stub_with(Pleroma.CachexMock, Pleroma.NullCache) :ok @@ -99,4 +101,64 @@ defmodule Pleroma.Web.RichMedia.BackfillTest do Backfill.run(%{"url" => url, "activity_id" => "#{activity.data["id"]}", "stream" => false}) end + + # NOTE: Below two MastoAPI tests cover almost the same code paths. + # index.json will always prefetch rich media, while show.json will try to get the card and + # fetch it when it isn't cached (both use Card.get_by_activity in the end). + # So if index.json doesn't fetch the rich media, show.json will when it renders the post, + # hence why index.json test will only call ActivityPub.stream_out twice, + # if streaming is re-enabled for in both. + test "does not stream out in MastoAPI StatusView index" do + url = "https://example.com" + user = insert(:user) + + Tesla.Mock.mock(fn %{url: ^url} -> + {:ok, + %Tesla.Env{ + status: 200, + body: "" + }} + end) + + # CommonAPI federation processing will stream out once as a new post + Pleroma.Web.ActivityPub.ActivityPubMock + |> expect(:stream_out, 1, fn _ -> :ok end) + + {:ok, activity} = CommonAPI.post(user, %{status: "#cofe #{url}"}) + ObanHelpers.perform_all() + + # Clear cache to force backfill below + Pleroma.Activity.HTML.invalidate_cache_for(activity.id) + Pleroma.Web.RichMedia.Card.delete(url) + + Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{activities: [activity], as: :activity}) + ObanHelpers.perform_all() + end + + test "does not stream out in MastoAPI StatusView show" do + url = "https://example.com" + user = insert(:user) + + Tesla.Mock.mock(fn %{url: ^url} -> + {:ok, + %Tesla.Env{ + status: 200, + body: "" + }} + end) + + # CommonAPI federation processing will stream out once as a new post + Pleroma.Web.ActivityPub.ActivityPubMock + |> expect(:stream_out, 1, fn _ -> :ok end) + + {:ok, activity} = CommonAPI.post(user, %{status: "#cofe #{url}"}) + ObanHelpers.perform_all() + + # Clear cache to force backfill below + Pleroma.Activity.HTML.invalidate_cache_for(activity.id) + Pleroma.Web.RichMedia.Card.delete(url) + + Pleroma.Web.MastodonAPI.StatusView.render("show.json", activity: activity) + ObanHelpers.perform_all() + end end From 0c0d6873306244ad98273a6ae661a4414cee875b Mon Sep 17 00:00:00 2001 From: Phantasm Date: Sat, 23 May 2026 19:49:22 +0200 Subject: [PATCH 6/7] credo, lint --- lib/pleroma/web/mastodon_api/views/status_view.ex | 1 + lib/pleroma/web/rich_media/backfill.ex | 3 ++- lib/pleroma/web/rich_media/card.ex | 11 +++++++++-- test/pleroma/web/rich_media/backfill_test.exs | 10 +++++++--- 4 files changed, 19 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index cc807f603..6949c5f2d 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -32,6 +32,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do # otherwise old posts can show up on timelines. defp fetch_rich_media_for_activities(activities, opts) do opts = Map.put(opts, :stream, false) + Enum.each(activities, fn activity -> Card.get_by_activity(activity, opts) end) diff --git a/lib/pleroma/web/rich_media/backfill.ex b/lib/pleroma/web/rich_media/backfill.ex index 607c5238a..a100b39ab 100644 --- a/lib/pleroma/web/rich_media/backfill.ex +++ b/lib/pleroma/web/rich_media/backfill.ex @@ -53,7 +53,8 @@ defmodule Pleroma.Web.RichMedia.Backfill do end end - defp maybe_update_stream(%{"activity_id" => activity_id, "stream" => true}) when is_binary(activity_id) do + defp maybe_update_stream(%{"activity_id" => activity_id, "stream" => true}) + when is_binary(activity_id) do Pleroma.Activity.get_by_id(activity_id) |> Pleroma.Activity.normalize() |> @stream_out_impl.stream_out() diff --git a/lib/pleroma/web/rich_media/card.ex b/lib/pleroma/web/rich_media/card.ex index 86a4f577a..1e9e66ec1 100644 --- a/lib/pleroma/web/rich_media/card.ex +++ b/lib/pleroma/web/rich_media/card.ex @@ -90,12 +90,19 @@ defmodule Pleroma.Web.RichMedia.Card do nil -> activity_id = Keyword.get(opts, :activity_id, nil) - # Nested opts, first layer comes from get_by_activity/2 as Keyword, second from API views/Federation as Map. + + # Nested opts, first layer comes from get_by_activity/2 as Keyword, + # second from API views/Federation as Map. # Provide default Map when called directly. opts = Keyword.get(opts, :opts, %{}) stream = Map.get(opts, :stream, true) - RichMediaWorker.new(%{"op" => "backfill", "url" => url, "activity_id" => activity_id, "stream" => stream}) + RichMediaWorker.new(%{ + "op" => "backfill", + "url" => url, + "activity_id" => activity_id, + "stream" => stream + }) |> Oban.insert() nil diff --git a/test/pleroma/web/rich_media/backfill_test.exs b/test/pleroma/web/rich_media/backfill_test.exs index a4b2d34fd..a1949533a 100644 --- a/test/pleroma/web/rich_media/backfill_test.exs +++ b/test/pleroma/web/rich_media/backfill_test.exs @@ -5,15 +5,15 @@ defmodule Pleroma.Web.RichMedia.BackfillTest do use Pleroma.DataCase + alias Pleroma.Tests.ObanHelpers alias Pleroma.Web.CommonAPI alias Pleroma.Web.RichMedia.Backfill alias Pleroma.Web.RichMedia.Card - alias Pleroma.Tests.ObanHelpers import Mox import Pleroma.Factory - setup do + setup do clear_config([:rich_media, :enabled], true) Mox.stub_with(Pleroma.UnstubbedConfigMock, Pleroma.Test.StaticConfig) @@ -131,7 +131,11 @@ defmodule Pleroma.Web.RichMedia.BackfillTest do Pleroma.Activity.HTML.invalidate_cache_for(activity.id) Pleroma.Web.RichMedia.Card.delete(url) - Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{activities: [activity], as: :activity}) + Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{ + activities: [activity], + as: :activity + }) + ObanHelpers.perform_all() end From cdb0f103a8f7f141185e2bff6d686b047f368fa2 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Mon, 25 May 2026 14:08:07 +0400 Subject: [PATCH 7/7] Tighten rich media backfill stream test --- test/pleroma/web/rich_media/backfill_test.exs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test/pleroma/web/rich_media/backfill_test.exs b/test/pleroma/web/rich_media/backfill_test.exs index a1949533a..071f9b48a 100644 --- a/test/pleroma/web/rich_media/backfill_test.exs +++ b/test/pleroma/web/rich_media/backfill_test.exs @@ -74,9 +74,12 @@ defmodule Pleroma.Web.RichMedia.BackfillTest do |> expect(:put, fn :rich_media_cache, _, _ -> {:ok, true} end) Pleroma.Web.ActivityPub.ActivityPubMock - |> expect(:stream_out, fn _ -> :ok end) + |> expect(:stream_out, fn %Pleroma.Activity{id: id} -> + assert id == activity.id + :ok + end) - Backfill.run(%{"url" => url, "activity_id" => "#{activity.data["id"]}", "stream" => true}) + Backfill.run(%{"url" => url, "activity_id" => activity.id, "stream" => true}) end test "does not stream out update when stream == false" do