hj/pleroma
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

feat(retention): walk activities incrementally with a persisted cursor

Browse source 
Grouping the whole activities table by thread every run is unusable on
a large instance: on a 312 GB database it plans as a seq scan plus a
50M-row on-disk sort. Instead, walk activities in id order (flake ids
are time-ordered) from a persisted cursor up to the id that corresponds
to the deadline, and verify only the threads those activities belong to
through the (type, context) index. Measured on that instance: 147 ms
for a 50k-activity range scan, 349 ms warm / 14 s cold to verify ~700
threads.

The cursor is persisted per chunk so a killed run resumes. The
watermark walk keeps a cursor of its own so it cannot leave the
age-based walk stranded behind the deadline. Adds the retention_cursors
table.
This commit is contained in:
Lain Soykaf 2026-09-02 12:32:09 +04:00
commit 6c8ed0a3b8
No known key found for this signature in database
9 changed files with 357 additions and 37 deletions
Download patch file Download diff file Expand all files Collapse all files

143
test/pleroma/retention_test.exs
View file

@ -12,6 +12,8 @@ defmodule Pleroma.RetentionTest do
alias Pleroma.Retention
alias Pleroma.Web.CommonAPI
import Bitwise
import Ecto.Query
import Pleroma.Factory
setup do
@ -29,19 +31,34 @@ defmodule Pleroma.RetentionTest do
%{old_date: old_date}
end
# Posts made through CommonAPI are local; turn the activity into a remote one
# and backdate it so it looks like an old federated post.
defp make_remote_and_old(%Activity{} = activity, old_date) do
activity
|> Ecto.Changeset.change(%{local: false, updated_at: old_date})
|> Repo.update!()
# Activity ids are time-ordered flakes and the retention walk goes by id, so
# an "old" activity needs an old id as well as an old timestamp. Must be
# applied before anything references the activity (bookmarks, notifications).
defp age(%Activity{} = activity, old_date, changes) do
ms = old_date |> DateTime.from_naive!("Etc/UTC") |> DateTime.to_unix(:millisecond)
old_id =
FlakeId.to_string(<<ms::integer-size(64), :rand.uniform(1 <<< 62)::integer-size(64)>>)
changes = Map.merge(%{id: old_id, updated_at: old_date}, changes)
{1, _} =
Activity
|> where([a], a.id == ^activity.id)
|> Repo.update_all(set: Map.to_list(changes))
Map.merge(activity, changes)
rescue
e in Postgrex.Error ->
reraise "age/3 must run before anything references the activity: #{Exception.message(e)}",
__STACKTRACE__
end
defp make_old(%Activity{} = activity, old_date) do
activity
|> Ecto.Changeset.change(%{updated_at: old_date})
|> Repo.update!()
end
# Posts made through CommonAPI are local; turn the activity into a remote one
# and backdate it so it looks like an old federated post.
defp make_remote_and_old(activity, old_date), do: age(activity, old_date, %{local: false})
defp make_old(activity, old_date), do: age(activity, old_date, %{})
defp old_remote_post(user, old_date, params \\ %{}) do
{:ok, activity} = CommonAPI.post(user, Map.merge(%{status: "some thing"}, params))
@ -139,9 +156,10 @@ defmodule Pleroma.RetentionTest do
remote_user = insert(:user, local: false)
local_user = insert(:user)
post = old_remote_post(remote_user, old_date, %{status: "hey @#{local_user.nickname}"})
assert [_] = Repo.all(Pleroma.Notification)
{:ok, post} = CommonAPI.post(remote_user, %{status: "hey @#{local_user.nickname}"})
Repo.delete_all(Pleroma.Notification)
post = make_remote_and_old(post, old_date)
assert {:ok, [_]} = Pleroma.Notification.create_notifications(post)
assert %{contexts: 0} = Retention.run()
assert Object.get_by_ap_id(post.data["object"])
@ -293,7 +311,7 @@ defmodule Pleroma.RetentionTest do
recipients: []
})
make_old(flag, old_date)
flag = make_old(flag, old_date)
{:ok, _} = Pleroma.ReportNote.create(admin.id, flag.id, "looking into it")
assert %{contexts: 0} = Retention.run()
@ -336,6 +354,101 @@ defmodule Pleroma.RetentionTest do
assert Object.get_by_ap_id(post.data["object"])
end
test "remembers where it got to and does not rescan", %{old_date: old_date} do
clear_config([:retention, :batch_size], 1)
remote_user = insert(:user, local: false)
post = old_remote_post(remote_user, old_date)
assert is_nil(Retention.Cursor.get("activities"))
assert %{contexts: 1} = Retention.run()
assert Retention.Cursor.get("activities") == post.id
# An older activity that appears behind the cursor is not visited again
# until the cursor is reset.
older = old_remote_post(remote_user, NaiveDateTime.add(old_date, -86_400))
assert %{contexts: 0} = Retention.run()
assert Object.get_by_ap_id(older.data["object"])
Retention.Cursor.reset("activities")
assert %{contexts: 1} = Retention.run()
refute Object.get_by_ap_id(older.data["object"])
end
test "keeps an old thread that is still active", %{old_date: old_date} do
remote_user = insert(:user, local: false)
remote_user2 = insert(:user, local: false)
post = old_remote_post(remote_user, old_date)
{:ok, reply} =
CommonAPI.post(remote_user2, %{status: "still here", in_reply_to_status_id: post.id})
reply |> Ecto.Changeset.change(%{local: false}) |> Repo.update!()
assert %{contexts: 0} = Retention.run()
assert Object.get_by_ap_id(post.data["object"])
end
test "keeps threads pinned by each local interaction type", %{old_date: old_date} do
remote_user = insert(:user, local: false)
local_user = insert(:user)
liked = old_remote_post(remote_user, old_date)
{:ok, like} = CommonAPI.favorite(liked.id, local_user)
make_old(like, old_date)
repeated = old_remote_post(remote_user, old_date)
{:ok, announce} = CommonAPI.repeat(repeated.id, local_user)
make_old(announce, old_date)
reacted = old_remote_post(remote_user, old_date)
{:ok, react} = CommonAPI.react_with_emoji(reacted.id, local_user, "👍")
make_old(react, old_date)
replied = old_remote_post(remote_user, old_date)
{:ok, reply} = CommonAPI.post(local_user, %{status: "r", in_reply_to_status_id: replied.id})
make_old(reply, old_date)
assert %{contexts: 0} = Retention.run()
for post <- [liked, repeated, reacted, replied] do
assert Object.get_by_ap_id(post.data["object"])
end
end
test "a watermark run does not stall the age-based walk", %{old_date: old_date} do
remote_user = insert(:user, local: false)
clear_config([:retention, :max_objects], 0)
{:ok, recent} = CommonAPI.post(remote_user, %{status: "recent"})
recent |> Ecto.Changeset.change(%{local: false}) |> Repo.update!()
assert %{contexts: 1} = Retention.run()
refute Object.get_by_ap_id(recent.data["object"])
clear_config([:retention, :max_objects], nil)
old = old_remote_post(remote_user, old_date)
assert %{contexts: 1} = Retention.run()
refute Object.get_by_ap_id(old.data["object"])
end
test "moves the cursor to the deadline once everything old is done", %{old_date: old_date} do
remote_user = insert(:user, local: false)
old_remote_post(remote_user, old_date)
Retention.run()
<<cursor_ms::integer-size(64), _::integer-size(64)>> =
FlakeId.from_string(Retention.Cursor.get("activities"))
deadline_ms =
Retention.deadline()
|> DateTime.from_naive!("Etc/UTC")
|> DateTime.to_unix(:millisecond)
assert_in_delta cursor_ms, deadline_ms, 5_000
end
test "removes hashtags that no longer have objects", %{old_date: old_date} do
remote_user = insert(:user, local: false)
old_remote_post(remote_user, old_date, %{status: "#lonelytag"})

11
test/pleroma/workers/cron/retention_worker_test.exs
View file

@ -10,6 +10,7 @@ defmodule Pleroma.Workers.Cron.RetentionWorkerTest do
alias Pleroma.Web.CommonAPI
alias Pleroma.Workers.Cron.RetentionWorker
import Ecto.Query
import Pleroma.Factory
setup do
@ -23,9 +24,13 @@ defmodule Pleroma.Workers.Cron.RetentionWorkerTest do
|> NaiveDateTime.add(-200 * 86_400)
|> NaiveDateTime.truncate(:second)
post
|> Ecto.Changeset.change(%{local: false, updated_at: old_date})
|> Repo.update!()
ms = old_date |> DateTime.from_naive!("Etc/UTC") |> DateTime.to_unix(:millisecond)
old_id = FlakeId.to_string(<<ms::integer-size(64), 1::integer-size(64)>>)
{1, _} =
Pleroma.Activity
|> where([a], a.id == ^post.id)
|> Repo.update_all(set: [id: old_id, local: false, updated_at: old_date])
%{object_id: post.data["object"]}
end