feat(retention): walk activities incrementally with a persisted cursor

Grouping the whole activities table by thread every run is unusable on
a large instance: on a 312 GB database it plans as a seq scan plus a
50M-row on-disk sort. Instead, walk activities in id order (flake ids
are time-ordered) from a persisted cursor up to the id that corresponds
to the deadline, and verify only the threads those activities belong to
through the (type, context) index. Measured on that instance: 147 ms
for a 50k-activity range scan, 349 ms warm / 14 s cold to verify ~700
threads.

The cursor is persisted per chunk so a killed run resumes. The
watermark walk keeps a cursor of its own so it cannot leave the
age-based walk stranded behind the deadline. Adds the retention_cursors
table.

test/mix/tasks/pleroma/database_test.exs

@@ -424,6 +424,7 @@ defmodule Mix.Tasks.Pleroma.DatabaseTest do
                ["push_subscriptions"],
                ["registrations"],
                ["report_notes"],
+               ["retention_cursors"],
                ["rich_media_card"],
                ["rules"],
                ["scheduled_activities"],

test/pleroma/retention_test.exs

@@ -12,6 +12,8 @@ defmodule Pleroma.RetentionTest do
   alias Pleroma.Retention
   alias Pleroma.Web.CommonAPI
 
+  import Bitwise
+  import Ecto.Query
   import Pleroma.Factory
 
   setup do
@@ -29,19 +31,34 @@ defmodule Pleroma.RetentionTest do
     %{old_date: old_date}
   end
 
-  # Posts made through CommonAPI are local; turn the activity into a remote one
-  # and backdate it so it looks like an old federated post.
-  defp make_remote_and_old(%Activity{} = activity, old_date) do
-    activity
-    |> Ecto.Changeset.change(%{local: false, updated_at: old_date})
-    |> Repo.update!()
+  # Activity ids are time-ordered flakes and the retention walk goes by id, so
+  # an "old" activity needs an old id as well as an old timestamp. Must be
+  # applied before anything references the activity (bookmarks, notifications).
+  defp age(%Activity{} = activity, old_date, changes) do
+    ms = old_date |> DateTime.from_naive!("Etc/UTC") |> DateTime.to_unix(:millisecond)
+
+    old_id =
+      FlakeId.to_string(<<ms::integer-size(64), :rand.uniform(1 <<< 62)::integer-size(64)>>)
+
+    changes = Map.merge(%{id: old_id, updated_at: old_date}, changes)
+
+    {1, _} =
+      Activity
+      |> where([a], a.id == ^activity.id)
+      |> Repo.update_all(set: Map.to_list(changes))
+
+    Map.merge(activity, changes)
+  rescue
+    e in Postgrex.Error ->
+      reraise "age/3 must run before anything references the activity: #{Exception.message(e)}",
+              __STACKTRACE__
   end
 
-  defp make_old(%Activity{} = activity, old_date) do
-    activity
-    |> Ecto.Changeset.change(%{updated_at: old_date})
-    |> Repo.update!()
-  end
+  # Posts made through CommonAPI are local; turn the activity into a remote one
+  # and backdate it so it looks like an old federated post.
+  defp make_remote_and_old(activity, old_date), do: age(activity, old_date, %{local: false})
+
+  defp make_old(activity, old_date), do: age(activity, old_date, %{})
 
   defp old_remote_post(user, old_date, params \\ %{}) do
     {:ok, activity} = CommonAPI.post(user, Map.merge(%{status: "some thing"}, params))
@@ -139,9 +156,10 @@ defmodule Pleroma.RetentionTest do
       remote_user = insert(:user, local: false)
       local_user = insert(:user)
 
-      post = old_remote_post(remote_user, old_date, %{status: "hey @#{local_user.nickname}"})
-
-      assert [_] = Repo.all(Pleroma.Notification)
+      {:ok, post} = CommonAPI.post(remote_user, %{status: "hey @#{local_user.nickname}"})
+      Repo.delete_all(Pleroma.Notification)
+      post = make_remote_and_old(post, old_date)
+      assert {:ok, [_]} = Pleroma.Notification.create_notifications(post)
 
       assert %{contexts: 0} = Retention.run()
       assert Object.get_by_ap_id(post.data["object"])
@@ -293,7 +311,7 @@ defmodule Pleroma.RetentionTest do
           recipients: []
         })
 
-      make_old(flag, old_date)
+      flag = make_old(flag, old_date)
       {:ok, _} = Pleroma.ReportNote.create(admin.id, flag.id, "looking into it")
 
       assert %{contexts: 0} = Retention.run()
@@ -336,6 +354,101 @@ defmodule Pleroma.RetentionTest do
       assert Object.get_by_ap_id(post.data["object"])
     end
 
+    test "remembers where it got to and does not rescan", %{old_date: old_date} do
+      clear_config([:retention, :batch_size], 1)
+      remote_user = insert(:user, local: false)
+
+      post = old_remote_post(remote_user, old_date)
+      assert is_nil(Retention.Cursor.get("activities"))
+
+      assert %{contexts: 1} = Retention.run()
+      assert Retention.Cursor.get("activities") == post.id
+
+      # An older activity that appears behind the cursor is not visited again
+      # until the cursor is reset.
+      older = old_remote_post(remote_user, NaiveDateTime.add(old_date, -86_400))
+      assert %{contexts: 0} = Retention.run()
+      assert Object.get_by_ap_id(older.data["object"])
+
+      Retention.Cursor.reset("activities")
+      assert %{contexts: 1} = Retention.run()
+      refute Object.get_by_ap_id(older.data["object"])
+    end
+
+    test "keeps an old thread that is still active", %{old_date: old_date} do
+      remote_user = insert(:user, local: false)
+      remote_user2 = insert(:user, local: false)
+
+      post = old_remote_post(remote_user, old_date)
+
+      {:ok, reply} =
+        CommonAPI.post(remote_user2, %{status: "still here", in_reply_to_status_id: post.id})
+
+      reply |> Ecto.Changeset.change(%{local: false}) |> Repo.update!()
+
+      assert %{contexts: 0} = Retention.run()
+      assert Object.get_by_ap_id(post.data["object"])
+    end
+
+    test "keeps threads pinned by each local interaction type", %{old_date: old_date} do
+      remote_user = insert(:user, local: false)
+      local_user = insert(:user)
+
+      liked = old_remote_post(remote_user, old_date)
+      {:ok, like} = CommonAPI.favorite(liked.id, local_user)
+      make_old(like, old_date)
+
+      repeated = old_remote_post(remote_user, old_date)
+      {:ok, announce} = CommonAPI.repeat(repeated.id, local_user)
+      make_old(announce, old_date)
+
+      reacted = old_remote_post(remote_user, old_date)
+      {:ok, react} = CommonAPI.react_with_emoji(reacted.id, local_user, "👍")
+      make_old(react, old_date)
+
+      replied = old_remote_post(remote_user, old_date)
+      {:ok, reply} = CommonAPI.post(local_user, %{status: "r", in_reply_to_status_id: replied.id})
+      make_old(reply, old_date)
+
+      assert %{contexts: 0} = Retention.run()
+
+      for post <- [liked, repeated, reacted, replied] do
+        assert Object.get_by_ap_id(post.data["object"])
+      end
+    end
+
+    test "a watermark run does not stall the age-based walk", %{old_date: old_date} do
+      remote_user = insert(:user, local: false)
+
+      clear_config([:retention, :max_objects], 0)
+      {:ok, recent} = CommonAPI.post(remote_user, %{status: "recent"})
+      recent |> Ecto.Changeset.change(%{local: false}) |> Repo.update!()
+      assert %{contexts: 1} = Retention.run()
+      refute Object.get_by_ap_id(recent.data["object"])
+
+      clear_config([:retention, :max_objects], nil)
+      old = old_remote_post(remote_user, old_date)
+      assert %{contexts: 1} = Retention.run()
+      refute Object.get_by_ap_id(old.data["object"])
+    end
+
+    test "moves the cursor to the deadline once everything old is done", %{old_date: old_date} do
+      remote_user = insert(:user, local: false)
+      old_remote_post(remote_user, old_date)
+
+      Retention.run()
+
+      <<cursor_ms::integer-size(64), _::integer-size(64)>> =
+        FlakeId.from_string(Retention.Cursor.get("activities"))
+
+      deadline_ms =
+        Retention.deadline()
+        |> DateTime.from_naive!("Etc/UTC")
+        |> DateTime.to_unix(:millisecond)
+
+      assert_in_delta cursor_ms, deadline_ms, 5_000
+    end
+
     test "removes hashtags that no longer have objects", %{old_date: old_date} do
       remote_user = insert(:user, local: false)
       old_remote_post(remote_user, old_date, %{status: "#lonelytag"})

test/pleroma/workers/cron/retention_worker_test.exs

@@ -10,6 +10,7 @@ defmodule Pleroma.Workers.Cron.RetentionWorkerTest do
   alias Pleroma.Web.CommonAPI
   alias Pleroma.Workers.Cron.RetentionWorker
 
+  import Ecto.Query
   import Pleroma.Factory
 
   setup do
@@ -23,9 +24,13 @@ defmodule Pleroma.Workers.Cron.RetentionWorkerTest do
       |> NaiveDateTime.add(-200 * 86_400)
       |> NaiveDateTime.truncate(:second)
 
-    post
-    |> Ecto.Changeset.change(%{local: false, updated_at: old_date})
-    |> Repo.update!()
+    ms = old_date |> DateTime.from_naive!("Etc/UTC") |> DateTime.to_unix(:millisecond)
+    old_id = FlakeId.to_string(<<ms::integer-size(64), 1::integer-size(64)>>)
+
+    {1, _} =
+      Pleroma.Activity
+      |> where([a], a.id == ^post.id)
+      |> Repo.update_all(set: [id: old_id, local: false, updated_at: old_date])
 
     %{object_id: post.data["object"]}
   end