fix(retention): keep threads younger than a day over the watermark
Over the watermark the walk went up to the present, so posts that had just arrived could be evicted while people were reading them.
This commit is contained in:
Lain Soykaf
parent
1a0212bf9a
commit
3cbcae2de3
4 changed files with 45 additions and 36 deletions
|
|
@ -3328,7 +3328,7 @@ config :pleroma, :config_description, [
|
|||
type: :integer,
|
||||
description:
|
||||
"Optional watermark. When the objects table is estimated to hold more rows than this," <>
|
||||
" the oldest unpinned remote threads are evicted regardless of their age.",
|
||||
" the oldest unpinned remote threads quiet for more than a day are evicted, whatever the retention period.",
|
||||
suggestions: [nil, 1_000_000]
|
||||
},
|
||||
%{
|
||||
|
|
|
|||
|
|
@ -1195,12 +1195,12 @@ A thread is never evicted if it contains a local post, reply, favourite, repeat
|
|||
A local quote post does not pin the post it quotes, since quotes do not share the quoted thread's context. Old quoted remote posts can be evicted and will be refetched when the quote is viewed.
|
||||
|
||||
* `enabled`: Run the worker. Defaults to `false`.
|
||||
* `max_objects`: Optional watermark. When the `objects` table is estimated to hold more rows than this, the oldest unpinned remote threads are evicted regardless of age. Defaults to `nil` (age only).
|
||||
* `max_objects`: Optional watermark. When the `objects` table is estimated to hold more rows than this, the oldest unpinned remote threads that have been quiet for more than a day are evicted, whatever `remote_post_retention_days` says. Defaults to `nil` (age only).
|
||||
* `batch_size`: Number of activities walked per run. Their threads are checked and, if unpinned, evicted. Defaults to `50000`.
|
||||
* `keep_non_public`: Also keep threads that contain a non-public post. Defaults to `false`.
|
||||
|
||||
!!! note
|
||||
The worker walks the `activities` table in id order, `batch_size` activities per run, up to the id that corresponds to the retention deadline, and remembers its position in the `retention_cursors` table. Each activity is visited once, when it becomes old enough, and only the threads it belongs to are checked through the context index. Over the `max_objects` watermark a second walk with its own cursor runs up to the present instead. The first run starts from the oldest activity; on an instance that never pruned this backlog takes many runs, so run `prune_objects --keep-threads --prune-orphaned-activities` once for the initial cut. Threads that were kept and later lost their pin (removed bookmarks, cleared notifications of other kinds) are only revisited on a fresh walk: delete the row from `retention_cursors` to restart from the beginning. Space freed by eviction is reused by new rows; it is not handed back to the filesystem unless you `VACUUM FULL`.
|
||||
The worker walks the `activities` table in id order, `batch_size` activities per run, up to the id that corresponds to the retention deadline, and remembers its position in the `retention_cursors` table. Each activity is visited once, when it becomes old enough, and only the threads it belongs to are checked through the context index. Over the `max_objects` watermark a second walk with its own cursor runs up to one day ago instead. The first run starts from the oldest activity; on an instance that never pruned this backlog takes many runs, so run `prune_objects --keep-threads --prune-orphaned-activities` once for the initial cut. Threads that were kept and later lost their pin (removed bookmarks, cleared notifications of other kinds) are only revisited on a fresh walk: delete the row from `retention_cursors` to restart from the beginning. Space freed by eviction is reused by new rows; it is not handed back to the filesystem unless you `VACUUM FULL`.
|
||||
|
||||
## Pleroma.User.Backup
|
||||
|
||||
|
|
|
|||
|
|
@ -9,8 +9,8 @@ defmodule Pleroma.Retention do
|
|||
Local content is data; remote content is a cache of somebody else's data.
|
||||
This module treats it that way: a remote thread is evictable when nobody
|
||||
local has interacted with it and it has been quiet for longer than
|
||||
`remote_post_retention_days`, or when the objects table has grown past the
|
||||
configured watermark. Local posts are never evicted.
|
||||
`remote_post_retention_days`, or for a day when the objects table has grown
|
||||
past the configured watermark. Local posts are never evicted.
|
||||
|
||||
A thread is pinned (never evicted) if any activity in it
|
||||
|
||||
|
|
@ -40,7 +40,7 @@ defmodule Pleroma.Retention do
|
|||
A thread that was kept because an activity's `updated_at` was bumped
|
||||
after insertion is only re-checked on the next full walk, i.e. after a
|
||||
cursor reset. Over the object watermark a second walk, with its own
|
||||
cursor, runs up to the present instead of the deadline.
|
||||
cursor, runs up to a day ago instead of the deadline.
|
||||
|
||||
The first run starts from the oldest activity, which doubles as the
|
||||
initial cut on an instance that never pruned. `prune_objects --keep-threads`
|
||||
|
|
@ -69,6 +69,9 @@ defmodule Pleroma.Retention do
|
|||
@empty_stats %{contexts: 0, objects: 0, activities: 0}
|
||||
|
||||
@default_batch_size 50_000
|
||||
# Over the watermark a thread must still have been quiet this long, so
|
||||
# posts people are reading right now are not evicted from under them.
|
||||
@watermark_min_age 86_400
|
||||
# Activities per verification chunk; the cursor is persisted after each.
|
||||
@chunk 5_000
|
||||
|
||||
|
|
@ -89,21 +92,20 @@ defmodule Pleroma.Retention do
|
|||
"""
|
||||
@spec run(keyword()) :: stats()
|
||||
def run(config \\ Config.get(:retention, [])) do
|
||||
deadline = deadline()
|
||||
over_watermark = over_watermark?(config[:max_objects])
|
||||
budget = config[:batch_size] || @default_batch_size
|
||||
|
||||
# Over the watermark every unpinned thread is fair game, so a separate
|
||||
# walk goes right up to the present. It must not share the cursor with
|
||||
# the age-based walk, or that one would be left behind the deadline.
|
||||
{cursor, boundary} =
|
||||
if over_watermark,
|
||||
do: {"activities_watermark", id_at(NaiveDateTime.utc_now())},
|
||||
else: {"activities", id_at(deadline)}
|
||||
# Over the watermark every unpinned thread older than a day is fair game,
|
||||
# so a separate walk goes up to that point. It must not share the cursor
|
||||
# with the age-based walk, or that one would be left behind the deadline.
|
||||
{cursor, deadline} =
|
||||
if over_watermark?(config[:max_objects]),
|
||||
do: {"activities_watermark", watermark_deadline()},
|
||||
else: {"activities", deadline()}
|
||||
|
||||
boundary = id_at(deadline)
|
||||
|
||||
opts = [
|
||||
keep_non_public: config[:keep_non_public],
|
||||
ignore_deadline: over_watermark,
|
||||
skip_reported: reported_contexts()
|
||||
]
|
||||
|
||||
|
|
@ -192,6 +194,12 @@ defmodule Pleroma.Retention do
|
|||
NaiveDateTime.add(NaiveDateTime.utc_now(), -days * 86_400)
|
||||
end
|
||||
|
||||
# The watermark deadline is never further back than the normal one.
|
||||
defp watermark_deadline do
|
||||
min_age = NaiveDateTime.add(NaiveDateTime.utc_now(), -@watermark_min_age)
|
||||
Enum.max([deadline(), min_age], NaiveDateTime)
|
||||
end
|
||||
|
||||
@doc """
|
||||
Activities grouped by thread context, restricted to threads that may be evicted.
|
||||
|
||||
|
|
@ -201,7 +209,6 @@ defmodule Pleroma.Retention do
|
|||
Options:
|
||||
|
||||
* `:keep_non_public` - also pin threads containing a non-public post
|
||||
* `:ignore_deadline` - consider every unpinned thread, not just quiet ones
|
||||
* `:contexts` - only look at these threads, using the `(type, context)`
|
||||
index; without it the whole table is grouped
|
||||
* `:skip_reported` - a `MapSet` of reported contexts already removed from
|
||||
|
|
@ -209,8 +216,6 @@ defmodule Pleroma.Retention do
|
|||
"""
|
||||
@spec evictable_contexts_query(NaiveDateTime.t(), keyword()) :: Ecto.Query.t()
|
||||
def evictable_contexts_query(deadline, opts \\ []) do
|
||||
ignore_deadline = Keyword.get(opts, :ignore_deadline, false) == true
|
||||
|
||||
contexts =
|
||||
case {Keyword.get(opts, :contexts), Keyword.get(opts, :skip_reported)} do
|
||||
{nil, _} -> nil
|
||||
|
|
@ -225,7 +230,7 @@ defmodule Pleroma.Retention do
|
|||
|> maybe_exclude_reported(is_nil(Keyword.get(opts, :skip_reported)))
|
||||
|> maybe_only_contexts(contexts)
|
||||
|> group_by([a], fragment("? ->> 'context'", a.data))
|
||||
|> having([a], max(a.updated_at) < ^deadline or ^ignore_deadline)
|
||||
|> having([a], max(a.updated_at) < ^deadline)
|
||||
|> having([a], not fragment("bool_or(?)", a.local))
|
||||
|> having([_, b], fragment("max(?::text) is null", b.id))
|
||||
|> having([_, _, n], fragment("max(?) is null", n.id))
|
||||
|
|
|
|||
|
|
@ -58,6 +58,12 @@ defmodule Pleroma.RetentionTest do
|
|||
# and backdate it so it looks like an old federated post.
|
||||
defp make_remote_and_old(activity, old_date), do: age(activity, old_date, %{local: false})
|
||||
|
||||
defp days_ago(days) do
|
||||
NaiveDateTime.utc_now()
|
||||
|> NaiveDateTime.add(-days * 86_400)
|
||||
|> NaiveDateTime.truncate(:second)
|
||||
end
|
||||
|
||||
defp make_old(activity, old_date), do: age(activity, old_date, %{})
|
||||
|
||||
defp old_remote_post(user, old_date, params \\ %{}) do
|
||||
|
|
@ -232,26 +238,15 @@ defmodule Pleroma.RetentionTest do
|
|||
refute Object.get_by_ap_id(newer.data["object"])
|
||||
end
|
||||
|
||||
test "evicts recent unpinned threads when over the object watermark" do
|
||||
test "evicts threads younger than the retention period when over the object watermark" do
|
||||
clear_config([:retention, :max_objects], 1)
|
||||
clear_config([:retention, :batch_size], 1)
|
||||
|
||||
remote_user = insert(:user, local: false)
|
||||
local_user = insert(:user)
|
||||
|
||||
{:ok, older} = CommonAPI.post(remote_user, %{status: "older"})
|
||||
|
||||
older
|
||||
|> Ecto.Changeset.change(%{
|
||||
local: false,
|
||||
updated_at:
|
||||
NaiveDateTime.utc_now() |> NaiveDateTime.add(-60) |> NaiveDateTime.truncate(:second)
|
||||
})
|
||||
|> Repo.update!()
|
||||
|
||||
{:ok, newer} = CommonAPI.post(remote_user, %{status: "newer"})
|
||||
newer |> Ecto.Changeset.change(%{local: false}) |> Repo.update!()
|
||||
|
||||
older = old_remote_post(remote_user, days_ago(3))
|
||||
newer = old_remote_post(remote_user, days_ago(2))
|
||||
{:ok, mine} = CommonAPI.post(local_user, %{status: "mine"})
|
||||
|
||||
assert %{contexts: 1} = Retention.run()
|
||||
|
|
@ -260,6 +255,16 @@ defmodule Pleroma.RetentionTest do
|
|||
assert Object.get_by_ap_id(mine.data["object"])
|
||||
end
|
||||
|
||||
test "keeps threads quiet for less than a day, even over the watermark" do
|
||||
clear_config([:retention, :max_objects], 0)
|
||||
remote_user = insert(:user, local: false)
|
||||
|
||||
post = old_remote_post(remote_user, NaiveDateTime.add(days_ago(1), 3_600))
|
||||
|
||||
assert %{contexts: 0} = Retention.run()
|
||||
assert Object.get_by_ap_id(post.data["object"])
|
||||
end
|
||||
|
||||
test "does not evict anything below the watermark" do
|
||||
clear_config([:retention, :max_objects], 100)
|
||||
remote_user = insert(:user, local: false)
|
||||
|
|
@ -447,8 +452,7 @@ defmodule Pleroma.RetentionTest do
|
|||
remote_user = insert(:user, local: false)
|
||||
|
||||
clear_config([:retention, :max_objects], 0)
|
||||
{:ok, recent} = CommonAPI.post(remote_user, %{status: "recent"})
|
||||
recent |> Ecto.Changeset.change(%{local: false}) |> Repo.update!()
|
||||
recent = old_remote_post(remote_user, days_ago(2))
|
||||
assert %{contexts: 1} = Retention.run()
|
||||
refute Object.get_by_ap_id(recent.data["object"])
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue