diff --git a/lib/pleroma/retention.ex b/lib/pleroma/retention.ex new file mode 100644 index 000000000..59ccf449a --- /dev/null +++ b/lib/pleroma/retention.ex @@ -0,0 +1,283 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Retention do + @moduledoc """ + Bounded retention of remote content. + + Local content is data; remote content is a cache of somebody else's data. + This module treats it that way: a remote thread is evictable when nobody + local has interacted with it and it has been quiet for longer than + `remote_post_retention_days`, or when the objects table has grown past the + configured watermark. Local posts are never evicted. + + A thread is pinned (never evicted) if any activity in it + + * is local (posts, replies, favourites, repeats and reactions by local + users all carry the thread's context), + * is bookmarked by a local user, + * still has a notification for a local user (mentions), + * is a report (Flag), + + or if any object in it is referenced by a report. + + Only activities that carry a context take part; relationship activities + such as Follow or Block and chat messages have none and are never touched. + + Eviction works in bounded batches so it can run continuously from a cron + worker (`Pleroma.Workers.Cron.RetentionWorker`); the aim is a database that + plateaus rather than one that has to be shrunk in maintenance windows. + """ + + import Ecto.Query + + alias Pleroma.Activity + alias Pleroma.Bookmark + alias Pleroma.Config + alias Pleroma.Notification + alias Pleroma.Object + alias Pleroma.Repo + + require Logger + require Pleroma.Constants + + @type stats :: %{ + contexts: non_neg_integer(), + objects: non_neg_integer(), + activities: non_neg_integer() + } + + @empty_stats %{contexts: 0, objects: 0, activities: 0} + + @doc """ + Runs one eviction batch using the `:retention` config and returns what was removed. + """ + @spec run(keyword()) :: stats() + def run(config \\ Config.get(:retention, [])) do + deadline = deadline() + over_watermark = over_watermark?(config[:max_objects]) + + query = + deadline + |> evictable_contexts_query( + keep_non_public: config[:keep_non_public], + ignore_deadline: over_watermark + ) + |> order_by([a], asc: max(a.updated_at), asc: fragment("min(?::text)", a.id)) + |> limit(^(config[:batch_size] || 500)) + |> select([a], %{ + activity_ids: type(fragment("array_agg(?)", a.id), {:array, FlakeId.Ecto.CompatType}), + object_ids: fragment("array_agg(associated_object_id(?))", a.data) + }) + + # Select and delete in one transaction, so a local interaction that lands + # in between cannot be left pointing at an evicted object. + {:ok, stats} = + Repo.transaction( + fn -> + query + |> Repo.all(timeout: :infinity) + |> evict() + end, + timeout: :infinity + ) + + if stats.contexts > 0, do: delete_unused_hashtags() + + stats + end + + @doc """ + The point in time before which a quiet remote thread becomes evictable. + """ + @spec deadline() :: NaiveDateTime.t() + def deadline do + days = Config.get([:instance, :remote_post_retention_days]) + NaiveDateTime.add(NaiveDateTime.utc_now(), -days * 86_400) + end + + @doc """ + Activities grouped by thread context, restricted to threads that may be evicted. + + The query has no `select`; callers pick what they need from the group (the + context alone for `IN (subquery)`, or aggregates of ids for deletion). + + Options: + + * `:keep_non_public` - also pin threads containing a non-public post + * `:ignore_deadline` - consider every unpinned thread, not just quiet ones + """ + @spec evictable_contexts_query(NaiveDateTime.t(), keyword()) :: Ecto.Query.t() + def evictable_contexts_query(deadline, opts \\ []) do + ignore_deadline = Keyword.get(opts, :ignore_deadline, false) == true + + Activity + |> join(:left, [a], b in Bookmark, on: a.id == b.activity_id) + |> join(:left, [a], n in Notification, on: a.id == n.activity_id) + |> where([a], not is_nil(fragment("? ->> 'context'", a.data))) + |> where([a], fragment("? ->> 'context'", a.data) not in subquery(reported_contexts_query())) + |> group_by([a], fragment("? ->> 'context'", a.data)) + |> having([a], max(a.updated_at) < ^deadline or ^ignore_deadline) + |> having([a], not fragment("bool_or(?)", a.local)) + |> having([_, b], fragment("max(?::text) is null", b.id)) + |> having([_, _, n], fragment("max(?) is null", n.id)) + |> maybe_keep_non_public(Keyword.get(opts, :keep_non_public, false) == true) + end + + defp maybe_keep_non_public(query, false), do: query + + defp maybe_keep_non_public(query, true) do + having( + query, + [a], + not fragment( + # Posts (checked on Create Activity) is non-public + "bool_or((not(?->'to' \\? ? OR ?->'cc' \\? ?)) and ? ->> 'type' = 'Create')", + a.data, + ^Pleroma.Constants.as_public(), + a.data, + ^Pleroma.Constants.as_public(), + a.data + ) + ) + end + + # Contexts touched by reports: the contexts of the reported objects, plus the + # Flag activities' own contexts (a report gets a fresh context of its own). + # Flag objects are a list mixing actor ids, object ids and embedded copies + # of objects with an "id"; a single object instead of a list is tolerated. + defp reported_contexts_query do + flags = where(Activity, [f], fragment("? ->> 'type' = 'Flag'", f.data)) + + reported_objects = + flags + |> join( + :inner, + [f], + fo in fragment( + "jsonb_array_elements(CASE WHEN jsonb_typeof(? -> 'object') = 'array' THEN ? -> 'object' ELSE jsonb_build_array(? -> 'object') END)", + f.data, + f.data, + f.data + ), + on: true + ) + |> join(:inner, [_, fo], o in Object, + on: fragment("? ->> 'id' = coalesce(? ->> 'id', ? #>> '{}')", o.data, fo.value, fo.value) + ) + |> where([_, _, o], not is_nil(fragment("? ->> 'context'", o.data))) + |> select([_, _, o], fragment("? ->> 'context'", o.data)) + + flags + |> where([f], not is_nil(fragment("? ->> 'context'", f.data))) + |> select([f], fragment("? ->> 'context'", f.data)) + |> union(^reported_objects) + end + + @doc """ + Deletes the given threads. Each entry carries the activity ids and object + ids collected by `run/1`, so deletion uses primary key and ap_id indexes only. + """ + @spec evict([%{activity_ids: [String.t()], object_ids: [String.t() | nil]}]) :: stats() + def evict([]), do: @empty_stats + + def evict(threads) do + activity_ids = Enum.flat_map(threads, & &1.activity_ids) + object_ids = threads |> Enum.flat_map(& &1.object_ids) |> Enum.reject(&is_nil/1) + + {:ok, stats} = + Repo.transaction( + fn -> + {objects, deleted_ap_ids} = + Object + |> where([o], fragment("? ->> 'id'", o.data) in ^object_ids) + |> where( + [o], + fragment( + "left(? ->> 'actor', ?) != ?", + o.data, + ^local_prefix_length(), + ^local_prefix() + ) + ) + |> select([o], fragment("? ->> 'id'", o.data)) + |> Repo.delete_all(timeout: :infinity) + + {activities, _} = + Activity + |> where([a], a.id in ^activity_ids) + |> where([a], a.local == false) + |> where([a], fragment("? ->> 'type' != 'Flag'", a.data)) + |> Repo.delete_all(timeout: :infinity) + + # Remote activities outside the thread that point at objects we just + # removed (e.g. Deletes carry no context). Flags are kept as they + # may have report notes attached. + {referencing, _} = + Activity + |> where([a], a.local == false) + |> where([a], fragment("associated_object_id(?)", a.data) in ^deleted_ap_ids) + |> where([a], fragment("? ->> 'type' != 'Flag'", a.data)) + |> Repo.delete_all(timeout: :infinity) + + %{ + contexts: length(threads), + objects: objects, + activities: activities + referencing + } + end, + timeout: :infinity + ) + + stats + end + + @doc """ + Whether the objects table is estimated to hold more rows than `max_objects`. + """ + @spec over_watermark?(non_neg_integer() | nil) :: boolean() + def over_watermark?(nil), do: false + def over_watermark?(max_objects), do: estimated_object_count() > max_objects + + @doc """ + Row count of the objects table from planner statistics, falling back to an + exact count when the table has never been analyzed. + """ + @spec estimated_object_count() :: non_neg_integer() + def estimated_object_count do + %{rows: [[estimate]]} = + Repo.query!("SELECT reltuples::bigint FROM pg_class WHERE oid = 'objects'::regclass") + + if estimate > 0, do: estimate, else: Repo.aggregate(Object, :count, timeout: :infinity) + end + + @doc """ + Removes hashtags no longer attached to any object and not followed by anyone. + """ + @spec delete_unused_hashtags() :: :ok + def delete_unused_hashtags do + Repo.query!( + """ + DELETE FROM hashtags AS ht + WHERE NOT EXISTS ( + SELECT 1 FROM hashtags_objects hto + WHERE ht.id = hto.hashtag_id + ) + AND NOT EXISTS ( + SELECT 1 FROM user_follows_hashtag ufh + WHERE ht.id = ufh.hashtag_id + ) + """, + [], + timeout: :infinity + ) + + :ok + end + + # Actor ids of local users start with the instance url; an exact prefix + # comparison avoids LIKE wildcards and host/port ambiguity. + defp local_prefix, do: Pleroma.Web.Endpoint.url() <> "/" + defp local_prefix_length, do: String.length(local_prefix()) +end diff --git a/test/pleroma/retention_test.exs b/test/pleroma/retention_test.exs new file mode 100644 index 000000000..67460b38a --- /dev/null +++ b/test/pleroma/retention_test.exs @@ -0,0 +1,349 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.RetentionTest do + use Pleroma.DataCase, async: false + + alias Pleroma.Activity + alias Pleroma.Bookmark + alias Pleroma.Object + alias Pleroma.Repo + alias Pleroma.Retention + alias Pleroma.Web.CommonAPI + + import Pleroma.Factory + + setup do + clear_config([:instance, :remote_post_retention_days], 90) + clear_config([:retention, :enabled], true) + clear_config([:retention, :max_objects], nil) + clear_config([:retention, :batch_size], 500) + clear_config([:retention, :keep_non_public], false) + + old_date = + NaiveDateTime.utc_now() + |> NaiveDateTime.add(-200 * 86_400) + |> NaiveDateTime.truncate(:second) + + %{old_date: old_date} + end + + # Posts made through CommonAPI are local; turn the activity into a remote one + # and backdate it so it looks like an old federated post. + defp make_remote_and_old(%Activity{} = activity, old_date) do + activity + |> Ecto.Changeset.change(%{local: false, updated_at: old_date}) + |> Repo.update!() + end + + defp make_old(%Activity{} = activity, old_date) do + activity + |> Ecto.Changeset.change(%{updated_at: old_date}) + |> Repo.update!() + end + + defp old_remote_post(user, old_date, params \\ %{}) do + {:ok, activity} = CommonAPI.post(user, Map.merge(%{status: "some thing"}, params)) + make_remote_and_old(activity, old_date) + end + + describe "estimated_object_count/0" do + test "falls back to an exact count when the planner has no statistics" do + insert(:note) + insert(:note) + + assert Retention.estimated_object_count() == Repo.aggregate(Object, :count) + end + end + + describe "run/1" do + test "evicts an old remote thread nobody local touched", %{old_date: old_date} do + remote_user = insert(:user, local: false) + remote_user2 = insert(:user, local: false) + + post = old_remote_post(remote_user, old_date) + + {:ok, reply} = + CommonAPI.post(remote_user2, %{status: "reply", in_reply_to_status_id: post.id}) + + make_remote_and_old(reply, old_date) + + {:ok, like} = CommonAPI.favorite(post.id, remote_user2) + make_remote_and_old(like, old_date) + + assert %{contexts: 1, objects: 2, activities: 3} = Retention.run() + + refute Object.get_by_ap_id(post.data["object"]) + refute Object.get_by_ap_id(reply.data["object"]) + assert Repo.all(Activity) == [] + end + + test "keeps recent remote threads" do + remote_user = insert(:user, local: false) + {:ok, post} = CommonAPI.post(remote_user, %{status: "fresh"}) + post |> Ecto.Changeset.change(%{local: false}) |> Repo.update!() + + assert %{contexts: 0, objects: 0} = Retention.run() + assert Object.get_by_ap_id(post.data["object"]) + end + + test "keeps old local posts", %{old_date: old_date} do + user = insert(:user) + {:ok, post} = CommonAPI.post(user, %{status: "mine"}) + make_old(post, old_date) + + assert %{contexts: 0, objects: 0} = Retention.run() + assert Object.get_by_ap_id(post.data["object"]) + end + + test "keeps an old remote thread a local user favourited", %{old_date: old_date} do + remote_user = insert(:user, local: false) + local_user = insert(:user) + + post = old_remote_post(remote_user, old_date) + {:ok, like} = CommonAPI.favorite(post.id, local_user) + make_old(like, old_date) + + assert %{contexts: 0} = Retention.run() + assert Object.get_by_ap_id(post.data["object"]) + end + + test "keeps an old remote thread a local user replied to", %{old_date: old_date} do + remote_user = insert(:user, local: false) + local_user = insert(:user) + + post = old_remote_post(remote_user, old_date) + + {:ok, reply} = + CommonAPI.post(local_user, %{status: "reply", in_reply_to_status_id: post.id}) + + make_old(reply, old_date) + + assert %{contexts: 0} = Retention.run() + assert Object.get_by_ap_id(post.data["object"]) + end + + test "keeps an old remote thread a local user bookmarked", %{old_date: old_date} do + remote_user = insert(:user, local: false) + local_user = insert(:user) + + post = old_remote_post(remote_user, old_date) + {:ok, _} = Bookmark.create(local_user.id, post.id) + + assert %{contexts: 0} = Retention.run() + assert Object.get_by_ap_id(post.data["object"]) + end + + test "keeps an old remote post that mentioned a local user", %{old_date: old_date} do + remote_user = insert(:user, local: false) + local_user = insert(:user) + + post = old_remote_post(remote_user, old_date, %{status: "hey @#{local_user.nickname}"}) + + assert [_] = Repo.all(Pleroma.Notification) + + assert %{contexts: 0} = Retention.run() + assert Object.get_by_ap_id(post.data["object"]) + end + + test "keeps an old remote post that was reported", %{old_date: old_date} do + remote_user = insert(:user, local: false) + local_user = insert(:user) + + post = old_remote_post(remote_user, old_date) + + {:ok, _flag} = + CommonAPI.report(local_user, %{account_id: remote_user.id, status_ids: [post.id]}) + + assert %{contexts: 0} = Retention.run() + assert Object.get_by_ap_id(post.data["object"]) + end + + test "evicts an old remote thread even if remote users interacted with it", %{ + old_date: old_date + } do + remote_user = insert(:user, local: false) + remote_user2 = insert(:user, local: false) + + post = old_remote_post(remote_user, old_date) + {:ok, announce} = CommonAPI.repeat(post.id, remote_user2) + make_remote_and_old(announce, old_date) + + assert %{contexts: 1, objects: 1, activities: 2} = Retention.run() + refute Object.get_by_ap_id(post.data["object"]) + end + + test "evicts old threads in batches, oldest first", %{old_date: old_date} do + clear_config([:retention, :batch_size], 1) + remote_user = insert(:user, local: false) + + older = old_remote_post(remote_user, NaiveDateTime.add(old_date, -86_400)) + newer = old_remote_post(remote_user, old_date) + + assert %{contexts: 1} = Retention.run() + refute Object.get_by_ap_id(older.data["object"]) + assert Object.get_by_ap_id(newer.data["object"]) + + assert %{contexts: 1} = Retention.run() + refute Object.get_by_ap_id(newer.data["object"]) + end + + test "evicts recent unpinned threads when over the object watermark" do + clear_config([:retention, :max_objects], 1) + clear_config([:retention, :batch_size], 1) + + remote_user = insert(:user, local: false) + local_user = insert(:user) + + {:ok, older} = CommonAPI.post(remote_user, %{status: "older"}) + + older + |> Ecto.Changeset.change(%{ + local: false, + updated_at: + NaiveDateTime.utc_now() |> NaiveDateTime.add(-60) |> NaiveDateTime.truncate(:second) + }) + |> Repo.update!() + + {:ok, newer} = CommonAPI.post(remote_user, %{status: "newer"}) + newer |> Ecto.Changeset.change(%{local: false}) |> Repo.update!() + + {:ok, mine} = CommonAPI.post(local_user, %{status: "mine"}) + + assert %{contexts: 1} = Retention.run() + refute Object.get_by_ap_id(older.data["object"]) + assert Object.get_by_ap_id(newer.data["object"]) + assert Object.get_by_ap_id(mine.data["object"]) + end + + test "does not evict anything below the watermark" do + clear_config([:retention, :max_objects], 100) + remote_user = insert(:user, local: false) + + {:ok, post} = CommonAPI.post(remote_user, %{status: "fresh"}) + post |> Ecto.Changeset.change(%{local: false}) |> Repo.update!() + + assert %{contexts: 0} = Retention.run() + assert Object.get_by_ap_id(post.data["object"]) + end + + test "with keep_non_public it keeps old non-public remote threads", %{old_date: old_date} do + clear_config([:retention, :keep_non_public], true) + remote_user = insert(:user, local: false) + + {:ok, private} = CommonAPI.post(remote_user, %{status: "psst", visibility: "private"}) + make_remote_and_old(private, old_date) + + public = old_remote_post(remote_user, old_date) + + assert %{contexts: 1} = Retention.run() + assert Object.get_by_ap_id(private.data["object"]) + refute Object.get_by_ap_id(public.data["object"]) + end + + test "never touches activities without a context, even over the watermark" do + clear_config([:retention, :max_objects], 1) + remote_user = insert(:user, local: false) + remote_user2 = insert(:user, local: false) + local_user = insert(:user) + + {:ok, _, _, follow} = CommonAPI.follow(local_user, remote_user) + {:ok, block} = CommonAPI.block(remote_user2, local_user) + + # Enough objects to trip the watermark + {:ok, post} = CommonAPI.post(remote_user, %{status: "a"}) + post |> Ecto.Changeset.change(%{local: false}) |> Repo.update!() + {:ok, post2} = CommonAPI.post(remote_user, %{status: "b"}) + post2 |> Ecto.Changeset.change(%{local: false}) |> Repo.update!() + + for activity <- [follow, block] do + assert is_nil(activity.data["context"]) + + activity + |> Ecto.Changeset.change(%{local: false}) + |> Repo.update!() + end + + Retention.run() + + assert Activity.get_by_id(follow.id) + assert Activity.get_by_id(block.id) + end + + test "keeps remote reports and the threads they point at", %{old_date: old_date} do + remote_user = insert(:user, local: false) + reporter = insert(:user, local: false) + admin = insert(:user, is_admin: true) + + post = old_remote_post(remote_user, old_date) + + flag = + Repo.insert!(%Activity{ + data: %{ + "id" => "https://domain1.com/activities/#{Ecto.UUID.generate()}", + "type" => "Flag", + "actor" => reporter.ap_id, + "object" => [remote_user.ap_id, post.data["object"]], + "context" => "https://domain1.com/contexts/#{Ecto.UUID.generate()}", + "content" => "spam" + }, + local: false, + actor: reporter.ap_id, + recipients: [] + }) + + make_old(flag, old_date) + {:ok, _} = Pleroma.ReportNote.create(admin.id, flag.id, "looking into it") + + assert %{contexts: 0} = Retention.run() + assert Activity.get_by_id(flag.id) + assert Object.get_by_ap_id(post.data["object"]) + end + + test "removes remote activities outside the thread that point at evicted objects", %{ + old_date: old_date + } do + remote_user = insert(:user, local: false) + post = old_remote_post(remote_user, old_date) + + delete = + Repo.insert!(%Activity{ + data: %{ + "id" => "https://domain1.com/activities/#{Ecto.UUID.generate()}", + "type" => "Delete", + "actor" => remote_user.ap_id, + "object" => post.data["object"] + }, + local: false, + actor: remote_user.ap_id, + recipients: [] + }) + + assert %{contexts: 1, objects: 1, activities: 2} = Retention.run() + refute Activity.get_by_id(delete.id) + end + + test "never deletes local objects, even inside an otherwise remote thread", %{ + old_date: old_date + } do + local_user = insert(:user) + {:ok, post} = CommonAPI.post(local_user, %{status: "mine, honestly"}) + # Pretend the Create activity federated in from elsewhere + make_remote_and_old(post, old_date) + + assert %{contexts: 1, objects: 0} = Retention.run() + assert Object.get_by_ap_id(post.data["object"]) + end + + test "removes hashtags that no longer have objects", %{old_date: old_date} do + remote_user = insert(:user, local: false) + old_remote_post(remote_user, old_date, %{status: "#lonelytag"}) + assert Pleroma.Hashtag.get_by_name("lonelytag") + + Retention.run() + + refute Pleroma.Hashtag.get_by_name("lonelytag") + end + end +end