hj/pleroma
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

schedule activity expiration in Oban

Browse source
This commit is contained in:
Alexander Strizhakov 2020-08-22 20:46:01 +03:00 • committed by rinpatch
commit 9bf1065a06
23 changed files with 229 additions and 386 deletions
Download patch file Download diff file Expand all files Collapse all files

18
lib/mix/tasks/pleroma/database.ex
View file

@ -133,8 +133,7 @@ defmodule Mix.Tasks.Pleroma.Database do
days = Pleroma.Config.get([:mrf_activity_expiration, :days], 365)
Pleroma.Activity
|> join(:left, [a], u in assoc(a, :expiration))
|> join(:inner, [a, _u], o in Object,
|> join(:inner, [a], o in Object,
on:
fragment(
"(?->>'id') = COALESCE((?)->'object'->> 'id', (?)->>'object')",
@ -144,14 +143,21 @@ defmodule Mix.Tasks.Pleroma.Database do
)
)
|> where(local: true)
|> where([a, u], is_nil(u))
|> where([a], fragment("(? ->> 'type'::text) = 'Create'", a.data))
|> where([_a, _u, o], fragment("?->>'type' = 'Note'", o.data))
|> where([_a, o], fragment("?->>'type' = 'Note'", o.data))
|> Pleroma.RepoStreamer.chunk_stream(100)
|> Stream.each(fn activities ->
Enum.each(activities, fn activity ->
expires_at = Timex.shift(activity.inserted_at, days: days)
Pleroma.ActivityExpiration.create(activity, expires_at, false)
expires_at =
activity.inserted_at
|> DateTime.from_naive!("Etc/UTC")
|> Timex.shift(days: days)
Pleroma.Workers.PurgeExpiredActivity.enqueue(%{
activity_id: activity.id,
expires_at: expires_at,
validate: false
})
end)
end)
|> Stream.run()

3
lib/pleroma/activity.ex
View file

@ -7,7 +7,6 @@ defmodule Pleroma.Activity do
alias Pleroma.Activity
alias Pleroma.Activity.Queries
alias Pleroma.ActivityExpiration
alias Pleroma.Bookmark
alias Pleroma.Notification
alias Pleroma.Object
@ -60,8 +59,6 @@ defmodule Pleroma.Activity do
# typical case.
has_one(:object, Object, on_delete: :nothing, foreign_key: :id)
has_one(:expiration, ActivityExpiration, on_delete: :delete_all)
timestamps()
end

74
lib/pleroma/activity_expiration.ex
View file

@ -1,74 +0,0 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.ActivityExpiration do
use Ecto.Schema
alias Pleroma.Activity
alias Pleroma.ActivityExpiration
alias Pleroma.Repo
import Ecto.Changeset
import Ecto.Query
@type t :: %__MODULE__{}
@min_activity_lifetime :timer.hours(1)
schema "activity_expirations" do
belongs_to(:activity, Activity, type: FlakeId.Ecto.CompatType)
field(:scheduled_at, :naive_datetime)
end
def changeset(%ActivityExpiration{} = expiration, attrs, validate_scheduled_at) do
expiration
|> cast(attrs, [:scheduled_at])
|> validate_required([:scheduled_at])
|> validate_scheduled_at(validate_scheduled_at)
end
def get_by_activity_id(activity_id) do
ActivityExpiration
|> where([exp], exp.activity_id == ^activity_id)
|> Repo.one()
end
def create(%Activity{} = activity, scheduled_at, validate_scheduled_at \\ true) do
%ActivityExpiration{activity_id: activity.id}
|> changeset(%{scheduled_at: scheduled_at}, validate_scheduled_at)
|> Repo.insert()
end
def due_expirations(offset \\ 0) do
naive_datetime =
NaiveDateTime.utc_now()
|> NaiveDateTime.add(offset, :millisecond)
ActivityExpiration
|> where([exp], exp.scheduled_at < ^naive_datetime)
|> limit(50)
|> preload(:activity)
|> Repo.all()
|> Enum.reject(fn %{activity: activity} ->
Activity.pinned_by_actor?(activity)
end)
end
def validate_scheduled_at(changeset, false), do: changeset
def validate_scheduled_at(changeset, true) do
validate_change(changeset, :scheduled_at, fn _, scheduled_at ->
if not expires_late_enough?(scheduled_at) do
[scheduled_at: "an ephemeral activity must live for at least one hour"]
else
[]
end
end)
end
def expires_late_enough?(scheduled_at) do
now = NaiveDateTime.utc_now()
diff = NaiveDateTime.diff(scheduled_at, now, :millisecond)
diff > @min_activity_lifetime
end
end

7
lib/pleroma/web/activity_pub/activity_pub.ex
View file

@ -5,7 +5,6 @@
defmodule Pleroma.Web.ActivityPub.ActivityPub do
alias Pleroma.Activity
alias Pleroma.Activity.Ir.Topics
alias Pleroma.ActivityExpiration
alias Pleroma.Config
alias Pleroma.Constants
alias Pleroma.Conversation
@ -165,7 +164,11 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
end
defp maybe_create_activity_expiration({:ok, %{data: %{"expires_at" => expires_at}} = activity}) do
with {:ok, _} <- ActivityExpiration.create(activity, expires_at) do
with {:ok, _job} <-
Pleroma.Workers.PurgeExpiredActivity.enqueue(%{
activity_id: activity.id,
expires_at: expires_at
}) do
{:ok, activity}
end
end

4
lib/pleroma/web/activity_pub/mrf/activity_expiration_policy.ex
View file

@ -31,10 +31,10 @@ defmodule Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicy do
defp maybe_add_expiration(activity) do
days = Pleroma.Config.get([:mrf_activity_expiration, :days], 365)
expires_at = NaiveDateTime.utc_now() |> Timex.shift(days: days)
expires_at = DateTime.utc_now() |> Timex.shift(days: days)
with %{"expires_at" => existing_expires_at} <- activity,
:lt <- NaiveDateTime.compare(existing_expires_at, expires_at) do
:lt <- DateTime.compare(existing_expires_at, expires_at) do
activity
else
_ -> Map.put(activity, "expires_at", expires_at)

6
lib/pleroma/web/activity_pub/side_effects.ex
View file

@ -7,7 +7,6 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
"""
alias Pleroma.Activity
alias Pleroma.Activity.Ir.Topics
alias Pleroma.ActivityExpiration
alias Pleroma.Chat
alias Pleroma.Chat.MessageReference
alias Pleroma.FollowingRelationship
@ -189,7 +188,10 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
end
if expires_at = activity.data["expires_at"] do
ActivityExpiration.create(activity, expires_at)
Pleroma.Workers.PurgeExpiredActivity.enqueue(%{
activity_id: activity.id,
expires_at: expires_at
})
end
BackgroundWorker.enqueue("fetch_data_for_activity", %{"activity_id" => activity.id})

2
lib/pleroma/web/common_api/activity_draft.ex
View file

@ -202,7 +202,7 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do
additional =
case draft.expires_at do
%NaiveDateTime{} = expires_at -> Map.put(additional, "expires_at", expires_at)
%DateTime{} = expires_at -> Map.put(additional, "expires_at", expires_at)
_ -> additional
end

5
lib/pleroma/web/common_api/common_api.ex
View file

@ -4,7 +4,6 @@
defmodule Pleroma.Web.CommonAPI do
alias Pleroma.Activity
alias Pleroma.ActivityExpiration
alias Pleroma.Conversation.Participation
alias Pleroma.Formatter
alias Pleroma.Object
@ -381,9 +380,9 @@ defmodule Pleroma.Web.CommonAPI do
def check_expiry_date({:ok, nil} = res), do: res
def check_expiry_date({:ok, in_seconds}) do
expiry = NaiveDateTime.utc_now() |> NaiveDateTime.add(in_seconds)
expiry = DateTime.add(DateTime.utc_now(), in_seconds)
if ActivityExpiration.expires_late_enough?(expiry) do
if Pleroma.Workers.PurgeExpiredActivity.expires_late_enough?(expiry) do
{:ok, expiry}
else
{:error, "Expiry date is too soon"}

5
lib/pleroma/web/mastodon_api/views/status_view.ex
View file

@ -8,7 +8,6 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
require Pleroma.Constants
alias Pleroma.Activity
alias Pleroma.ActivityExpiration
alias Pleroma.HTML
alias Pleroma.Object
alias Pleroma.Repo
@ -245,8 +244,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
expires_at =
with true <- client_posted_this_activity,
%ActivityExpiration{scheduled_at: scheduled_at} <-
ActivityExpiration.get_by_activity_id(activity.id) do
%Oban.Job{scheduled_at: scheduled_at} <-
Pleroma.Workers.PurgeExpiredActivity.get_expiration(activity.id) do
scheduled_at
else
_ -> nil

48
lib/pleroma/workers/cron/purge_expired_activities_worker.ex
View file

@ -1,48 +0,0 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Workers.Cron.PurgeExpiredActivitiesWorker do
@moduledoc """
The worker to purge expired activities.
"""
use Oban.Worker, queue: "activity_expiration"
alias Pleroma.Activity
alias Pleroma.ActivityExpiration
alias Pleroma.Config
alias Pleroma.User
alias Pleroma.Web.CommonAPI
require Logger
@interval :timer.minutes(1)
@impl Oban.Worker
def perform(_job) do
if Config.get([ActivityExpiration, :enabled]) do
Enum.each(ActivityExpiration.due_expirations(@interval), &delete_activity/1)
end
after
:ok
end
def delete_activity(%ActivityExpiration{activity_id: activity_id}) do
with {:activity, %Activity{} = activity} <-
{:activity, Activity.get_by_id_with_object(activity_id)},
{:user, %User{} = user} <- {:user, User.get_by_ap_id(activity.object.data["actor"])} do
CommonAPI.delete(activity.id, user)
else
{:activity, _} ->
Logger.error(
"#{__MODULE__} Couldn't delete expired activity: not found activity ##{activity_id}"
)
{:user, _} ->
Logger.error(
"#{__MODULE__} Couldn't delete expired activity: not found actor of ##{activity_id}"
)
end
end
end

72
lib/pleroma/workers/purge_expired_activity.ex Normal file
View file

@ -0,0 +1,72 @@
defmodule Pleroma.Workers.PurgeExpiredActivity do
@moduledoc """
Worker which purges expired activity.
"""
use Oban.Worker, queue: :activity_expiration, max_attempts: 1
import Ecto.Query
def enqueue(args) do
with true <- enabled?(),
args when is_map(args) <- validate_expires_at(args) do
{scheduled_at, args} = Map.pop(args, :expires_at)
args
|> __MODULE__.new(scheduled_at: scheduled_at)
|> Oban.insert()
end
end
@impl true
def perform(%Oban.Job{args: %{"activity_id" => id}}) do
with %Pleroma.Activity{} = activity <- find_activity(id),
%Pleroma.User{} = user <- find_user(activity.object.data["actor"]) do
Pleroma.Web.CommonAPI.delete(activity.id, user)
end
end
defp enabled? do
with false <- Pleroma.Config.get([__MODULE__, :enabled], false) do
{:error, :expired_activities_disabled}
end
end
defp validate_expires_at(%{validate: false} = args), do: Map.delete(args, :validate)
defp validate_expires_at(args) do
if expires_late_enough?(args[:expires_at]) do
args
else
{:error, :expiration_too_close}
end
end
defp find_activity(id) do
with nil <- Pleroma.Activity.get_by_id_with_object(id) do
{:error, :activity_not_found}
end
end
defp find_user(ap_id) do
with nil <- Pleroma.User.get_by_ap_id(ap_id) do
{:error, :user_not_found}
end
end
def get_expiration(id) do
from(j in Oban.Job,
where: j.state == "scheduled",
where: j.queue == "activity_expiration",
where: fragment("?->>'activity_id' = ?", j.args, ^id)
)
|> Pleroma.Repo.one()
end
@spec expires_late_enough?(DateTime.t()) :: boolean()
def expires_late_enough?(scheduled_at) do
now = DateTime.utc_now()
diff = DateTime.diff(scheduled_at, now, :millisecond)
diff > :timer.hours(1)
end
end