[#1149] Oban jobs implementation for :federator_incoming and :federator_outgoing queues.
This commit is contained in:
Ivan Tashkinov
parent
23d279e03e
commit
b7fad8d395
13 changed files with 280 additions and 138 deletions
|
|
@ -3,22 +3,15 @@
|
|||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.Web.Federator do
|
||||
alias Pleroma.Activity
|
||||
alias Pleroma.Object.Containment
|
||||
alias Pleroma.User
|
||||
alias Pleroma.Web.ActivityPub.ActivityPub
|
||||
alias Pleroma.Web.ActivityPub.Transmogrifier
|
||||
alias Pleroma.Web.ActivityPub.Utils
|
||||
alias Pleroma.Web.Federator.Publisher
|
||||
alias Pleroma.Web.OStatus
|
||||
alias Pleroma.Web.Websub
|
||||
alias Pleroma.Workers.Publisher, as: PublisherWorker
|
||||
alias Pleroma.Workers.Receiver, as: ReceiverWorker
|
||||
alias Pleroma.Workers.Subscriber, as: SubscriberWorker
|
||||
|
||||
require Logger
|
||||
|
||||
def init do
|
||||
# 1 minute
|
||||
Process.sleep(1000 * 60)
|
||||
refresh_subscriptions()
|
||||
refresh_subscriptions(schedule_in: 60)
|
||||
end
|
||||
|
||||
@doc "Addresses [memory leaks on recursive replies fetching](https://git.pleroma.social/pleroma/pleroma/issues/161)"
|
||||
|
|
@ -36,111 +29,50 @@ defmodule Pleroma.Web.Federator do
|
|||
# Client API
|
||||
|
||||
def incoming_doc(doc) do
|
||||
PleromaJobQueue.enqueue(:federator_incoming, __MODULE__, [:incoming_doc, doc])
|
||||
%{"op" => "incoming_doc", "body" => doc}
|
||||
|> ReceiverWorker.new(worker_args(:federator_incoming))
|
||||
|> Pleroma.Repo.insert()
|
||||
end
|
||||
|
||||
def incoming_ap_doc(params) do
|
||||
PleromaJobQueue.enqueue(:federator_incoming, __MODULE__, [:incoming_ap_doc, params])
|
||||
%{"op" => "incoming_ap_doc", "params" => params}
|
||||
|> ReceiverWorker.new(worker_args(:federator_incoming))
|
||||
|> Pleroma.Repo.insert()
|
||||
end
|
||||
|
||||
def publish(activity, priority \\ 1) do
|
||||
PleromaJobQueue.enqueue(:federator_outgoing, __MODULE__, [:publish, activity], priority)
|
||||
def publish(%{id: "pleroma:fakeid"} = activity) do
|
||||
PublisherWorker.perform_publish(activity)
|
||||
end
|
||||
|
||||
def publish(activity) do
|
||||
%{"op" => "publish", "activity_id" => activity.id}
|
||||
|> PublisherWorker.new(worker_args(:federator_outgoing))
|
||||
|> Pleroma.Repo.insert()
|
||||
end
|
||||
|
||||
def verify_websub(websub) do
|
||||
PleromaJobQueue.enqueue(:federator_outgoing, __MODULE__, [:verify_websub, websub])
|
||||
%{"op" => "verify_websub", "websub_id" => websub.id}
|
||||
|> SubscriberWorker.new(worker_args(:federator_outgoing))
|
||||
|> Pleroma.Repo.insert()
|
||||
end
|
||||
|
||||
def request_subscription(sub) do
|
||||
PleromaJobQueue.enqueue(:federator_outgoing, __MODULE__, [:request_subscription, sub])
|
||||
def request_subscription(websub) do
|
||||
%{"op" => "request_subscription", "websub_id" => websub.id}
|
||||
|> SubscriberWorker.new(worker_args(:federator_outgoing))
|
||||
|> Pleroma.Repo.insert()
|
||||
end
|
||||
|
||||
def refresh_subscriptions do
|
||||
PleromaJobQueue.enqueue(:federator_outgoing, __MODULE__, [:refresh_subscriptions])
|
||||
def refresh_subscriptions(worker_args \\ []) do
|
||||
%{"op" => "refresh_subscriptions"}
|
||||
|> SubscriberWorker.new(worker_args ++ [max_attempts: 1] ++ worker_args(:federator_outgoing))
|
||||
|> Pleroma.Repo.insert()
|
||||
end
|
||||
|
||||
# Job Worker Callbacks
|
||||
|
||||
def perform(:refresh_subscriptions) do
|
||||
Logger.debug("Federator running refresh subscriptions")
|
||||
Websub.refresh_subscriptions()
|
||||
|
||||
spawn(fn ->
|
||||
# 6 hours
|
||||
Process.sleep(1000 * 60 * 60 * 6)
|
||||
refresh_subscriptions()
|
||||
end)
|
||||
end
|
||||
|
||||
def perform(:request_subscription, websub) do
|
||||
Logger.debug("Refreshing #{websub.topic}")
|
||||
|
||||
with {:ok, websub} <- Websub.request_subscription(websub) do
|
||||
Logger.debug("Successfully refreshed #{websub.topic}")
|
||||
defp worker_args(queue) do
|
||||
if max_attempts = Pleroma.Config.get([:workers, :retries, queue]) do
|
||||
[max_attempts: max_attempts]
|
||||
else
|
||||
_e -> Logger.debug("Couldn't refresh #{websub.topic}")
|
||||
end
|
||||
end
|
||||
|
||||
def perform(:publish, activity) do
|
||||
Logger.debug(fn -> "Running publish for #{activity.data["id"]}" end)
|
||||
|
||||
with %User{} = actor <- User.get_cached_by_ap_id(activity.data["actor"]),
|
||||
{:ok, actor} <- User.ensure_keys_present(actor) do
|
||||
Publisher.publish(actor, activity)
|
||||
end
|
||||
end
|
||||
|
||||
def perform(:verify_websub, websub) do
|
||||
Logger.debug(fn ->
|
||||
"Running WebSub verification for #{websub.id} (#{websub.topic}, #{websub.callback})"
|
||||
end)
|
||||
|
||||
Websub.verify(websub)
|
||||
end
|
||||
|
||||
def perform(:incoming_doc, doc) do
|
||||
Logger.info("Got document, trying to parse")
|
||||
OStatus.handle_incoming(doc)
|
||||
end
|
||||
|
||||
def perform(:incoming_ap_doc, params) do
|
||||
Logger.info("Handling incoming AP activity")
|
||||
|
||||
params = Utils.normalize_params(params)
|
||||
|
||||
# NOTE: we use the actor ID to do the containment, this is fine because an
|
||||
# actor shouldn't be acting on objects outside their own AP server.
|
||||
with {:ok, _user} <- ap_enabled_actor(params["actor"]),
|
||||
nil <- Activity.normalize(params["id"]),
|
||||
:ok <- Containment.contain_origin_from_id(params["actor"], params),
|
||||
{:ok, activity} <- Transmogrifier.handle_incoming(params) do
|
||||
{:ok, activity}
|
||||
else
|
||||
%Activity{} ->
|
||||
Logger.info("Already had #{params["id"]}")
|
||||
:error
|
||||
|
||||
_e ->
|
||||
# Just drop those for now
|
||||
Logger.info("Unhandled activity")
|
||||
Logger.info(Jason.encode!(params, pretty: true))
|
||||
:error
|
||||
end
|
||||
end
|
||||
|
||||
def perform(type, _) do
|
||||
Logger.debug(fn -> "Unknown task: #{type}" end)
|
||||
{:error, "Don't know what to do with this"}
|
||||
end
|
||||
|
||||
def ap_enabled_actor(id) do
|
||||
user = User.get_cached_by_ap_id(id)
|
||||
|
||||
if User.ap_enabled?(user) do
|
||||
{:ok, user}
|
||||
else
|
||||
ActivityPub.make_user_from_ap_id(id)
|
||||
[]
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ defmodule Pleroma.Web.Federator.Publisher do
|
|||
alias Pleroma.Activity
|
||||
alias Pleroma.Config
|
||||
alias Pleroma.User
|
||||
alias Pleroma.Workers.Publisher, as: PublisherWorker
|
||||
|
||||
require Logger
|
||||
|
||||
|
|
@ -30,8 +31,15 @@ defmodule Pleroma.Web.Federator.Publisher do
|
|||
"""
|
||||
@spec enqueue_one(module(), Map.t()) :: :ok
|
||||
def enqueue_one(module, %{} = params) do
|
||||
%{module: to_string(module), params: params}
|
||||
|> Pleroma.Workers.Publisher.new()
|
||||
worker_args =
|
||||
if max_attempts = Pleroma.Config.get([:workers, :retries, :federator_outgoing]) do
|
||||
[max_attempts: max_attempts]
|
||||
else
|
||||
[]
|
||||
end
|
||||
|
||||
%{"op" => "publish_one", "module" => to_string(module), "params" => params}
|
||||
|> PublisherWorker.new(worker_args)
|
||||
|> Pleroma.Repo.insert()
|
||||
end
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue