hj/pleroma
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge branch 'develop' into issue/733

Browse source
This commit is contained in:
Maksim Pechnikov 2019-09-16 07:34:22 +03:00
commit d75bc728e7
81 changed files with 4534 additions and 1014 deletions
Download patch file Download diff file Expand all files Collapse all files

3
lib/pleroma/web/activity_pub/activity_pub.ex
View file

@ -17,6 +17,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
alias Pleroma.Web.ActivityPub.MRF
alias Pleroma.Web.ActivityPub.Transmogrifier
alias Pleroma.Web.WebFinger
alias Pleroma.Workers.BackgroundWorker
import Ecto.Query
import Pleroma.Web.ActivityPub.Utils
@ -145,7 +146,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
activity
end
PleromaJobQueue.enqueue(:background, Pleroma.Web.RichMedia.Helpers, [:fetch, activity])
BackgroundWorker.enqueue("fetch_data_for_activity", %{"activity_id" => activity.id})
Notification.create_notifications(activity)

5
lib/pleroma/web/activity_pub/mrf/mediaproxy_warming_policy.ex
View file

@ -8,6 +8,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do
alias Pleroma.HTTP
alias Pleroma.Web.MediaProxy
alias Pleroma.Workers.BackgroundWorker
require Logger
@ -30,7 +31,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do
url
|> Enum.each(fn
%{"href" => href} ->
PleromaJobQueue.enqueue(:background, __MODULE__, [:prefetch, href])
BackgroundWorker.enqueue("media_proxy_prefetch", %{"url" => href})
x ->
Logger.debug("Unhandled attachment URL object #{inspect(x)}")
@ -46,7 +47,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do
%{"type" => "Create", "object" => %{"attachment" => attachments} = _object} = message
)
when is_list(attachments) and length(attachments) > 0 do
PleromaJobQueue.enqueue(:background, __MODULE__, [:preload, message])
BackgroundWorker.enqueue("media_proxy_preload", %{"message" => message})
{:ok, message}
end

16
lib/pleroma/web/activity_pub/publisher.ex
View file

@ -84,6 +84,15 @@ defmodule Pleroma.Web.ActivityPub.Publisher do
end
end
def publish_one(%{actor_id: actor_id} = params) do
actor = User.get_cached_by_id(actor_id)
params
|> Map.delete(:actor_id)
|> Map.put(:actor, actor)
|> publish_one()
end
defp should_federate?(inbox, public) do
if public do
true
@ -159,7 +168,8 @@ defmodule Pleroma.Web.ActivityPub.Publisher do
Publishes an activity with BCC to all relevant peers.
"""
def publish(actor, %{data: %{"bcc" => bcc}} = activity) when is_list(bcc) and bcc != [] do
def publish(%User{} = actor, %{data: %{"bcc" => bcc}} = activity)
when is_list(bcc) and bcc != [] do
public = is_public?(activity)
{:ok, data} = Transmogrifier.prepare_outgoing(activity.data)
@ -186,7 +196,7 @@ defmodule Pleroma.Web.ActivityPub.Publisher do
Pleroma.Web.Federator.Publisher.enqueue_one(__MODULE__, %{
inbox: inbox,
json: json,
actor: actor,
actor_id: actor.id,
id: activity.data["id"],
unreachable_since: unreachable_since
})
@ -221,7 +231,7 @@ defmodule Pleroma.Web.ActivityPub.Publisher do
%{
inbox: inbox,
json: json,
actor: actor,
actor_id: actor.id,
id: activity.data["id"],
unreachable_since: unreachable_since
}

7
lib/pleroma/web/activity_pub/transmogrifier.ex
View file

@ -15,6 +15,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
alias Pleroma.Web.ActivityPub.Utils
alias Pleroma.Web.ActivityPub.Visibility
alias Pleroma.Web.Federator
alias Pleroma.Workers.TransmogrifierWorker
import Ecto.Query
@ -185,12 +186,12 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
|> Map.put("context", replied_object.data["context"] || object["conversation"])
else
e ->
Logger.error("Couldn't fetch \"#{inspect(in_reply_to_id)}\", error: #{inspect(e)}")
Logger.error("Couldn't fetch #{inspect(in_reply_to_id)}, error: #{inspect(e)}")
object
end
e ->
Logger.error("Couldn't fetch \"#{inspect(in_reply_to_id)}\", error: #{inspect(e)}")
Logger.error("Couldn't fetch #{inspect(in_reply_to_id)}, error: #{inspect(e)}")
object
end
else
@ -1051,7 +1052,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
already_ap <- User.ap_enabled?(user),
{:ok, user} <- upgrade_user(user, data) do
if not already_ap do
PleromaJobQueue.enqueue(:transmogrifier, __MODULE__, [:user_upgrade, user])
TransmogrifierWorker.enqueue("user_upgrade", %{"user_id" => user.id})
end
{:ok, user}

176
lib/pleroma/web/activity_pub/utils.ex
View file

@ -85,15 +85,13 @@ defmodule Pleroma.Web.ActivityPub.Utils do
defp extract_list(_), do: []
def maybe_splice_recipient(ap_id, params) do
need_splice =
need_splice? =
!recipient_in_collection(ap_id, params["to"]) &&
!recipient_in_collection(ap_id, params["cc"])
cc_list = extract_list(params["cc"])
if need_splice do
params
|> Map.put("cc", [ap_id | cc_list])
if need_splice? do
cc_list = extract_list(params["cc"])
Map.put(params, "cc", [ap_id | cc_list])
else
params
end
@ -139,7 +137,7 @@ defmodule Pleroma.Web.ActivityPub.Utils do
"object" => object
}
Notification.get_notified_from_activity(%Activity{data: fake_create_activity}, false)
get_notified_from_object(fake_create_activity)
end
def get_notified_from_object(object) do
@ -169,14 +167,7 @@ defmodule Pleroma.Web.ActivityPub.Utils do
@spec maybe_federate(any()) :: :ok
def maybe_federate(%Activity{local: true} = activity) do
if Pleroma.Config.get!([:instance, :federating]) do
priority =
case activity.data["type"] do
"Delete" -> 10
"Create" -> 1
_ -> 5
end
Pleroma.Web.Federator.publish(activity, priority)
Pleroma.Web.Federator.publish(activity)
end
:ok
@ -188,9 +179,9 @@ defmodule Pleroma.Web.ActivityPub.Utils do
Adds an id and a published data if they aren't there,
also adds it to an included object
"""
def lazy_put_activity_defaults(map, fake \\ false) do
def lazy_put_activity_defaults(map, fake? \\ false) do
map =
unless fake do
if not fake? do
%{data: %{"id" => context}, id: context_id} = create_context(map["context"])
map
@ -207,7 +198,7 @@ defmodule Pleroma.Web.ActivityPub.Utils do
end
if is_map(map["object"]) do
object = lazy_put_object_defaults(map["object"], map, fake)
object = lazy_put_object_defaults(map["object"], map, fake?)
%{map | "object" => object}
else
map
@ -217,9 +208,9 @@ defmodule Pleroma.Web.ActivityPub.Utils do
@doc """
Adds an id and published date if they aren't there.
"""
def lazy_put_object_defaults(map, activity \\ %{}, fake)
def lazy_put_object_defaults(map, activity \\ %{}, fake?)
def lazy_put_object_defaults(map, activity, true = _fake) do
def lazy_put_object_defaults(map, activity, true = _fake?) do
map
|> Map.put_new_lazy("published", &make_date/0)
|> Map.put_new("id", "pleroma:fake_object_id")
@ -228,7 +219,7 @@ defmodule Pleroma.Web.ActivityPub.Utils do
|> Map.put_new("context_id", activity["context_id"])
end
def lazy_put_object_defaults(map, activity, _fake) do
def lazy_put_object_defaults(map, activity, _fake?) do
map
|> Map.put_new_lazy("id", &generate_object_id/0)
|> Map.put_new_lazy("published", &make_date/0)
@ -242,9 +233,7 @@ defmodule Pleroma.Web.ActivityPub.Utils do
def insert_full_object(%{"object" => %{"type" => type} = object_data} = map)
when is_map(object_data) and type in @supported_object_types do
with {:ok, object} <- Object.create(object_data) do
map =
map
|> Map.put("object", object.data["id"])
map = Map.put(map, "object", object.data["id"])
{:ok, map, object}
end
@ -263,7 +252,7 @@ defmodule Pleroma.Web.ActivityPub.Utils do
|> Activity.Queries.by_actor()
|> Activity.Queries.by_object_id(id)
|> Activity.Queries.by_type("Like")
|> Activity.Queries.limit(1)
|> limit(1)
|> Repo.one()
end
@ -380,12 +369,11 @@ defmodule Pleroma.Web.ActivityPub.Utils do
%Activity{data: %{"actor" => actor, "object" => object}} = activity,
state
) do
with new_data <-
activity.data
|> Map.put("state", state),
changeset <- Changeset.change(activity, data: new_data),
{:ok, activity} <- Repo.update(changeset),
_ <- User.set_follow_state_cache(actor, object, state) do
new_data = Map.put(activity.data, "state", state)
changeset = Changeset.change(activity, data: new_data)
with {:ok, activity} <- Repo.update(changeset) do
User.set_follow_state_cache(actor, object, state)
{:ok, activity}
end
end
@ -410,28 +398,14 @@ defmodule Pleroma.Web.ActivityPub.Utils do
end
def fetch_latest_follow(%User{ap_id: follower_id}, %User{ap_id: followed_id}) do
query =
from(
activity in Activity,
where:
fragment(
"? ->> 'type' = 'Follow'",
activity.data
),
where: activity.actor == ^follower_id,
# this is to use the index
where:
fragment(
"coalesce((?)->'object'->>'id', (?)->>'object') = ?",
activity.data,
activity.data,
^followed_id
),
order_by: [fragment("? desc nulls last", activity.id)],
limit: 1
)
Repo.one(query)
"Follow"
|> Activity.Queries.by_type()
|> where(actor: ^follower_id)
# this is to use the index
|> Activity.Queries.by_object_id(followed_id)
|> order_by([activity], fragment("? desc nulls last", activity.id))
|> limit(1)
|> Repo.one()
end
#### Announce-related helpers
@ -439,23 +413,13 @@ defmodule Pleroma.Web.ActivityPub.Utils do
@doc """
Retruns an existing announce activity if the notice has already been announced
"""
def get_existing_announce(actor, %{data: %{"id" => id}}) do
query =
from(
activity in Activity,
where: activity.actor == ^actor,
# this is to use the index
where:
fragment(
"coalesce((?)->'object'->>'id', (?)->>'object') = ?",
activity.data,
activity.data,
^id
),
where: fragment("(?)->>'type' = 'Announce'", activity.data)
)
Repo.one(query)
def get_existing_announce(actor, %{data: %{"id" => ap_id}}) do
"Announce"
|> Activity.Queries.by_type()
|> where(actor: ^actor)
# this is to use the index
|> Activity.Queries.by_object_id(ap_id)
|> Repo.one()
end
@doc """
@ -538,11 +502,13 @@ defmodule Pleroma.Web.ActivityPub.Utils do
object
) do
announcements =
if is_list(object.data["announcements"]), do: object.data["announcements"], else: []
if is_list(object.data["announcements"]) do
Enum.uniq([actor | object.data["announcements"]])
else
[actor]
end
with announcements <- [actor | announcements] |> Enum.uniq() do
update_element_in_object("announcement", announcements, object)
end
update_element_in_object("announcement", announcements, object)
end
def add_announce_to_object(_, object), do: {:ok, object}
@ -570,28 +536,14 @@ defmodule Pleroma.Web.ActivityPub.Utils do
#### Block-related helpers
def fetch_latest_block(%User{ap_id: blocker_id}, %User{ap_id: blocked_id}) do
query =
from(
activity in Activity,
where:
fragment(
"? ->> 'type' = 'Block'",
activity.data
),
where: activity.actor == ^blocker_id,
# this is to use the index
where:
fragment(
"coalesce((?)->'object'->>'id', (?)->>'object') = ?",
activity.data,
activity.data,
^blocked_id
),
order_by: [fragment("? desc nulls last", activity.id)],
limit: 1
)
Repo.one(query)
"Block"
|> Activity.Queries.by_type()
|> where(actor: ^blocker_id)
# this is to use the index
|> Activity.Queries.by_object_id(blocked_id)
|> order_by([activity], fragment("? desc nulls last", activity.id))
|> limit(1)
|> Repo.one()
end
def make_block_data(blocker, blocked, activity_id) do
@ -695,11 +647,11 @@ defmodule Pleroma.Web.ActivityPub.Utils do
#### Report-related helpers
def update_report_state(%Activity{} = activity, state) when state in @supported_report_states do
with new_data <- Map.put(activity.data, "state", state),
changeset <- Changeset.change(activity, data: new_data),
{:ok, activity} <- Repo.update(changeset) do
{:ok, activity}
end
new_data = Map.put(activity.data, "state", state)
activity
|> Changeset.change(data: new_data)
|> Repo.update()
end
def update_report_state(_, _), do: {:error, "Unsupported state"}
@ -766,21 +718,13 @@ defmodule Pleroma.Web.ActivityPub.Utils do
end
def get_existing_votes(actor, %{data: %{"id" => id}}) do
query =
from(
[activity, object: object] in Activity.with_preloaded_object(Activity),
where: fragment("(?)->>'type' = 'Create'", activity.data),
where: fragment("(?)->>'actor' = ?", activity.data, ^actor),
where:
fragment(
"(?)->>'inReplyTo' = ?",
object.data,
^to_string(id)
),
where: fragment("(?)->>'type' = 'Answer'", object.data)
)
Repo.all(query)
actor
|> Activity.Queries.by_actor()
|> Activity.Queries.by_type("Create")
|> Activity.with_preloaded_object()
|> where([a, object: o], fragment("(?)->>'inReplyTo' = ?", o.data, ^to_string(id)))
|> where([a, object: o], fragment("(?)->>'type' = 'Answer'", o.data))
|> Repo.all()
end
defp maybe_put(map, _key, nil), do: map

15
lib/pleroma/web/admin_api/config.ex
View file

@ -90,6 +90,8 @@ defmodule Pleroma.Web.AdminAPI.Config do
for v <- entity, into: [], do: do_convert(v)
end
defp do_convert(%Regex{} = entity), do: inspect(entity)
defp do_convert(entity) when is_map(entity) do
for {k, v} <- entity, into: %{}, do: {do_convert(k), do_convert(v)}
end
@ -122,7 +124,7 @@ defmodule Pleroma.Web.AdminAPI.Config do
def transform(entity), do: :erlang.term_to_binary(entity)
defp do_transform(%Regex{} = entity) when is_map(entity), do: entity
defp do_transform(%Regex{} = entity), do: entity
defp do_transform(%{"tuple" => [":dispatch", [entity]]}) do
{dispatch_settings, []} = do_eval(entity)
@ -154,8 +156,15 @@ defmodule Pleroma.Web.AdminAPI.Config do
defp do_transform(entity), do: entity
defp do_transform_string("~r/" <> pattern) do
pattern = String.trim_trailing(pattern, "/")
~r/#{pattern}/
modificator = String.split(pattern, "/") |> List.last()
pattern = String.trim_trailing(pattern, "/" <> modificator)
case modificator do
"" -> ~r/#{pattern}/
"i" -> ~r/#{pattern}/i
"u" -> ~r/#{pattern}/u
"s" -> ~r/#{pattern}/s
end
end
defp do_transform_string(":" <> atom), do: String.to_atom(atom)

95
lib/pleroma/web/controller_helper.ex
View file

@ -34,79 +34,38 @@ defmodule Pleroma.Web.ControllerHelper do
defp param_to_integer(_, default), do: default
def add_link_headers(
conn,
method,
activities,
param \\ nil,
params \\ %{},
func3 \\ nil,
func4 \\ nil
) do
params =
conn.params
|> Map.drop(["since_id", "max_id", "min_id"])
|> Map.merge(params)
def add_link_headers(conn, activities, extra_params \\ %{}) do
case List.last(activities) do
%{id: max_id} ->
params =
conn.params
|> Map.drop(Map.keys(conn.path_params))
|> Map.drop(["since_id", "max_id", "min_id"])
|> Map.merge(extra_params)
last = List.last(activities)
limit =
params
|> Map.get("limit", "20")
|> String.to_integer()
func3 = func3 || (&mastodon_api_url/3)
func4 = func4 || (&mastodon_api_url/4)
min_id =
if length(activities) <= limit do
activities
|> List.first()
|> Map.get(:id)
else
activities
|> Enum.at(limit * -1)
|> Map.get(:id)
end
if last do
max_id = last.id
next_url = current_url(conn, Map.merge(params, %{max_id: max_id}))
prev_url = current_url(conn, Map.merge(params, %{min_id: min_id}))
limit =
params
|> Map.get("limit", "20")
|> String.to_integer()
put_resp_header(conn, "link", "<#{next_url}>; rel=\"next\", <#{prev_url}>; rel=\"prev\"")
min_id =
if length(activities) <= limit do
activities
|> List.first()
|> Map.get(:id)
else
activities
|> Enum.at(limit * -1)
|> Map.get(:id)
end
{next_url, prev_url} =
if param do
{
func4.(
Pleroma.Web.Endpoint,
method,
param,
Map.merge(params, %{max_id: max_id})
),
func4.(
Pleroma.Web.Endpoint,
method,
param,
Map.merge(params, %{min_id: min_id})
)
}
else
{
func3.(
Pleroma.Web.Endpoint,
method,
Map.merge(params, %{max_id: max_id})
),
func3.(
Pleroma.Web.Endpoint,
method,
Map.merge(params, %{min_id: min_id})
)
}
end
conn
|> put_resp_header("link", "<#{next_url}>; rel=\"next\", <#{prev_url}>; rel=\"prev\"")
else
conn
_ ->
conn
end
end
end

90
lib/pleroma/web/federator/federator.ex
View file

@ -10,16 +10,17 @@ defmodule Pleroma.Web.Federator do
alias Pleroma.Web.ActivityPub.Transmogrifier
alias Pleroma.Web.ActivityPub.Utils
alias Pleroma.Web.Federator.Publisher
alias Pleroma.Web.Federator.RetryQueue
alias Pleroma.Web.OStatus
alias Pleroma.Web.Websub
alias Pleroma.Workers.PublisherWorker
alias Pleroma.Workers.ReceiverWorker
alias Pleroma.Workers.SubscriberWorker
require Logger
def init do
# 1 minute
Process.sleep(1000 * 60)
refresh_subscriptions()
# To do: consider removing this call in favor of scheduled execution (`quantum`-based)
refresh_subscriptions(schedule_in: 60)
end
@doc "Addresses [memory leaks on recursive replies fetching](https://git.pleroma.social/pleroma/pleroma/issues/161)"
@ -37,50 +38,38 @@ defmodule Pleroma.Web.Federator do
# Client API
def incoming_doc(doc) do
PleromaJobQueue.enqueue(:federator_incoming, __MODULE__, [:incoming_doc, doc])
ReceiverWorker.enqueue("incoming_doc", %{"body" => doc})
end
def incoming_ap_doc(params) do
PleromaJobQueue.enqueue(:federator_incoming, __MODULE__, [:incoming_ap_doc, params])
ReceiverWorker.enqueue("incoming_ap_doc", %{"params" => params})
end
def publish(activity, priority \\ 1) do
PleromaJobQueue.enqueue(:federator_outgoing, __MODULE__, [:publish, activity], priority)
def publish(%{id: "pleroma:fakeid"} = activity) do
perform(:publish, activity)
end
def publish(activity) do
PublisherWorker.enqueue("publish", %{"activity_id" => activity.id})
end
def verify_websub(websub) do
PleromaJobQueue.enqueue(:federator_outgoing, __MODULE__, [:verify_websub, websub])
SubscriberWorker.enqueue("verify_websub", %{"websub_id" => websub.id})
end
def request_subscription(sub) do
PleromaJobQueue.enqueue(:federator_outgoing, __MODULE__, [:request_subscription, sub])
def request_subscription(websub) do
SubscriberWorker.enqueue("request_subscription", %{"websub_id" => websub.id})
end
def refresh_subscriptions do
PleromaJobQueue.enqueue(:federator_outgoing, __MODULE__, [:refresh_subscriptions])
def refresh_subscriptions(worker_args \\ []) do
SubscriberWorker.enqueue("refresh_subscriptions", %{}, worker_args ++ [max_attempts: 1])
end
# Job Worker Callbacks
def perform(:refresh_subscriptions) do
Logger.debug("Federator running refresh subscriptions")
Websub.refresh_subscriptions()
spawn(fn ->
# 6 hours
Process.sleep(1000 * 60 * 60 * 6)
refresh_subscriptions()
end)
end
def perform(:request_subscription, websub) do
Logger.debug("Refreshing #{websub.topic}")
with {:ok, websub} <- Websub.request_subscription(websub) do
Logger.debug("Successfully refreshed #{websub.topic}")
else
_e -> Logger.debug("Couldn't refresh #{websub.topic}")
end
@spec perform(atom(), module(), any()) :: {:ok, any()} | {:error, any()}
def perform(:publish_one, module, params) do
apply(module, :publish_one, [params])
end
def perform(:publish, activity) do
@ -92,14 +81,6 @@ defmodule Pleroma.Web.Federator do
end
end
def perform(:verify_websub, websub) do
Logger.debug(fn ->
"Running WebSub verification for #{websub.id} (#{websub.topic}, #{websub.callback})"
end)
Websub.verify(websub)
end
def perform(:incoming_doc, doc) do
Logger.info("Got document, trying to parse")
OStatus.handle_incoming(doc)
@ -130,22 +111,27 @@ defmodule Pleroma.Web.Federator do
end
end
def perform(
:publish_single_websub,
%{xml: _xml, topic: _topic, callback: _callback, secret: _secret} = params
) do
case Websub.publish_one(params) do
{:ok, _} ->
:ok
def perform(:request_subscription, websub) do
Logger.debug("Refreshing #{websub.topic}")
{:error, _} ->
RetryQueue.enqueue(params, Websub)
with {:ok, websub} <- Websub.request_subscription(websub) do
Logger.debug("Successfully refreshed #{websub.topic}")
else
_e -> Logger.debug("Couldn't refresh #{websub.topic}")
end
end
def perform(type, _) do
Logger.debug(fn -> "Unknown task: #{type}" end)
{:error, "Don't know what to do with this"}
def perform(:verify_websub, websub) do
Logger.debug(fn ->
"Running WebSub verification for #{websub.id} (#{websub.topic}, #{websub.callback})"
end)
Websub.verify(websub)
end
def perform(:refresh_subscriptions) do
Logger.debug("Federator running refresh subscriptions")
Websub.refresh_subscriptions()
end
def ap_enabled_actor(id) do

24
lib/pleroma/web/federator/publisher.ex
View file

@ -6,7 +6,7 @@ defmodule Pleroma.Web.Federator.Publisher do
alias Pleroma.Activity
alias Pleroma.Config
alias Pleroma.User
alias Pleroma.Web.Federator.RetryQueue
alias Pleroma.Workers.PublisherWorker
require Logger
@ -30,23 +30,11 @@ defmodule Pleroma.Web.Federator.Publisher do
Enqueue publishing a single activity.
"""
@spec enqueue_one(module(), Map.t()) :: :ok
def enqueue_one(module, %{} = params),
do: PleromaJobQueue.enqueue(:federator_outgoing, __MODULE__, [:publish_one, module, params])
@spec perform(atom(), module(), any()) :: {:ok, any()} | {:error, any()}
def perform(:publish_one, module, params) do
case apply(module, :publish_one, [params]) do
{:ok, _} ->
:ok
{:error, _e} ->
RetryQueue.enqueue(params, module)
end
end
def perform(type, _, _) do
Logger.debug("Unknown task: #{type}")
{:error, "Don't know what to do with this"}
def enqueue_one(module, %{} = params) do
PublisherWorker.enqueue(
"publish_one",
%{"module" => to_string(module), "params" => params}
)
end
@doc """

239
lib/pleroma/web/federator/retry_queue.ex
View file

@ -1,239 +0,0 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.Federator.RetryQueue do
use GenServer
require Logger
def init(args) do
queue_table = :ets.new(:pleroma_retry_queue, [:bag, :protected])
{:ok, %{args | queue_table: queue_table, running_jobs: :sets.new()}}
end
def start_link(_) do
enabled =
if Pleroma.Config.get(:env) == :test,
do: true,
else: Pleroma.Config.get([__MODULE__, :enabled], false)
if enabled do
Logger.info("Starting retry queue")
linkres =
GenServer.start_link(
__MODULE__,
%{delivered: 0, dropped: 0, queue_table: nil, running_jobs: nil},
name: __MODULE__
)
maybe_kickoff_timer()
linkres
else
Logger.info("Retry queue disabled")
:ignore
end
end
def enqueue(data, transport, retries \\ 0) do
GenServer.cast(__MODULE__, {:maybe_enqueue, data, transport, retries + 1})
end
def get_stats do
GenServer.call(__MODULE__, :get_stats)
end
def reset_stats do
GenServer.call(__MODULE__, :reset_stats)
end
def get_retry_params(retries) do
if retries > Pleroma.Config.get([__MODULE__, :max_retries]) do
{:drop, "Max retries reached"}
else
{:retry, growth_function(retries)}
end
end
def get_retry_timer_interval do
Pleroma.Config.get([:retry_queue, :interval], 1000)
end
defp ets_count_expires(table, current_time) do
:ets.select_count(
table,
[
{
{:"$1", :"$2"},
[{:"=<", :"$1", {:const, current_time}}],
[true]
}
]
)
end
defp ets_pop_n_expired(table, current_time, desired) do
{popped, _continuation} =
:ets.select(
table,
[
{
{:"$1", :"$2"},
[{:"=<", :"$1", {:const, current_time}}],
[:"$_"]
}
],
desired
)
popped
|> Enum.each(fn e ->
:ets.delete_object(table, e)
end)
popped
end
def maybe_start_job(running_jobs, queue_table) do
# we don't want to hit the ets or the DateTime more times than we have to
# could optimize slightly further by not using the count, and instead grabbing
# up to N objects early...
current_time = DateTime.to_unix(DateTime.utc_now())
n_running_jobs = :sets.size(running_jobs)
if n_running_jobs < Pleroma.Config.get([__MODULE__, :max_jobs]) do
n_ready_jobs = ets_count_expires(queue_table, current_time)
if n_ready_jobs > 0 do
# figure out how many we could start
available_job_slots = Pleroma.Config.get([__MODULE__, :max_jobs]) - n_running_jobs
start_n_jobs(running_jobs, queue_table, current_time, available_job_slots)
else
running_jobs
end
else
running_jobs
end
end
defp start_n_jobs(running_jobs, _queue_table, _current_time, 0) do
running_jobs
end
defp start_n_jobs(running_jobs, queue_table, current_time, available_job_slots)
when available_job_slots > 0 do
candidates = ets_pop_n_expired(queue_table, current_time, available_job_slots)
candidates
|> List.foldl(running_jobs, fn {_, e}, rj ->
{:ok, pid} = Task.start(fn -> worker(e) end)
mref = Process.monitor(pid)
:sets.add_element(mref, rj)
end)
end
def worker({:send, data, transport, retries}) do
case transport.publish_one(data) do
{:ok, _} ->
GenServer.cast(__MODULE__, :inc_delivered)
:delivered
{:error, _reason} ->
enqueue(data, transport, retries)
:retry
end
end
def handle_call(:get_stats, _from, %{delivered: delivery_count, dropped: drop_count} = state) do
{:reply, %{delivered: delivery_count, dropped: drop_count}, state}
end
def handle_call(:reset_stats, _from, %{delivered: delivery_count, dropped: drop_count} = state) do
{:reply, %{delivered: delivery_count, dropped: drop_count},
%{state | delivered: 0, dropped: 0}}
end
def handle_cast(:reset_stats, state) do
{:noreply, %{state | delivered: 0, dropped: 0}}
end
def handle_cast(
{:maybe_enqueue, data, transport, retries},
%{dropped: drop_count, queue_table: queue_table, running_jobs: running_jobs} = state
) do
case get_retry_params(retries) do
{:retry, timeout} ->
:ets.insert(queue_table, {timeout, {:send, data, transport, retries}})
running_jobs = maybe_start_job(running_jobs, queue_table)
{:noreply, %{state | running_jobs: running_jobs}}
{:drop, message} ->
Logger.debug(message)
{:noreply, %{state | dropped: drop_count + 1}}
end
end
def handle_cast(:kickoff_timer, state) do
retry_interval = get_retry_timer_interval()
Process.send_after(__MODULE__, :retry_timer_run, retry_interval)
{:noreply, state}
end
def handle_cast(:inc_delivered, %{delivered: delivery_count} = state) do
{:noreply, %{state | delivered: delivery_count + 1}}
end
def handle_cast(:inc_dropped, %{dropped: drop_count} = state) do
{:noreply, %{state | dropped: drop_count + 1}}
end
def handle_info({:send, data, transport, retries}, %{delivered: delivery_count} = state) do
case transport.publish_one(data) do
{:ok, _} ->
{:noreply, %{state | delivered: delivery_count + 1}}
{:error, _reason} ->
enqueue(data, transport, retries)
{:noreply, state}
end
end
def handle_info(
:retry_timer_run,
%{queue_table: queue_table, running_jobs: running_jobs} = state
) do
maybe_kickoff_timer()
running_jobs = maybe_start_job(running_jobs, queue_table)
{:noreply, %{state | running_jobs: running_jobs}}
end
def handle_info({:DOWN, ref, :process, _pid, _reason}, state) do
%{running_jobs: running_jobs, queue_table: queue_table} = state
running_jobs = :sets.del_element(ref, running_jobs)
running_jobs = maybe_start_job(running_jobs, queue_table)
{:noreply, %{state | running_jobs: running_jobs}}
end
def handle_info(unknown, state) do
Logger.debug("RetryQueue: don't know what to do with #{inspect(unknown)}, ignoring")
{:noreply, state}
end
if Pleroma.Config.get(:env) == :test do
defp growth_function(_retries) do
_shutit = Pleroma.Config.get([__MODULE__, :initial_timeout])
DateTime.to_unix(DateTime.utc_now()) - 1
end
else
defp growth_function(retries) do
round(Pleroma.Config.get([__MODULE__, :initial_timeout]) * :math.pow(retries, 3)) +
DateTime.to_unix(DateTime.utc_now())
end
end
defp maybe_kickoff_timer do
GenServer.cast(__MODULE__, :kickoff_timer)
end
end

32
lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex
View file

@ -6,7 +6,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
use Pleroma.Web, :controller
import Pleroma.Web.ControllerHelper,
only: [json_response: 3, add_link_headers: 5, add_link_headers: 4, add_link_headers: 3]
only: [json_response: 3, add_link_headers: 2, add_link_headers: 3]
alias Ecto.Changeset
alias Pleroma.Activity
@ -365,7 +365,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
|> Enum.reverse()
conn
|> add_link_headers(:home_timeline, activities)
|> add_link_headers(activities)
|> put_view(StatusView)
|> render("index.json", %{activities: activities, for: user, as: :activity})
end
@ -384,7 +384,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
|> Enum.reverse()
conn
|> add_link_headers(:public_timeline, activities, false, %{"local" => local_only})
|> add_link_headers(activities, %{"local" => local_only})
|> put_view(StatusView)
|> render("index.json", %{activities: activities, for: user, as: :activity})
end
@ -398,7 +398,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
activities = ActivityPub.fetch_user_activities(user, reading_user, params)
conn
|> add_link_headers(:user_statuses, activities, params["id"])
|> add_link_headers(activities)
|> put_view(StatusView)
|> render("index.json", %{
activities: activities,
@ -422,7 +422,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
|> Pagination.fetch_paginated(params)
conn
|> add_link_headers(:dm_timeline, activities)
|> add_link_headers(activities)
|> put_view(StatusView)
|> render("index.json", %{activities: activities, for: user, as: :activity})
end
@ -537,7 +537,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
def scheduled_statuses(%{assigns: %{user: user}} = conn, params) do
with scheduled_activities <- MastodonAPI.get_scheduled_activities(user, params) do
conn
|> add_link_headers(:scheduled_statuses, scheduled_activities)
|> add_link_headers(scheduled_activities)
|> put_view(ScheduledActivityView)
|> render("index.json", %{scheduled_activities: scheduled_activities})
end
@ -720,7 +720,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
notifications = MastodonAPI.get_notifications(user, params)
conn
|> add_link_headers(:notifications, notifications)
|> add_link_headers(notifications)
|> put_view(NotificationView)
|> render("index.json", %{notifications: notifications, for: user})
end
@ -842,6 +842,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
def favourited_by(%{assigns: %{user: user}} = conn, %{"id" => id}) do
with %Activity{} = activity <- Activity.get_by_id_with_object(id),
{:visible, true} <- {:visible, Visibility.visible_for_user?(activity, user)},
%Object{data: %{"likes" => likes}} <- Object.normalize(activity) do
q = from(u in User, where: u.ap_id in ^likes)
@ -853,12 +854,14 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
|> put_view(AccountView)
|> render("accounts.json", %{for: user, users: users, as: :user})
else
{:visible, false} -> {:error, :not_found}
_ -> json(conn, [])
end
end
def reblogged_by(%{assigns: %{user: user}} = conn, %{"id" => id}) do
with %Activity{} = activity <- Activity.get_by_id_with_object(id),
{:visible, true} <- {:visible, Visibility.visible_for_user?(activity, user)},
%Object{data: %{"announcements" => announces}} <- Object.normalize(activity) do
q = from(u in User, where: u.ap_id in ^announces)
@ -870,6 +873,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
|> put_view(AccountView)
|> render("accounts.json", %{for: user, users: users, as: :user})
else
{:visible, false} -> {:error, :not_found}
_ -> json(conn, [])
end
end
@ -908,7 +912,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
|> Enum.reverse()
conn
|> add_link_headers(:hashtag_timeline, activities, params["tag"], %{"local" => local_only})
|> add_link_headers(activities, %{"local" => local_only})
|> put_view(StatusView)
|> render("index.json", %{activities: activities, for: user, as: :activity})
end
@ -924,7 +928,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
end
conn
|> add_link_headers(:followers, followers, user)
|> add_link_headers(followers)
|> put_view(AccountView)
|> render("accounts.json", %{for: for_user, users: followers, as: :user})
end
@ -941,7 +945,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
end
conn
|> add_link_headers(:following, followers, user)
|> add_link_headers(followers)
|> put_view(AccountView)
|> render("accounts.json", %{for: for_user, users: followers, as: :user})
end
@ -1166,7 +1170,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
|> Enum.reverse()
conn
|> add_link_headers(:favourites, activities)
|> add_link_headers(activities)
|> put_view(StatusView)
|> render("index.json", %{activities: activities, for: user, as: :activity})
end
@ -1193,7 +1197,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
|> Enum.reverse()
conn
|> add_link_headers(:favourites, activities)
|> add_link_headers(activities)
|> put_view(StatusView)
|> render("index.json", %{activities: activities, for: for_user, as: :activity})
else
@ -1214,7 +1218,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
|> Enum.map(fn b -> Map.put(b.activity, :bookmark, Map.delete(b, :activity)) end)
conn
|> add_link_headers(:bookmarks, bookmarks)
|> add_link_headers(bookmarks)
|> put_view(StatusView)
|> render("index.json", %{activities: activities, for: user, as: :activity})
end
@ -1654,7 +1658,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
end)
conn
|> add_link_headers(:conversations, participations)
|> add_link_headers(participations)
|> json(conversations)
end

5
lib/pleroma/web/oauth/token/clean_worker.ex
View file

@ -17,6 +17,7 @@ defmodule Pleroma.Web.OAuth.Token.CleanWorker do
)
alias Pleroma.Web.OAuth.Token
alias Pleroma.Workers.BackgroundWorker
def start_link(_), do: GenServer.start_link(__MODULE__, %{})
@ -27,9 +28,11 @@ defmodule Pleroma.Web.OAuth.Token.CleanWorker do
@doc false
def handle_info(:perform, state) do
Token.delete_expired_tokens()
BackgroundWorker.enqueue("clean_expired_tokens", %{})
Process.send_after(self(), :perform, @interval)
{:noreply, state}
end
def perform(:clean), do: Token.delete_expired_tokens()
end

27
lib/pleroma/web/pleroma_api/pleroma_api_controller.ex
View file

@ -5,7 +5,7 @@
defmodule Pleroma.Web.PleromaAPI.PleromaAPIController do
use Pleroma.Web, :controller
import Pleroma.Web.ControllerHelper, only: [add_link_headers: 7]
import Pleroma.Web.ControllerHelper, only: [add_link_headers: 2]
alias Pleroma.Conversation.Participation
alias Pleroma.Notification
@ -27,31 +27,22 @@ defmodule Pleroma.Web.PleromaAPI.PleromaAPIController do
%{assigns: %{user: user}} = conn,
%{"id" => participation_id} = params
) do
params =
params
|> Map.put("blocking_user", user)
|> Map.put("muting_user", user)
|> Map.put("user", user)
participation =
participation_id
|> Participation.get(preload: [:conversation])
participation = Participation.get(participation_id, preload: [:conversation])
if user.id == participation.user_id do
params =
params
|> Map.put("blocking_user", user)
|> Map.put("muting_user", user)
|> Map.put("user", user)
activities =
participation.conversation.ap_id
|> ActivityPub.fetch_activities_for_context(params)
|> Enum.reverse()
conn
|> add_link_headers(
:conversation_statuses,
activities,
participation_id,
params,
nil,
&pleroma_api_url/4
)
|> add_link_headers(activities)
|> put_view(StatusView)
|> render("index.json", %{activities: activities, for: user, as: :activity})
end

7
lib/pleroma/web/push/push.ex
View file

@ -3,7 +3,7 @@
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.Push do
alias Pleroma.Web.Push.Impl
alias Pleroma.Workers.WebPusherWorker
require Logger
@ -31,6 +31,7 @@ defmodule Pleroma.Web.Push do
end
end
def send(notification),
do: PleromaJobQueue.enqueue(:web_push, Impl, [notification])
def send(notification) do
WebPusherWorker.enqueue("web_push", %{"notification_id" => notification.id})
end
end

1
lib/pleroma/web/router.ex
View file

@ -224,6 +224,7 @@ defmodule Pleroma.Web.Router do
scope [] do
pipe_through(:oauth_write)
post("/change_email", UtilController, :change_email)
post("/change_password", UtilController, :change_password)
post("/delete_account", UtilController, :delete_account)
put("/notification_settings", UtilController, :update_notificaton_settings)

11
lib/pleroma/web/salmon/salmon.ex
View file

@ -170,6 +170,15 @@ defmodule Pleroma.Web.Salmon do
end
end
def publish_one(%{recipient_id: recipient_id} = params) do
recipient = User.get_cached_by_id(recipient_id)
params
|> Map.delete(:recipient_id)
|> Map.put(:recipient, recipient)
|> publish_one()
end
def publish_one(_), do: :noop
@supported_activities [
@ -218,7 +227,7 @@ defmodule Pleroma.Web.Salmon do
Logger.debug(fn -> "Sending Salmon to #{remote_user.ap_id}" end)
Publisher.enqueue_one(__MODULE__, %{
recipient: remote_user,
recipient_id: remote_user.id,
feed: feed,
unreachable_since: reachable_urls_metadata[remote_user.info.salmon]
})

33
lib/pleroma/web/twitter_api/controllers/util_controller.ex
View file

@ -265,12 +265,7 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do
String.split(line, ",") |> List.first()
end)
|> List.delete("Account address") do
PleromaJobQueue.enqueue(:background, User, [
:follow_import,
follower,
followed_identifiers
])
User.follow_import(follower, followed_identifiers)
json(conn, "job started")
end
end
@ -281,12 +276,7 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do
def blocks_import(%{assigns: %{user: blocker}} = conn, %{"list" => list}) do
with blocked_identifiers <- String.split(list) do
PleromaJobQueue.enqueue(:background, User, [
:blocks_import,
blocker,
blocked_identifiers
])
User.blocks_import(blocker, blocked_identifiers)
json(conn, "job started")
end
end
@ -314,6 +304,25 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do
end
end
def change_email(%{assigns: %{user: user}} = conn, params) do
case CommonAPI.Utils.confirm_current_password(user, params["password"]) do
{:ok, user} ->
with {:ok, _user} <- User.change_email(user, params["email"]) do
json(conn, %{status: "success"})
else
{:error, changeset} ->
{_, {error, _}} = Enum.at(changeset.errors, 0)
json(conn, %{error: "Email #{error}."})
_ ->
json(conn, %{error: "Unable to change email."})
end
{:error, msg} ->
json(conn, %{error: msg})
end
end
def delete_account(%{assigns: %{user: user}} = conn, params) do
case CommonAPI.Utils.confirm_current_password(user, params["password"]) do
{:ok, user} ->