hj/pleroma
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge remote-tracking branch 'origin/develop' into translate-posts

Browse source 
Signed-off-by: marcin mikołajczak <git@mkljczk.pl>
This commit is contained in:
marcin mikołajczak 2024-06-12 17:13:23 +02:00
commit b07fd324fb
193 changed files with 4639 additions and 784 deletions
Download patch file Download diff file Expand all files Collapse all files

165
lib/mix/tasks/pleroma/database.ex
View file

@ -67,43 +67,168 @@ defmodule Mix.Tasks.Pleroma.Database do
OptionParser.parse(
args,
strict: [
vacuum: :boolean
vacuum: :boolean,
keep_threads: :boolean,
keep_non_public: :boolean,
prune_orphaned_activities: :boolean
]
)
start_pleroma()
deadline = Pleroma.Config.get([:instance, :remote_post_retention_days])
time_deadline = NaiveDateTime.utc_now() |> NaiveDateTime.add(-(deadline * 86_400))
Logger.info("Pruning objects older than #{deadline} days")
log_message = "Pruning objects older than #{deadline} days"
time_deadline =
NaiveDateTime.utc_now()
|> NaiveDateTime.add(-(deadline * 86_400))
log_message =
if Keyword.get(options, :keep_non_public) do
log_message <> ", keeping non public posts"
else
log_message
end
from(o in Object,
where:
fragment(
"?->'to' \\? ? OR ?->'cc' \\? ?",
o.data,
^Pleroma.Constants.as_public(),
o.data,
^Pleroma.Constants.as_public()
),
where: o.inserted_at < ^time_deadline,
where:
log_message =
if Keyword.get(options, :keep_threads) do
log_message <> ", keeping threads intact"
else
log_message
end
log_message =
if Keyword.get(options, :prune_orphaned_activities) do
log_message <> ", pruning orphaned activities"
else
log_message
end
log_message =
if Keyword.get(options, :vacuum) do
log_message <>
", doing a full vacuum (you shouldn't do this as a recurring maintanance task)"
else
log_message
end
Logger.info(log_message)
if Keyword.get(options, :keep_threads) do
# We want to delete objects from threads where
# 1. the newest post is still old
# 2. none of the activities is local
# 3. none of the activities is bookmarked
# 4. optionally none of the posts is non-public
deletable_context =
if Keyword.get(options, :keep_non_public) do
Pleroma.Activity
|> join(:left, [a], b in Pleroma.Bookmark, on: a.id == b.activity_id)
|> group_by([a], fragment("? ->> 'context'::text", a.data))
|> having(
[a],
not fragment(
# Posts (checked on Create Activity) is non-public
"bool_or((not(?->'to' \\? ? OR ?->'cc' \\? ?)) and ? ->> 'type' = 'Create')",
a.data,
^Pleroma.Constants.as_public(),
a.data,
^Pleroma.Constants.as_public(),
a.data
)
)
else
Pleroma.Activity
|> join(:left, [a], b in Pleroma.Bookmark, on: a.id == b.activity_id)
|> group_by([a], fragment("? ->> 'context'::text", a.data))
end
|> having([a], max(a.updated_at) < ^time_deadline)
|> having([a], not fragment("bool_or(?)", a.local))
|> having([_, b], fragment("max(?::text) is null", b.id))
|> select([a], fragment("? ->> 'context'::text", a.data))
Pleroma.Object
|> where([o], fragment("? ->> 'context'::text", o.data) in subquery(deletable_context))
else
if Keyword.get(options, :keep_non_public) do
Pleroma.Object
|> where(
[o],
fragment(
"?->'to' \\? ? OR ?->'cc' \\? ?",
o.data,
^Pleroma.Constants.as_public(),
o.data,
^Pleroma.Constants.as_public()
)
)
else
Pleroma.Object
end
|> where([o], o.updated_at < ^time_deadline)
|> where(
[o],
fragment("split_part(?->>'actor', '/', 3) != ?", o.data, ^Pleroma.Web.Endpoint.host())
)
)
end
|> Repo.delete_all(timeout: :infinity)
prune_hashtags_query = """
if !Keyword.get(options, :keep_threads) do
# Without the --keep-threads option, it's possible that bookmarked
# objects have been deleted. We remove the corresponding bookmarks.
"""
delete from public.bookmarks
where id in (
select b.id from public.bookmarks b
left join public.activities a on b.activity_id = a.id
left join public.objects o on a."data" ->> 'object' = o.data ->> 'id'
where o.id is null
)
"""
|> Repo.query([], timeout: :infinity)
end
if Keyword.get(options, :prune_orphaned_activities) do
# Prune activities who link to a single object
"""
delete from public.activities
where id in (
select a.id from public.activities a
left join public.objects o on a.data ->> 'object' = o.data ->> 'id'
left join public.activities a2 on a.data ->> 'object' = a2.data ->> 'id'
left join public.users u on a.data ->> 'object' = u.ap_id
where not a.local
and jsonb_typeof(a."data" -> 'object') = 'string'
and o.id is null
and a2.id is null
and u.id is null
)
"""
|> Repo.query([], timeout: :infinity)
# Prune activities who link to an array of objects
"""
delete from public.activities
where id in (
select a.id from public.activities a
join json_array_elements_text((a."data" -> 'object')::json) as j on jsonb_typeof(a."data" -> 'object') = 'array'
left join public.objects o on j.value = o.data ->> 'id'
left join public.activities a2 on j.value = a2.data ->> 'id'
left join public.users u on j.value = u.ap_id
group by a.id
having max(o.data ->> 'id') is null
and max(a2.data ->> 'id') is null
and max(u.ap_id) is null
)
"""
|> Repo.query([], timeout: :infinity)
end
"""
DELETE FROM hashtags AS ht
WHERE NOT EXISTS (
SELECT 1 FROM hashtags_objects hto
WHERE ht.id = hto.hashtag_id)
"""
Repo.query(prune_hashtags_query)
|> Repo.query()
if Keyword.get(options, :vacuum) do
Maintenance.vacuum("full")

83
lib/mix/tasks/pleroma/search/indexer.ex Normal file
View file

@ -0,0 +1,83 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Mix.Tasks.Pleroma.Search.Indexer do
import Mix.Pleroma
import Ecto.Query
alias Pleroma.Workers.SearchIndexingWorker
def run(["create_index"]) do
start_pleroma()
with :ok <- Pleroma.Config.get([Pleroma.Search, :module]).create_index() do
IO.puts("Index created")
else
e -> IO.puts("Could not create index: #{inspect(e)}")
end
end
def run(["drop_index"]) do
start_pleroma()
with :ok <- Pleroma.Config.get([Pleroma.Search, :module]).drop_index() do
IO.puts("Index dropped")
else
e -> IO.puts("Could not drop index: #{inspect(e)}")
end
end
def run(["index" | options]) do
{options, [], []} =
OptionParser.parse(
options,
strict: [
chunk: :integer,
limit: :integer,
step: :integer
]
)
start_pleroma()
chunk_size = Keyword.get(options, :chunk, 100)
limit = Keyword.get(options, :limit, 100_000)
per_step = Keyword.get(options, :step, 1000)
chunks = max(div(limit, per_step), 1)
1..chunks
|> Enum.each(fn step ->
q =
from(a in Pleroma.Activity,
limit: ^per_step,
offset: ^per_step * (^step - 1),
select: [:id],
order_by: [desc: :id]
)
{:ok, ids} =
Pleroma.Repo.transaction(fn ->
Pleroma.Repo.stream(q, timeout: :infinity)
|> Enum.map(fn a ->
a.id
end)
end)
IO.puts("Got #{length(ids)} activities, adding to indexer")
ids
|> Enum.chunk_every(chunk_size)
|> Enum.each(fn chunk ->
IO.puts("Adding #{length(chunk)} activities to indexing queue")
chunk
|> Enum.map(fn id ->
SearchIndexingWorker.new(%{"op" => "add_to_index", "activity" => id})
end)
|> Oban.insert_all()
end)
end)
end
end

13
lib/pleroma/application.ex
View file

@ -14,6 +14,7 @@ defmodule Pleroma.Application do
@name Mix.Project.config()[:name]
@version Mix.Project.config()[:version]
@repository Mix.Project.config()[:source_url]
@compile_env Mix.env()
def name, do: @name
def version, do: @version
@ -51,7 +52,11 @@ defmodule Pleroma.Application do
Pleroma.HTML.compile_scrubbers()
Pleroma.Config.Oban.warn()
Config.DeprecationWarnings.warn()
Pleroma.Web.Plugs.HTTPSecurityPlug.warn_if_disabled()
if @compile_env != :test do
Pleroma.Web.Plugs.HTTPSecurityPlug.warn_if_disabled()
end
Pleroma.ApplicationRequirements.verify!()
load_custom_modules()
Pleroma.Docs.JSON.compile()
@ -109,7 +114,8 @@ defmodule Pleroma.Application do
streamer_registry() ++
background_migrators() ++
shout_child(shout_enabled?()) ++
[Pleroma.Gopher.Server]
[Pleroma.Gopher.Server] ++
[Pleroma.Search.Healthcheck]
# See http://elixir-lang.org/docs/stable/elixir/Supervisor.html
# for other strategies and supported options
@ -163,6 +169,7 @@ defmodule Pleroma.Application do
limit: 500_000
),
build_cachex("rel_me", limit: 2500),
build_cachex("host_meta", default_ttl: :timer.minutes(120), limit: 5_000),
build_cachex("translations", default_ttl: :timer.hours(24), limit: 5_000)
]
end
@ -286,8 +293,6 @@ defmodule Pleroma.Application do
config = Config.get(ConcurrentLimiter, [])
[
Pleroma.Web.RichMedia.Helpers,
Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy,
Pleroma.Search
]
|> Enum.each(fn module ->

11
lib/pleroma/helpers/inet_helper.ex
View file

@ -16,4 +16,15 @@ defmodule Pleroma.Helpers.InetHelper do
def parse_address(ip) do
:inet.parse_address(ip)
end
def parse_cidr(proxy) when is_binary(proxy) do
proxy =
cond do
"/" in String.codepoints(proxy) -> proxy
InetCidr.v4?(InetCidr.parse_address!(proxy)) -> proxy <> "/32"
InetCidr.v6?(InetCidr.parse_address!(proxy)) -> proxy <> "/128"
end
InetCidr.parse_cidr!(proxy, true)
end
end

16
lib/pleroma/helpers/media_helper.ex
View file

@ -25,7 +25,7 @@ defmodule Pleroma.Helpers.MediaHelper do
end
def image_resize(url, options) do
with {:ok, env} <- HTTP.get(url, [], pool: :media),
with {:ok, env} <- HTTP.get(url, [], http_client_opts()),
{:ok, resized} <-
Operation.thumbnail_buffer(env.body, options.max_width,
height: options.max_height,
@ -45,8 +45,8 @@ defmodule Pleroma.Helpers.MediaHelper do
@spec video_framegrab(String.t()) :: {:ok, binary()} | {:error, any()}
def video_framegrab(url) do
with executable when is_binary(executable) <- System.find_executable("ffmpeg"),
false <- @cachex.exists?(:failed_media_helper_cache, url),
{:ok, env} <- HTTP.get(url, [], pool: :media),
{:ok, false} <- @cachex.exists?(:failed_media_helper_cache, url),
{:ok, env} <- HTTP.get(url, [], http_client_opts()),
{:ok, pid} <- StringIO.open(env.body) do
body_stream = IO.binstream(pid, 1)
@ -71,17 +71,19 @@ defmodule Pleroma.Helpers.MediaHelper do
end)
case Task.yield(task, 5_000) do
nil ->
{:ok, result} ->
{:ok, result}
_ ->
Task.shutdown(task)
@cachex.put(:failed_media_helper_cache, url, nil)
{:error, {:ffmpeg, :timeout}}
result ->
{:ok, result}
end
else
nil -> {:error, {:ffmpeg, :command_not_found}}
{:error, _} = error -> error
end
end
defp http_client_opts, do: Pleroma.Config.get([:media_proxy, :proxy_opts, :http], pool: :media)
end

4
lib/pleroma/http.ex
View file

@ -37,7 +37,7 @@ defmodule Pleroma.HTTP do
See `Pleroma.HTTP.request/5`
"""
@spec post(Request.url(), String.t(), Request.headers(), keyword()) ::
@spec post(Request.url(), Tesla.Env.body(), Request.headers(), keyword()) ::
{:ok, Env.t()} | {:error, any()}
def post(url, body, headers \\ [], options \\ []),
do: request(:post, url, body, headers, options)
@ -56,7 +56,7 @@ defmodule Pleroma.HTTP do
`{:ok, %Tesla.Env{}}` or `{:error, error}`
"""
@spec request(method(), Request.url(), String.t(), Request.headers(), keyword()) ::
@spec request(method(), Request.url(), Tesla.Env.body(), Request.headers(), keyword()) ::
{:ok, Env.t()} | {:error, any()}
def request(method, url, body, headers, options) when is_binary(url) do
uri = URI.parse(url)

2
lib/pleroma/http/adapter_helper/gun.ex
View file

@ -15,7 +15,7 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do
retry_timeout: 1_000
]
@type pool() :: :federation | :upload | :media | :default
@type pool() :: :federation | :upload | :media | :rich_media | :default
@spec options(keyword(), URI.t()) :: keyword()
def options(incoming_opts \\ [], %URI{} = uri) do

4
lib/pleroma/http_signatures_api.ex Normal file
View file

@ -0,0 +1,4 @@
defmodule Pleroma.HTTPSignaturesAPI do
@callback validate_conn(conn :: Plug.Conn.t()) :: boolean
@callback signature_for_conn(conn :: Plug.Conn.t()) :: map
end

57
lib/pleroma/notification.ex
View file

@ -73,6 +73,7 @@ defmodule Pleroma.Notification do
pleroma:report
reblog
poll
status
}
def changeset(%Notification{} = notification, attrs) do
@ -280,15 +281,10 @@ defmodule Pleroma.Notification do
select: n.id
)
{:ok, %{ids: {_, notification_ids}}} =
Multi.new()
|> Multi.update_all(:ids, query, set: [seen: true, updated_at: NaiveDateTime.utc_now()])
|> Marker.multi_set_last_read_id(user, "notifications")
|> Repo.transaction()
for_user_query(user)
|> where([n], n.id in ^notification_ids)
|> Repo.all()
Multi.new()
|> Multi.update_all(:ids, query, set: [seen: true, updated_at: NaiveDateTime.utc_now()])
|> Marker.multi_set_last_read_id(user, "notifications")
|> Repo.transaction()
end
@spec read_one(User.t(), String.t()) ::
@ -299,10 +295,6 @@ defmodule Pleroma.Notification do
|> Multi.update(:update, changeset(notification, %{seen: true}))
|> Marker.multi_set_last_read_id(user, "notifications")
|> Repo.transaction()
|> case do
{:ok, %{update: notification}} -> {:ok, notification}
{:error, :update, changeset, _} -> {:error, changeset}
end
end
end
@ -384,10 +376,15 @@ defmodule Pleroma.Notification do
defp do_create_notifications(%Activity{} = activity) do
enabled_receivers = get_notified_from_activity(activity)
enabled_subscribers = get_notified_subscribers_from_activity(activity)
notifications =
Enum.map(enabled_receivers, fn user ->
create_notification(activity, user)
end)
(Enum.map(enabled_receivers, fn user ->
create_notification(activity, user)
end) ++
Enum.map(enabled_subscribers -- enabled_receivers, fn user ->
create_notification(activity, user, type: "status")
end))
|> Enum.reject(&is_nil/1)
{:ok, notifications}
@ -492,7 +489,7 @@ defmodule Pleroma.Notification do
NOTE: might be called for FAKE Activities, see ActivityPub.Utils.get_notified_from_object/1
"""
@spec get_notified_from_activity(Activity.t(), boolean()) :: {list(User.t()), list(User.t())}
@spec get_notified_from_activity(Activity.t(), boolean()) :: list(User.t())
def get_notified_from_activity(activity, local_only \\ true)
def get_notified_from_activity(%Activity{data: %{"type" => type}} = activity, local_only)
@ -520,7 +517,25 @@ defmodule Pleroma.Notification do
Enum.filter(potential_receivers, fn u -> u.ap_id in notification_enabled_ap_ids end)
end
def get_notified_from_activity(_, _local_only), do: {[], []}
def get_notified_from_activity(_, _local_only), do: []
def get_notified_subscribers_from_activity(activity, local_only \\ true)
def get_notified_subscribers_from_activity(
%Activity{data: %{"type" => "Create"}} = activity,
local_only
) do
notification_enabled_ap_ids =
[]
|> Utils.maybe_notify_subscribers(activity)
potential_receivers =
User.get_users_from_set(notification_enabled_ap_ids, local_only: local_only)
Enum.filter(potential_receivers, fn u -> u.ap_id in notification_enabled_ap_ids end)
end
def get_notified_subscribers_from_activity(_, _), do: []
# For some activities, only notify the author of the object
def get_potential_receiver_ap_ids(%{data: %{"type" => type, "object" => object_id}})
@ -563,7 +578,6 @@ defmodule Pleroma.Notification do
[]
|> Utils.maybe_notify_to_recipients(activity)
|> Utils.maybe_notify_mentioned_recipients(activity)
|> Utils.maybe_notify_subscribers(activity)
|> Utils.maybe_notify_followers(activity)
|> Enum.uniq()
end
@ -743,8 +757,9 @@ defmodule Pleroma.Notification do
|> Repo.update_all(set: [seen: true])
end
@spec send(list(Notification.t())) :: :ok
def send(notifications) do
@doc "Streams a list of notifications over websockets and web push"
@spec stream(list(Notification.t())) :: :ok
def stream(notifications) do
Enum.each(notifications, fn notification ->
Streamer.stream(["user", "user:notification"], notification)
Push.send(notification)

16
lib/pleroma/reverse_proxy.ex
View file

@ -8,7 +8,7 @@ defmodule Pleroma.ReverseProxy do
~w(if-unmodified-since if-none-match) ++ @range_headers
@resp_cache_headers ~w(etag date last-modified)
@keep_resp_headers @resp_cache_headers ++
~w(content-type content-disposition content-encoding) ++
~w(content-length content-type content-disposition content-encoding) ++
~w(content-range accept-ranges vary)
@default_cache_control_header "public, max-age=1209600"
@valid_resp_codes [200, 206, 304]
@ -180,6 +180,7 @@ defmodule Pleroma.ReverseProxy do
result =
conn
|> put_resp_headers(build_resp_headers(headers, opts))
|> streaming_compat
|> send_chunked(status)
|> chunk_reply(client, opts)
@ -417,4 +418,17 @@ defmodule Pleroma.ReverseProxy do
@cachex.put(:failed_proxy_url_cache, url, true, ttl: ttl)
end
# When Cowboy handles a chunked response with a content-length header it streams
# over HTTP 1.1 instead of chunking. Bandit cannot stream over HTTP 1.1 so the header
# must be stripped or it breaks RFC compliance for Transfer Encoding: Chunked. RFC9112§6.2
#
# HTTP2 is always streamed for all adapters.
defp streaming_compat(conn) do
with Phoenix.Endpoint.Cowboy2Adapter <- Pleroma.Web.Endpoint.config(:adapter) do
conn
else
_ -> delete_resp_header(conn, "content-length")
end
end
end

2
lib/pleroma/scheduled_activity.ex
View file

@ -204,7 +204,7 @@ defmodule Pleroma.ScheduledActivity do
def job_query(scheduled_activity_id) do
from(j in Oban.Job,
where: j.queue == "scheduled_activities",
where: j.queue == "federator_outgoing",
where: fragment("args ->> 'activity_id' = ?::text", ^to_string(scheduled_activity_id))
)
end

8
lib/pleroma/search.ex
View file

@ -10,8 +10,12 @@ defmodule Pleroma.Search do
end
def search(query, options) do
search_module = Pleroma.Config.get([Pleroma.Search, :module], Pleroma.Activity)
search_module = Pleroma.Config.get([Pleroma.Search, :module])
search_module.search(options[:for_user], query, options)
end
def healthcheck_endpoints do
search_module = Pleroma.Config.get([Pleroma.Search, :module])
search_module.healthcheck_endpoints
end
end

45
lib/pleroma/search/database_search.ex
View file

@ -28,7 +28,7 @@ defmodule Pleroma.Search.DatabaseSearch do
|> Activity.with_preloaded_object()
|> Activity.restrict_deactivated_users()
|> restrict_public(user)
|> query_with(index_type, search_query, :websearch)
|> query_with(index_type, search_query)
|> maybe_restrict_local(user)
|> maybe_restrict_author(author)
|> maybe_restrict_blocked(user)
@ -48,6 +48,15 @@ defmodule Pleroma.Search.DatabaseSearch do
@impl true
def remove_from_index(_object), do: :ok
@impl true
def create_index, do: :ok
@impl true
def drop_index, do: :ok
@impl true
def healthcheck_endpoints, do: nil
def maybe_restrict_author(query, %User{} = author) do
Activity.Queries.by_author(query, author)
end
@ -79,25 +88,7 @@ defmodule Pleroma.Search.DatabaseSearch do
)
end
defp query_with(q, :gin, search_query, :plain) do
%{rows: [[tsc]]} =
Ecto.Adapters.SQL.query!(
Pleroma.Repo,
"select current_setting('default_text_search_config')::regconfig::oid;"
)
from([a, o] in q,
where:
fragment(
"to_tsvector(?::oid::regconfig, ?->>'content') @@ plainto_tsquery(?)",
^tsc,
o.data,
^search_query
)
)
end
defp query_with(q, :gin, search_query, :websearch) do
defp query_with(q, :gin, search_query) do
%{rows: [[tsc]]} =
Ecto.Adapters.SQL.query!(
Pleroma.Repo,
@ -115,19 +106,7 @@ defmodule Pleroma.Search.DatabaseSearch do
)
end
defp query_with(q, :rum, search_query, :plain) do
from([a, o] in q,
where:
fragment(
"? @@ plainto_tsquery(?)",
o.fts_content,
^search_query
),
order_by: [fragment("? <=> now()::date", o.inserted_at)]
)
end
defp query_with(q, :rum, search_query, :websearch) do
defp query_with(q, :rum, search_query) do
from([a, o] in q,
where:
fragment(

86
lib/pleroma/search/healthcheck.ex Normal file
View file

@ -0,0 +1,86 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2024 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Search.Healthcheck do
@doc """
Monitors health of search backend to control processing of events based on health and availability.
"""
use GenServer
require Logger
@queue :search_indexing
@tick :timer.seconds(5)
@timeout :timer.seconds(2)
def start_link(_) do
GenServer.start_link(__MODULE__, [], name: __MODULE__)
end
@impl true
def init(_) do
state = %{healthy: false}
{:ok, state, {:continue, :start}}
end
@impl true
def handle_continue(:start, state) do
tick()
{:noreply, state}
end
@impl true
def handle_info(:check, state) do
urls = Pleroma.Search.healthcheck_endpoints()
new_state =
if check(urls) do
Oban.resume_queue(queue: @queue)
Map.put(state, :healthy, true)
else
Oban.pause_queue(queue: @queue)
Map.put(state, :healthy, false)
end
maybe_log_state_change(state, new_state)
tick()
{:noreply, new_state}
end
@impl true
def handle_call(:state, _from, state) do
{:reply, state, state, :hibernate}
end
def state, do: GenServer.call(__MODULE__, :state)
def check([]), do: true
def check(urls) when is_list(urls) do
Enum.all?(
urls,
fn url ->
case Pleroma.HTTP.get(url, [], recv_timeout: @timeout) do
{:ok, %{status: 200}} -> true
_ -> false
end
end
)
end
def check(_), do: true
defp tick do
Process.send_after(self(), :check, @tick)
end
defp maybe_log_state_change(%{healthy: true}, %{healthy: false}) do
Logger.error("Pausing Oban queue #{@queue} due to search backend healthcheck failure")
end
defp maybe_log_state_change(%{healthy: false}, %{healthy: true}) do
Logger.info("Resuming Oban queue #{@queue} due to search backend healthcheck pass")
end
defp maybe_log_state_change(_, _), do: :ok
end

17
lib/pleroma/search/meilisearch.ex
View file

@ -10,6 +10,12 @@ defmodule Pleroma.Search.Meilisearch do
@behaviour Pleroma.Search.SearchBackend
@impl true
def create_index, do: :ok
@impl true
def drop_index, do: :ok
defp meili_headers do
private_key = Config.get([Pleroma.Search.Meilisearch, :private_key])
@ -178,4 +184,15 @@ defmodule Pleroma.Search.Meilisearch do
def remove_from_index(object) do
meili_delete("/indexes/objects/documents/#{object.id}")
end
@impl true
def healthcheck_endpoints do
endpoint =
Config.get([Pleroma.Search.Meilisearch, :url])
|> URI.parse()
|> Map.put(:path, "/health")
|> URI.to_string()
[endpoint]
end
end

182
lib/pleroma/search/qdrant_search.ex Normal file
View file

@ -0,0 +1,182 @@
defmodule Pleroma.Search.QdrantSearch do
@behaviour Pleroma.Search.SearchBackend
import Ecto.Query
alias Pleroma.Activity
alias Pleroma.Config.Getting, as: Config
alias __MODULE__.OpenAIClient
alias __MODULE__.QdrantClient
import Pleroma.Search.Meilisearch, only: [object_to_search_data: 1]
import Pleroma.Search.DatabaseSearch, only: [maybe_fetch: 3]
@impl true
def create_index do
payload = Config.get([Pleroma.Search.QdrantSearch, :qdrant_index_configuration])
with {:ok, %{status: 200}} <- QdrantClient.put("/collections/posts", payload) do
:ok
else
e -> {:error, e}
end
end
@impl true
def drop_index do
with {:ok, %{status: 200}} <- QdrantClient.delete("/collections/posts") do
:ok
else
e -> {:error, e}
end
end
def get_embedding(text) do
with {:ok, %{body: %{"data" => [%{"embedding" => embedding}]}}} <-
OpenAIClient.post("/v1/embeddings", %{
input: text,
model: Config.get([Pleroma.Search.QdrantSearch, :openai_model])
}) do
{:ok, embedding}
else
_ ->
{:error, "Failed to get embedding"}
end
end
defp actor_from_activity(%{data: %{"actor" => actor}}) do
actor
end
defp actor_from_activity(_), do: nil
defp build_index_payload(activity, embedding) do
actor = actor_from_activity(activity)
published_at = activity.data["published"]
%{
points: [
%{
id: activity.id |> FlakeId.from_string() |> Ecto.UUID.cast!(),
vector: embedding,
payload: %{actor: actor, published_at: published_at}
}
]
}
end
defp build_search_payload(embedding, options) do
base = %{
vector: embedding,
limit: options[:limit] || 20,
offset: options[:offset] || 0
}
if author = options[:author] do
Map.put(base, :filter, %{
must: [%{key: "actor", match: %{value: author.ap_id}}]
})
else
base
end
end
@impl true
def add_to_index(activity) do
# This will only index public or unlisted notes
maybe_search_data = object_to_search_data(activity.object)
if activity.data["type"] == "Create" and maybe_search_data do
with {:ok, embedding} <- get_embedding(maybe_search_data.content),
{:ok, %{status: 200}} <-
QdrantClient.put(
"/collections/posts/points",
build_index_payload(activity, embedding)
) do
:ok
else
e -> {:error, e}
end
else
:ok
end
end
@impl true
def remove_from_index(object) do
activity = Activity.get_by_object_ap_id_with_object(object.data["id"])
id = activity.id |> FlakeId.from_string() |> Ecto.UUID.cast!()
with {:ok, %{status: 200}} <-
QdrantClient.post("/collections/posts/points/delete", %{"points" => [id]}) do
:ok
else
e -> {:error, e}
end
end
@impl true
def search(user, original_query, options) do
query = "Represent this sentence for searching relevant passages: #{original_query}"
with {:ok, embedding} <- get_embedding(query),
{:ok, %{body: %{"result" => result}}} <-
QdrantClient.post(
"/collections/posts/points/search",
build_search_payload(embedding, options)
) do
ids =
Enum.map(result, fn %{"id" => id} ->
Ecto.UUID.dump!(id)
end)
from(a in Activity, where: a.id in ^ids)
|> Activity.with_preloaded_object()
|> Activity.restrict_deactivated_users()
|> Ecto.Query.order_by([a], fragment("array_position(?, ?)", ^ids, a.id))
|> Pleroma.Repo.all()
|> maybe_fetch(user, original_query)
else
_ ->
[]
end
end
@impl true
def healthcheck_endpoints do
qdrant_health =
Config.get([Pleroma.Search.QdrantSearch, :qdrant_url])
|> URI.parse()
|> Map.put(:path, "/healthz")
|> URI.to_string()
openai_health = Config.get([Pleroma.Search.QdrantSearch, :openai_healthcheck_url])
[qdrant_health, openai_health] |> Enum.filter(& &1)
end
end
defmodule Pleroma.Search.QdrantSearch.OpenAIClient do
use Tesla
alias Pleroma.Config.Getting, as: Config
plug(Tesla.Middleware.BaseUrl, Config.get([Pleroma.Search.QdrantSearch, :openai_url]))
plug(Tesla.Middleware.JSON)
plug(Tesla.Middleware.Headers, [
{"Authorization",
"Bearer #{Pleroma.Config.get([Pleroma.Search.QdrantSearch, :openai_api_key])}"}
])
end
defmodule Pleroma.Search.QdrantSearch.QdrantClient do
use Tesla
alias Pleroma.Config.Getting, as: Config
plug(Tesla.Middleware.BaseUrl, Config.get([Pleroma.Search.QdrantSearch, :qdrant_url]))
plug(Tesla.Middleware.JSON)
plug(Tesla.Middleware.Headers, [
{"api-key", Pleroma.Config.get([Pleroma.Search.QdrantSearch, :qdrant_api_key])}
])
end

18
lib/pleroma/search/search_backend.ex
View file

@ -21,4 +21,22 @@ defmodule Pleroma.Search.SearchBackend do
from index.
"""
@callback remove_from_index(object :: Pleroma.Object.t()) :: :ok | {:error, any()}
@doc """
Create the index
"""
@callback create_index() :: :ok | {:error, any()}
@doc """
Drop the index
"""
@callback drop_index() :: :ok | {:error, any()}
@doc """
Healthcheck endpoints of search backend infrastructure to monitor for controlling
processing of jobs in the Oban queue.
It is expected a 200 response is healthy and other responses are unhealthy.
"""
@callback healthcheck_endpoints :: list() | nil
end

16
lib/pleroma/signature.ex
View file

@ -44,8 +44,7 @@ defmodule Pleroma.Signature do
defp remove_suffix(uri, []), do: uri
def fetch_public_key(conn) do
with %{"keyId" => kid} <- HTTPSignatures.signature_for_conn(conn),
{:ok, actor_id} <- key_id_to_actor_id(kid),
with {:ok, actor_id} <- get_actor_id(conn),
{:ok, public_key} <- User.get_public_key_for_ap_id(actor_id) do
{:ok, public_key}
else
@ -55,8 +54,7 @@ defmodule Pleroma.Signature do
end
def refetch_public_key(conn) do
with %{"keyId" => kid} <- HTTPSignatures.signature_for_conn(conn),
{:ok, actor_id} <- key_id_to_actor_id(kid),
with {:ok, actor_id} <- get_actor_id(conn),
{:ok, _user} <- ActivityPub.make_user_from_ap_id(actor_id),
{:ok, public_key} <- User.get_public_key_for_ap_id(actor_id) do
{:ok, public_key}
@ -66,6 +64,16 @@ defmodule Pleroma.Signature do
end
end
def get_actor_id(conn) do
with %{"keyId" => kid} <- HTTPSignatures.signature_for_conn(conn),
{:ok, actor_id} <- key_id_to_actor_id(kid) do
{:ok, actor_id}
else
e ->
{:error, e}
end
end
def sign(%User{keys: keys} = user, headers) do
with {:ok, private_key, _} <- Keys.keys_from_pem(keys) do
HTTPSignatures.sign(private_key, user.ap_id <> "#main-key", headers)

11
lib/pleroma/upload.ex
View file

@ -239,8 +239,12 @@ defmodule Pleroma.Upload do
""
end
[base_url, path]
|> Path.join()
if String.contains?(base_url, Pleroma.Uploaders.IPFS.placeholder()) do
String.replace(base_url, Pleroma.Uploaders.IPFS.placeholder(), path)
else
[base_url, path]
|> Path.join()
end
end
defp url_from_spec(_upload, _base_url, {:url, url}), do: url
@ -277,6 +281,9 @@ defmodule Pleroma.Upload do
Path.join([upload_base_url, bucket_with_namespace])
end
Pleroma.Uploaders.IPFS ->
@config_impl.get([Pleroma.Uploaders.IPFS, :get_gateway_url])
_ ->
public_endpoint || upload_base_url || Pleroma.Web.Endpoint.url() <> "/media/"
end

2
lib/pleroma/upload/filter/exiftool/strip_location.ex
View file

@ -9,8 +9,6 @@ defmodule Pleroma.Upload.Filter.Exiftool.StripLocation do
"""
@behaviour Pleroma.Upload.Filter
@spec filter(Pleroma.Upload.t()) :: {:ok, any()} | {:error, String.t()}
# Formats not compatible with exiftool at this time
def filter(%Pleroma.Upload{content_type: "image/heic"}), do: {:ok, :noop}
def filter(%Pleroma.Upload{content_type: "image/webp"}), do: {:ok, :noop}

1
lib/pleroma/upload/filter/mogrifun.ex
View file

@ -38,7 +38,6 @@ defmodule Pleroma.Upload.Filter.Mogrifun do
[{"fill", "yellow"}, {"tint", "40"}]
]
@spec filter(Pleroma.Upload.t()) :: {:ok, atom()} | {:error, String.t()}
def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do
try do
Filter.Mogrify.do_filter(file, [Enum.random(@filters)])

1
lib/pleroma/upload/filter/mogrify.ex
View file

@ -8,7 +8,6 @@ defmodule Pleroma.Upload.Filter.Mogrify do
@type conversion :: action :: String.t() | {action :: String.t(), opts :: String.t()}
@type conversions :: conversion() | [conversion()]
@spec filter(Pleroma.Upload.t()) :: {:ok, :atom} | {:error, String.t()}
def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do
try do
do_filter(file, Pleroma.Config.get!([__MODULE__, :args]))

72
lib/pleroma/uploaders/ipfs.ex Normal file
View file

@ -0,0 +1,72 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Uploaders.IPFS do
@behaviour Pleroma.Uploaders.Uploader
require Logger
alias Tesla.Multipart
@api_add "/api/v0/add"
@api_delete "/api/v0/files/rm"
@config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config)
@placeholder "{CID}"
def placeholder, do: @placeholder
@impl true
def get_file(file) do
b_url = Pleroma.Upload.base_url()
if String.contains?(b_url, @placeholder) do
{:ok, {:url, String.replace(b_url, @placeholder, URI.decode(file))}}
else
{:error, "IPFS Get URL doesn't contain 'cid' placeholder"}
end
end
@impl true
def put_file(%Pleroma.Upload{tempfile: tempfile}) do
mp =
Multipart.new()
|> Multipart.add_content_type_param("charset=utf-8")
|> Multipart.add_file(tempfile)
endpoint = ipfs_endpoint(@api_add)
with {:ok, %{body: body}} when is_binary(body) <-
Pleroma.HTTP.post(endpoint, mp, [], params: ["cid-version": "1"], pool: :upload),
{_, {:ok, decoded}} <- {:json, Jason.decode(body)},
{_, true} <- {:hash, Map.has_key?(decoded, "Hash")} do
{:ok, {:file, decoded["Hash"]}}
else
{:hash, false} ->
{:error, "JSON doesn't contain Hash key"}
{:json, error} ->
Logger.error("#{__MODULE__}: #{inspect(error)}")
{:error, "JSON decode failed"}
error ->
Logger.error("#{__MODULE__}: #{inspect(error)}")
{:error, "IPFS Gateway upload failed"}
end
end
@impl true
def delete_file(file) do
endpoint = ipfs_endpoint(@api_delete)
case Pleroma.HTTP.post(endpoint, "", [], params: [arg: file]) do
{:ok, %{status: 204}} -> :ok
error -> {:error, inspect(error)}
end
end
defp ipfs_endpoint(path) do
URI.parse(@config_impl.get([__MODULE__, :post_gateway_url]))
|> Map.put(:path, path)
|> URI.to_string()
end
end

39
lib/pleroma/user.ex
View file

@ -1404,6 +1404,40 @@ defmodule Pleroma.User do
|> Repo.all()
end
@spec get_familiar_followers_query(User.t(), User.t(), pos_integer() | nil) :: Ecto.Query.t()
def get_familiar_followers_query(%User{} = user, %User{} = current_user, nil) do
friends =
get_friends_query(current_user)
|> where([u], not u.hide_follows)
|> select([u], u.id)
User.Query.build(%{is_active: true})
|> where([u], u.id not in ^[user.id, current_user.id])
|> join(:inner, [u], r in FollowingRelationship,
as: :followers_relationships,
on: r.following_id == ^user.id and r.follower_id == u.id
)
|> where([followers_relationships: r], r.state == ^:follow_accept)
|> where([followers_relationships: r], r.follower_id in subquery(friends))
end
def get_familiar_followers_query(%User{} = user, %User{} = current_user, page) do
user
|> get_familiar_followers_query(current_user, nil)
|> User.Query.paginate(page, 20)
end
@spec get_familiar_followers_query(User.t(), User.t()) :: Ecto.Query.t()
def get_familiar_followers_query(%User{} = user, %User{} = current_user),
do: get_familiar_followers_query(user, current_user, nil)
@spec get_familiar_followers(User.t(), User.t(), pos_integer() | nil) :: {:ok, list(User.t())}
def get_familiar_followers(%User{} = user, %User{} = current_user, page \\ nil) do
user
|> get_familiar_followers_query(current_user, page)
|> Repo.all()
end
def increase_note_count(%User{} = user) do
User
|> where(id: ^user.id)
@ -2019,7 +2053,8 @@ defmodule Pleroma.User do
%{scheme: scheme, userinfo: nil, host: host}
when not_empty_string(host) and scheme in ["http", "https"] <-
URI.parse(value),
{:not_idn, true} <- {:not_idn, to_string(:idna.encode(host)) == host},
{:not_idn, true} <-
{:not_idn, match?(^host, to_string(:idna.encode(to_charlist(host))))},
"me" <- Pleroma.Web.RelMe.maybe_put_rel_me(value, profile_urls) do
CommonUtils.to_masto_date(NaiveDateTime.utc_now())
else
@ -2693,7 +2728,7 @@ defmodule Pleroma.User do
end
end
@spec add_to_block(User.t(), User.t()) ::
@spec remove_from_block(User.t(), User.t()) ::
{:ok, UserRelationship.t()} | {:ok, nil} | {:error, Ecto.Changeset.t()}
defp remove_from_block(%User{} = user, %User{} = blocked) do
with {:ok, relationship} <- UserRelationship.delete_block(user, blocked) do

32
lib/pleroma/web/activity_pub/activity_pub.ex
View file

@ -201,7 +201,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
def notify_and_stream(activity) do
{:ok, notifications} = Notification.create_notifications(activity)
Notification.send(notifications)
Notification.stream(notifications)
original_activity =
case activity do
@ -979,8 +979,9 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
defp restrict_replies(query, %{exclude_replies: true}) do
from(
[_activity, object] in query,
where: fragment("?->>'inReplyTo' is null", object.data)
[activity, object] in query,
where:
fragment("?->>'inReplyTo' is null or ?->>'type' = 'Announce'", object.data, activity.data)
)
end
@ -1793,24 +1794,25 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
end
end
def pinned_fetch_task(nil), do: nil
def pinned_fetch_task(%{pinned_objects: pins}) do
if Enum.all?(pins, fn {ap_id, _} ->
Object.get_cached_by_ap_id(ap_id) ||
match?({:ok, _object}, Fetcher.fetch_object_from_id(ap_id))
end) do
:ok
else
:error
end
def enqueue_pin_fetches(%{pinned_objects: pins}) do
# enqueue a task to fetch all pinned objects
Enum.each(pins, fn {ap_id, _} ->
if is_nil(Object.get_cached_by_ap_id(ap_id)) do
Pleroma.Workers.RemoteFetcherWorker.enqueue("fetch_remote", %{
"id" => ap_id,
"depth" => 1
})
end
end)
end
def enqueue_pin_fetches(_), do: nil
def make_user_from_ap_id(ap_id, additional \\ []) do
user = User.get_cached_by_ap_id(ap_id)
with {:ok, data} <- fetch_and_prepare_user_from_ap_id(ap_id, additional) do
{:ok, _pid} = Task.start(fn -> pinned_fetch_task(data) end)
enqueue_pin_fetches(data)
if user do
user

8
lib/pleroma/web/activity_pub/activity_pub_controller.ex
View file

@ -52,6 +52,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
when action in [:activity, :object]
)
plug(:log_inbox_metadata when action in [:inbox])
plug(:set_requester_reachable when action in [:inbox])
plug(:relay_active? when action in [:relay])
@ -521,6 +522,13 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
conn
end
defp log_inbox_metadata(%{params: %{"actor" => actor, "type" => type}} = conn, _) do
Logger.metadata(actor: actor, type: type)
conn
end
defp log_inbox_metadata(conn, _), do: conn
def upload_media(%{assigns: %{user: %User{} = user}} = conn, %{"file" => file} = data) do
with {:ok, object} <-
ActivityPub.upload(

87
lib/pleroma/web/activity_pub/mrf/anti_mention_spam_policy.ex Normal file
View file

@ -0,0 +1,87 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.ActivityPub.MRF.AntiMentionSpamPolicy do
alias Pleroma.Config
alias Pleroma.User
require Pleroma.Constants
@behaviour Pleroma.Web.ActivityPub.MRF.Policy
defp user_has_posted?(%User{} = u), do: u.note_count > 0
defp user_has_age?(%User{} = u) do
user_age_limit = Config.get([:mrf_antimentionspam, :user_age_limit], 30_000)
diff = NaiveDateTime.utc_now() |> NaiveDateTime.diff(u.inserted_at, :millisecond)
diff >= user_age_limit
end
defp good_reputation?(%User{} = u) do
user_has_age?(u) and user_has_posted?(u)
end
# copied from HellthreadPolicy
defp get_recipient_count(message) do
recipients = (message["to"] || []) ++ (message["cc"] || [])
follower_collection =
User.get_cached_by_ap_id(message["actor"] || message["attributedTo"]).follower_address
if Enum.member?(recipients, Pleroma.Constants.as_public()) do
recipients =
recipients
|> List.delete(Pleroma.Constants.as_public())
|> List.delete(follower_collection)
{:public, length(recipients)}
else
recipients =
recipients
|> List.delete(follower_collection)
{:not_public, length(recipients)}
end
end
defp object_has_recipients?(%{"object" => object} = activity) do
{_, object_count} = get_recipient_count(object)
{_, activity_count} = get_recipient_count(activity)
object_count + activity_count > 0
end
defp object_has_recipients?(object) do
{_, count} = get_recipient_count(object)
count > 0
end
@impl true
def filter(%{"type" => "Create", "actor" => actor} = activity) do
with {:ok, %User{local: false} = u} <- User.get_or_fetch_by_ap_id(actor),
{:has_mentions, true} <- {:has_mentions, object_has_recipients?(activity)},
{:good_reputation, true} <- {:good_reputation, good_reputation?(u)} do
{:ok, activity}
else
{:ok, %User{local: true}} ->
{:ok, activity}
{:has_mentions, false} ->
{:ok, activity}
{:good_reputation, false} ->
{:reject, "[AntiMentionSpamPolicy] User rejected"}
{:error, _} ->
{:reject, "[AntiMentionSpamPolicy] Failed to get or fetch user by ap_id"}
e ->
{:reject, "[AntiMentionSpamPolicy] Unhandled error #{inspect(e)}"}
end
end
# in all other cases, pass through
def filter(message), do: {:ok, message}
@impl true
def describe, do: {:ok, %{}}
end

146
lib/pleroma/web/activity_pub/mrf/dnsrbl_policy.ex Normal file
View file

@ -0,0 +1,146 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2024 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.ActivityPub.MRF.DNSRBLPolicy do
@moduledoc """
Dynamic activity filtering based on an RBL database
This MRF makes queries to a custom DNS server which will
respond with values indicating the classification of the domain
the activity originated from. This method has been widely used
in the email anti-spam industry for very fast reputation checks.
e.g., if the DNS response is 127.0.0.1 or empty, the domain is OK
Other values such as 127.0.0.2 may be used for specific classifications.
Information for why the host is blocked can be stored in a corresponding TXT record.
This method is fail-open so if the queries fail the activites are accepted.
An example of software meant for this purpsoe is rbldnsd which can be found
at http://www.corpit.ru/mjt/rbldnsd.html or mirrored at
https://git.pleroma.social/feld/rbldnsd
It is highly recommended that you run your own copy of rbldnsd and use an
external mechanism to sync/share the contents of the zone file. This is
important to keep the latency on the queries as low as possible and prevent
your DNS server from being attacked so it fails and content is permitted.
"""
@behaviour Pleroma.Web.ActivityPub.MRF.Policy
alias Pleroma.Config
require Logger
@query_retries 1
@query_timeout 500
@impl true
def filter(%{"actor" => actor} = object) do
actor_info = URI.parse(actor)
with {:ok, object} <- check_rbl(actor_info, object) do
{:ok, object}
else
_ -> {:reject, "[DNSRBLPolicy]"}
end
end
@impl true
def filter(object), do: {:ok, object}
@impl true
def describe do
mrf_dnsrbl =
Config.get(:mrf_dnsrbl)
|> Enum.into(%{})
{:ok, %{mrf_dnsrbl: mrf_dnsrbl}}
end
@impl true
def config_description do
%{
key: :mrf_dnsrbl,
related_policy: "Pleroma.Web.ActivityPub.MRF.DNSRBLPolicy",
label: "MRF DNSRBL",
description: "DNS RealTime Blackhole Policy",
children: [
%{
key: :nameserver,
type: {:string},
description: "DNSRBL Nameserver to Query (IP or hostame)",
suggestions: ["127.0.0.1"]
},
%{
key: :port,
type: {:string},
description: "Nameserver port",
suggestions: ["53"]
},
%{
key: :zone,
type: {:string},
description: "Root zone for querying",
suggestions: ["bl.pleroma.com"]
}
]
}
end
defp check_rbl(%{host: actor_host}, object) do
with false <- match?(^actor_host, Pleroma.Web.Endpoint.host()),
zone when not is_nil(zone) <- Keyword.get(Config.get([:mrf_dnsrbl]), :zone) do
query =
Enum.join([actor_host, zone], ".")
|> String.to_charlist()
rbl_response = rblquery(query)
if Enum.empty?(rbl_response) do
{:ok, object}
else
Task.start(fn ->
reason =
case rblquery(query, :txt) do
[[result]] -> result
_ -> "undefined"
end
Logger.warning(
"DNSRBL Rejected activity from #{actor_host} for reason: #{inspect(reason)}"
)
end)
:error
end
else
_ -> {:ok, object}
end
end
defp get_rblhost_ip(rblhost) do
case rblhost |> String.to_charlist() |> :inet_parse.address() do
{:ok, _} -> rblhost |> String.to_charlist() |> :inet_parse.address()
_ -> {:ok, rblhost |> String.to_charlist() |> :inet_res.lookup(:in, :a) |> Enum.random()}
end
end
defp rblquery(query, type \\ :a) do
config = Config.get([:mrf_dnsrbl])
case get_rblhost_ip(config[:nameserver]) do
{:ok, rblnsip} ->
:inet_res.lookup(query, :in, type,
nameservers: [{rblnsip, config[:port]}],
timeout: @query_timeout,
retry: @query_retries
)
_ ->
[]
end
end
end

18
lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex
View file

@ -11,11 +11,6 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do
require Logger
@adapter_options [
pool: :media,
recv_timeout: 10_000
]
@impl true
def history_awareness, do: :auto
@ -27,17 +22,14 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do
Logger.debug("Prefetching #{inspect(url)} as #{inspect(prefetch_url)}")
if Pleroma.Config.get(:env) == :test do
fetch(prefetch_url)
else
ConcurrentLimiter.limit(__MODULE__, fn ->
Task.start(fn -> fetch(prefetch_url) end)
end)
end
fetch(prefetch_url)
end
end
defp fetch(url), do: HTTP.get(url, [], @adapter_options)
defp fetch(url) do
http_client_opts = Pleroma.Config.get([:media_proxy, :proxy_opts, :http], pool: :media)
HTTP.get(url, [], http_client_opts)
end
defp preload(%{"object" => %{"attachment" => attachments}} = _message) do
Enum.each(attachments, fn

265
lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex Normal file
View file

@ -0,0 +1,265 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.ActivityPub.MRF.NsfwApiPolicy do
@moduledoc """
Hide, delete, or mark sensitive NSFW content with artificial intelligence.
Requires a NSFW API server, configured like so:
config :pleroma, Pleroma.Web.ActivityPub.MRF.NsfwMRF,
url: "http://127.0.0.1:5000/",
threshold: 0.7,
mark_sensitive: true,
unlist: false,
reject: false
The NSFW API server must implement an HTTP endpoint like this:
curl http://localhost:5000/?url=https://fedi.com/images/001.jpg
Returning a response like this:
{"score", 0.314}
Where a score is 0-1, with `1` being definitely NSFW.
A good API server is here: https://github.com/EugenCepoi/nsfw_api
You can run it with Docker with a one-liner:
docker run -it -p 127.0.0.1:5000:5000/tcp --env PORT=5000 eugencepoi/nsfw_api:latest
Options:
- `url`: Base URL of the API server. Default: "http://127.0.0.1:5000/"
- `threshold`: Lowest score to take action on. Default: `0.7`
- `mark_sensitive`: Mark sensitive all detected NSFW content? Default: `true`
- `unlist`: Unlist all detected NSFW content? Default: `false`
- `reject`: Reject all detected NSFW content (takes precedence)? Default: `false`
"""
alias Pleroma.Config
alias Pleroma.Constants
alias Pleroma.HTTP
alias Pleroma.User
require Logger
require Pleroma.Constants
@behaviour Pleroma.Web.ActivityPub.MRF.Policy
@policy :mrf_nsfw_api
def build_request_url(url) do
Config.get([@policy, :url])
|> URI.parse()
|> fix_path()
|> Map.put(:query, "url=#{url}")
|> URI.to_string()
end
def parse_url(url) do
request = build_request_url(url)
with {:ok, %Tesla.Env{body: body}} <- HTTP.get(request) do
Jason.decode(body)
else
error ->
Logger.warning("""
[NsfwApiPolicy]: The API server failed. Skipping.
#{inspect(error)}
""")
error
end
end
def check_url_nsfw(url) when is_binary(url) do
threshold = Config.get([@policy, :threshold])
case parse_url(url) do
{:ok, %{"score" => score}} when score >= threshold ->
{:nsfw, %{url: url, score: score, threshold: threshold}}
{:ok, %{"score" => score}} ->
{:sfw, %{url: url, score: score, threshold: threshold}}
_ ->
{:sfw, %{url: url, score: nil, threshold: threshold}}
end
end
def check_url_nsfw(%{"href" => url}) when is_binary(url) do
check_url_nsfw(url)
end
def check_url_nsfw(url) do
threshold = Config.get([@policy, :threshold])
{:sfw, %{url: url, score: nil, threshold: threshold}}
end
def check_attachment_nsfw(%{"url" => urls} = attachment) when is_list(urls) do
if Enum.all?(urls, &match?({:sfw, _}, check_url_nsfw(&1))) do
{:sfw, attachment}
else
{:nsfw, attachment}
end
end
def check_attachment_nsfw(%{"url" => url} = attachment) when is_binary(url) do
case check_url_nsfw(url) do
{:sfw, _} -> {:sfw, attachment}
{:nsfw, _} -> {:nsfw, attachment}
end
end
def check_attachment_nsfw(attachment), do: {:sfw, attachment}
def check_object_nsfw(%{"attachment" => attachments} = object) when is_list(attachments) do
if Enum.all?(attachments, &match?({:sfw, _}, check_attachment_nsfw(&1))) do
{:sfw, object}
else
{:nsfw, object}
end
end
def check_object_nsfw(%{"object" => %{} = child_object} = object) do
case check_object_nsfw(child_object) do
{:sfw, _} -> {:sfw, object}
{:nsfw, _} -> {:nsfw, object}
end
end
def check_object_nsfw(object), do: {:sfw, object}
@impl true
def filter(object) do
with {:sfw, object} <- check_object_nsfw(object) do
{:ok, object}
else
{:nsfw, _data} -> handle_nsfw(object)
_ -> {:reject, "NSFW: Attachment rejected"}
end
end
defp handle_nsfw(object) do
if Config.get([@policy, :reject]) do
{:reject, object}
else
{:ok,
object
|> maybe_unlist()
|> maybe_mark_sensitive()}
end
end
defp maybe_unlist(object) do
if Config.get([@policy, :unlist]) do
unlist(object)
else
object
end
end
defp maybe_mark_sensitive(object) do
if Config.get([@policy, :mark_sensitive]) do
mark_sensitive(object)
else
object
end
end
def unlist(%{"to" => to, "cc" => cc, "actor" => actor} = object) do
with %User{} = user <- User.get_cached_by_ap_id(actor) do
to =
[user.follower_address | to]
|> List.delete(Constants.as_public())
|> Enum.uniq()
cc =
[Constants.as_public() | cc]
|> List.delete(user.follower_address)
|> Enum.uniq()
object
|> Map.put("to", to)
|> Map.put("cc", cc)
else
_ -> raise "[NsfwApiPolicy]: Could not find user #{actor}"
end
end
def mark_sensitive(%{"object" => child_object} = object) when is_map(child_object) do
Map.put(object, "object", mark_sensitive(child_object))
end
def mark_sensitive(object) when is_map(object) do
tags = (object["tag"] || []) ++ ["nsfw"]
object
|> Map.put("tag", tags)
|> Map.put("sensitive", true)
end
# Hackney needs a trailing slash
defp fix_path(%URI{path: path} = uri) when is_binary(path) do
path = String.trim_trailing(path, "/") <> "/"
Map.put(uri, :path, path)
end
defp fix_path(%URI{path: nil} = uri), do: Map.put(uri, :path, "/")
@impl true
def describe do
options = %{
threshold: Config.get([@policy, :threshold]),
mark_sensitive: Config.get([@policy, :mark_sensitive]),
unlist: Config.get([@policy, :unlist]),
reject: Config.get([@policy, :reject])
}
{:ok, %{@policy => options}}
end
@impl true
def config_description do
%{
key: @policy,
related_policy: to_string(__MODULE__),
label: "NSFW API Policy",
description:
"Hide, delete, or mark sensitive NSFW content with artificial intelligence. Requires running an external API server.",
children: [
%{
key: :url,
type: :string,
description: "Base URL of the API server.",
suggestions: ["http://127.0.0.1:5000/"]
},
%{
key: :threshold,
type: :float,
description: "Lowest score to take action on. Between 0 and 1.",
suggestions: [0.7]
},
%{
key: :mark_sensitive,
type: :boolean,
description: "Mark sensitive all detected NSFW content?",
suggestions: [true]
},
%{
key: :unlist,
type: :boolean,
description: "Unlist sensitive all detected NSFW content?",
suggestions: [false]
},
%{
key: :reject,
type: :boolean,
description: "Reject sensitive all detected NSFW content (takes precedence)?",
suggestions: [false]
}
]
}
end
end

3
lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex
View file

@ -15,6 +15,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator do
field(:type, :string, default: "Link")
field(:mediaType, ObjectValidators.MIME, default: "application/octet-stream")
field(:name, :string)
field(:summary, :string)
field(:blurhash, :string)
embeds_many :url, UrlObjectValidator, primary_key: false do
@ -44,7 +45,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator do
|> fix_url()
struct
|> cast(data, [:id, :type, :mediaType, :name, :blurhash])
|> cast(data, [:id, :type, :mediaType, :name, :summary, :blurhash])
|> cast_embed(:url, with: &url_changeset/2, required: true)
|> validate_inclusion(:type, ~w[Link Document Audio Image Video])
|> validate_required([:type, :mediaType])

6
lib/pleroma/web/activity_pub/side_effects.ex
View file

@ -592,9 +592,9 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
with {:ok, _} <- Repo.delete(object), do: :ok
end
defp send_notifications(meta) do
defp stream_notifications(meta) do
Keyword.get(meta, :notifications, [])
|> Notification.send()
|> Notification.stream()
meta
end
@ -625,7 +625,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
@impl true
def handle_after_transaction(meta) do
meta
|> send_notifications()
|> stream_notifications()
|> send_streamables()
end
end

6
lib/pleroma/web/activity_pub/transmogrifier.ex
View file

@ -912,9 +912,11 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
def add_emoji_tags(object), do: object
defp build_emoji_tag({name, url}) do
def build_emoji_tag({name, url}) do
url = URI.encode(url)
%{
"icon" => %{"url" => "#{URI.encode(url)}", "type" => "Image"},
"icon" => %{"url" => "#{url}", "type" => "Image"},
"name" => ":" <> name <> ":",
"type" => "Emoji",
"updated" => "1970-01-01T00:00:00Z",

6
lib/pleroma/web/api_spec/cast_and_validate.ex
View file

@ -18,6 +18,8 @@ defmodule Pleroma.Web.ApiSpec.CastAndValidate do
alias OpenApiSpex.Plug.PutApiSpec
alias Plug.Conn
require Logger
@impl Plug
def init(opts) do
opts
@ -51,6 +53,10 @@ defmodule Pleroma.Web.ApiSpec.CastAndValidate do
conn
{:error, reason} ->
Logger.error(
"Strict ApiSpec: request denied to #{conn.request_path} with params #{inspect(conn.params)}"
)
opts = render_error.init(reason)
conn

43
lib/pleroma/web/api_spec/operations/account_operation.ex
View file

@ -11,6 +11,7 @@ defmodule Pleroma.Web.ApiSpec.AccountOperation do
alias Pleroma.Web.ApiSpec.Schemas.ActorType
alias Pleroma.Web.ApiSpec.Schemas.ApiError
alias Pleroma.Web.ApiSpec.Schemas.BooleanLike
alias Pleroma.Web.ApiSpec.Schemas.FlakeID
alias Pleroma.Web.ApiSpec.Schemas.List
alias Pleroma.Web.ApiSpec.Schemas.Status
alias Pleroma.Web.ApiSpec.Schemas.VisibilityScope
@ -513,6 +514,48 @@ defmodule Pleroma.Web.ApiSpec.AccountOperation do
}
end
def familiar_followers_operation do
%Operation{
tags: ["Retrieve account information"],
summary: "Followers that you follow",
operationId: "AccountController.familiar_followers",
description:
"Obtain a list of all accounts that follow a given account, filtered for accounts you follow.",
security: [%{"oAuth" => ["read:follows"]}],
parameters: [
Operation.parameter(
:id,
:query,
%Schema{
oneOf: [%Schema{type: :array, items: %Schema{type: :string}}, %Schema{type: :string}]
},
"Account IDs",
example: "123"
)
],
responses: %{
200 =>
Operation.response("Accounts", "application/json", %Schema{
title: "ArrayOfAccounts",
type: :array,
items: %Schema{
title: "Account",
type: :object,
properties: %{
id: FlakeID,
accounts: %Schema{
title: "ArrayOfAccounts",
type: :array,
items: Account,
example: [Account.schema().example]
}
}
}
})
}
}
end
defp create_request do
%Schema{
title: "AccountCreateRequest",

10
lib/pleroma/web/api_spec/operations/notification_operation.ex
View file

@ -202,7 +202,11 @@ defmodule Pleroma.Web.ApiSpec.NotificationOperation do
"pleroma:report",
"move",
"follow_request",
"poll"
"poll",
"status",
"update",
"admin.sign_up",
"admin.report"
],
description: """
The type of event that resulted in the notification.
@ -216,6 +220,10 @@ defmodule Pleroma.Web.ApiSpec.NotificationOperation do
- `pleroma:emoji_reaction` - Someone reacted with emoji to your status
- `pleroma:chat_mention` - Someone mentioned you in a chat message
- `pleroma:report` - Someone was reported
- `status` - Someone you are subscribed to created a status
- `update` - A status you boosted has been edited
- `admin.sign_up` - Someone signed up (optionally sent to admins)
- `admin.report` - A new report has been filed
"""
}
end

8
lib/pleroma/web/api_spec/operations/pleroma_notification_operation.ex
View file

@ -5,7 +5,6 @@
defmodule Pleroma.Web.ApiSpec.PleromaNotificationOperation do
alias OpenApiSpex.Operation
alias OpenApiSpex.Schema
alias Pleroma.Web.ApiSpec.NotificationOperation
alias Pleroma.Web.ApiSpec.Schemas.ApiError
import Pleroma.Web.ApiSpec.Helpers
@ -35,12 +34,7 @@ defmodule Pleroma.Web.ApiSpec.PleromaNotificationOperation do
Operation.response(
"A Notification or array of Notifications",
"application/json",
%Schema{
anyOf: [
%Schema{type: :array, items: NotificationOperation.notification()},
NotificationOperation.notification()
]
}
%Schema{type: :string}
),
400 => Operation.response("Bad Request", "application/json", ApiError)
}

6
lib/pleroma/web/api_spec/schemas/attachment.ex
View file

@ -50,7 +50,11 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Attachment do
pleroma: %Schema{
type: :object,
properties: %{
mime_type: %Schema{type: :string, description: "mime type of the attachment"}
mime_type: %Schema{type: :string, description: "mime type of the attachment"},
name: %Schema{
type: :string,
description: "Name of the attachment, typically the filename"
}
}
}
},

18
lib/pleroma/web/common_api/activity_draft.ex
View file

@ -134,8 +134,22 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do
defp in_reply_to(%{params: %{in_reply_to_status_id: ""}} = draft), do: draft
defp in_reply_to(%{params: %{in_reply_to_status_id: id}} = draft) when is_binary(id) do
%__MODULE__{draft | in_reply_to: Activity.get_by_id(id)}
defp in_reply_to(%{params: %{in_reply_to_status_id: :deleted}} = draft) do
add_error(draft, dgettext("errors", "Cannot reply to a deleted status"))
end
defp in_reply_to(%{params: %{in_reply_to_status_id: id} = params} = draft) when is_binary(id) do
activity = Activity.get_by_id(id)
params =
if is_nil(activity) do
# Deleted activities are returned as nil
Map.put(params, :in_reply_to_status_id, :deleted)
else
Map.put(params, :in_reply_to_status_id, activity)
end
in_reply_to(%{draft | params: params})
end
defp in_reply_to(%{params: %{in_reply_to_status_id: %Activity{} = in_reply_to}} = draft) do

2
lib/pleroma/web/endpoint.ex
View file

@ -38,6 +38,8 @@ defmodule Pleroma.Web.Endpoint do
plug(Plug.Telemetry, event_prefix: [:phoenix, :endpoint])
plug(Pleroma.Web.Plugs.LoggerMetadataPath)
plug(Pleroma.Web.Plugs.SetLocalePlug)
plug(CORSPlug)
plug(Pleroma.Web.Plugs.HTTPSecurityPlug)

2
lib/pleroma/web/federator.ex
View file

@ -44,7 +44,7 @@ defmodule Pleroma.Web.Federator do
end
def incoming_ap_doc(%{"type" => "Delete"} = params) do
ReceiverWorker.enqueue("incoming_ap_doc", %{"params" => params}, priority: 3)
ReceiverWorker.enqueue("incoming_ap_doc", %{"params" => params}, priority: 3, queue: :slow)
end
def incoming_ap_doc(params) do

34
lib/pleroma/web/mastodon_api/controllers/account_controller.ex
View file

@ -72,7 +72,10 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do
%{scopes: ["follow", "write:blocks"]} when action in [:block, :unblock]
)
plug(OAuthScopesPlug, %{scopes: ["read:follows"]} when action == :relationships)
plug(
OAuthScopesPlug,
%{scopes: ["read:follows"]} when action in [:relationships, :familiar_followers]
)
plug(
OAuthScopesPlug,
@ -629,6 +632,35 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do
)
end
@doc "GET /api/v1/accounts/familiar_followers"
def familiar_followers(
%{assigns: %{user: user}, private: %{open_api_spex: %{params: %{id: id}}}} = conn,
_id
) do
users =
User.get_all_by_ids(List.wrap(id))
|> Enum.map(&%{id: &1.id, accounts: get_familiar_followers(&1, user)})
conn
|> render("familiar_followers.json",
for: user,
users: users,
as: :user
)
end
defp get_familiar_followers(%{id: id} = user, %{id: id}) do
User.get_familiar_followers(user, user)
end
defp get_familiar_followers(%{hide_followers: true}, _current_user) do
[]
end
defp get_familiar_followers(user, current_user) do
User.get_familiar_followers(user, current_user)
end
@doc "GET /api/v1/identity_proofs"
def identity_proofs(conn, params), do: MastodonAPIController.empty_array(conn, params)
end

1
lib/pleroma/web/mastodon_api/controllers/notification_controller.ex
View file

@ -34,6 +34,7 @@ defmodule Pleroma.Web.MastodonAPI.NotificationController do
pleroma:emoji_reaction
poll
update
status
}
# GET /api/v1/notifications

19
lib/pleroma/web/mastodon_api/views/account_view.ex
View file

@ -193,6 +193,25 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do
render_many(targets, AccountView, "relationship.json", render_opts)
end
def render("familiar_followers.json", %{users: users} = opts) do
opts =
opts
|> Map.merge(%{as: :user})
|> Map.delete(:users)
users
|> render_many(AccountView, "familiar_followers.json", opts)
end
def render("familiar_followers.json", %{user: %{id: id, accounts: accounts}} = opts) do
accounts =
accounts
|> render_many(AccountView, "show.json", opts)
|> Enum.filter(&Enum.any?/1)
%{id: id, accounts: accounts}
end
defp do_render("show.json", %{user: user} = opts) do
self = opts[:for] == user

7
lib/pleroma/web/mastodon_api/views/instance_view.ex
View file

@ -164,6 +164,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do
def federation do
quarantined = Config.get([:instance, :quarantined_instances], [])
rejected = Config.get([:instance, :rejected_instances], [])
if Config.get([:mrf, :transparency]) do
{:ok, data} = MRF.describe()
@ -183,6 +184,12 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do
|> Enum.map(fn {instance, reason} -> {instance, %{"reason" => reason}} end)
|> Map.new()
})
|> Map.put(
:rejected_instances,
rejected
|> Enum.map(fn {instance, reason} -> {instance, %{"reason" => reason}} end)
|> Map.new()
)
else
%{}
end

3
lib/pleroma/web/mastodon_api/views/notification_view.ex
View file

@ -108,6 +108,9 @@ defmodule Pleroma.Web.MastodonAPI.NotificationView do
"mention" ->
put_status(response, activity, reading_user, status_render_opts)
"status" ->
put_status(response, activity, reading_user, status_render_opts)
"favourite" ->
put_status(response, parent_activity_fn.(), reading_user, status_render_opts)

17
lib/pleroma/web/mastodon_api/views/status_view.ex
View file

@ -624,6 +624,19 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
to_string(attachment["id"] || hash_id)
end
description =
if attachment["summary"] do
HTML.strip_tags(attachment["summary"])
else
attachment["name"]
end
name = if attachment["summary"], do: attachment["name"]
pleroma =
%{mime_type: media_type}
|> Maps.put_if_present(:name, name)
%{
id: attachment_id,
url: href,
@ -631,8 +644,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
preview_url: href_preview,
text_url: href,
type: type,
description: attachment["name"],
pleroma: %{mime_type: media_type},
description: description,
pleroma: pleroma,
blurhash: attachment["blurhash"]
}
|> Maps.put_if_present(:meta, meta)

3
lib/pleroma/web/media_proxy/media_proxy_controller.ex
View file

@ -54,9 +54,10 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
defp handle_preview(conn, url) do
media_proxy_url = MediaProxy.url(url)
http_client_opts = Pleroma.Config.get([:media_proxy, :proxy_opts, :http], pool: :media)
with {:ok, %{status: status} = head_response} when status in 200..299 <-
Pleroma.HTTP.request(:head, media_proxy_url, "", [], pool: :media) do
Pleroma.HTTP.request(:head, media_proxy_url, "", [], http_client_opts) do
content_type = Tesla.get_header(head_response, "content-type")
content_length = Tesla.get_header(head_response, "content-length")
content_length = content_length && String.to_integer(content_length)

2
lib/pleroma/web/o_auth/token.ex
View file

@ -96,7 +96,7 @@ defmodule Pleroma.Web.OAuth.Token do
|> validate_required([:valid_until])
end
@spec create(App.t(), User.t(), map()) :: {:ok, Token} | {:error, Ecto.Changeset.t()}
@spec create(App.t(), User.t(), map()) :: {:ok, Token.t()} | {:error, Ecto.Changeset.t()}
def create(%App{} = app, %User{} = user, attrs \\ %{}) do
with {:ok, token} <- do_create(app, user, attrs) do
if Pleroma.Config.get([:oauth2, :clean_expired_tokens]) do

20
lib/pleroma/web/pleroma_api/controllers/notification_controller.ex
View file

@ -23,8 +23,9 @@ defmodule Pleroma.Web.PleromaAPI.NotificationController do
} = conn,
_
) do
with {:ok, notification} <- Notification.read_one(user, notification_id) do
render(conn, "show.json", notification: notification, for: user)
with {:ok, _} <- Notification.read_one(user, notification_id) do
conn
|> json("ok")
else
{:error, message} ->
conn
@ -38,11 +39,14 @@ defmodule Pleroma.Web.PleromaAPI.NotificationController do
conn,
_
) do
notifications =
user
|> Notification.set_read_up_to(max_id)
|> Enum.take(80)
render(conn, "index.json", notifications: notifications, for: user)
with {:ok, _} <- Notification.set_read_up_to(user, max_id) do
conn
|> json("ok")
else
{:error, message} ->
conn
|> put_status(:bad_request)
|> json(%{"error" => message})
end
end
end

49
lib/pleroma/web/plugs/http_security_plug.ex
View file

@ -3,26 +3,27 @@
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do
alias Pleroma.Config
import Plug.Conn
require Logger
@config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config)
def init(opts), do: opts
def call(conn, _options) do
if Config.get([:http_security, :enabled]) do
if @config_impl.get([:http_security, :enabled]) do
conn
|> merge_resp_headers(headers())
|> maybe_send_sts_header(Config.get([:http_security, :sts]))
|> maybe_send_sts_header(@config_impl.get([:http_security, :sts]))
else
conn
end
end
def primary_frontend do
with %{"name" => frontend} <- Config.get([:frontends, :primary]),
available <- Config.get([:frontends, :available]),
with %{"name" => frontend} <- @config_impl.get([:frontends, :primary]),
available <- @config_impl.get([:frontends, :available]),
%{} = primary_frontend <- Map.get(available, frontend) do
{:ok, primary_frontend}
end
@ -37,8 +38,8 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do
end
def headers do
referrer_policy = Config.get([:http_security, :referrer_policy])
report_uri = Config.get([:http_security, :report_uri])
referrer_policy = @config_impl.get([:http_security, :referrer_policy])
report_uri = @config_impl.get([:http_security, :report_uri])
custom_http_frontend_headers = custom_http_frontend_headers()
headers = [
@ -86,10 +87,10 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do
@csp_start [Enum.join(static_csp_rules, ";") <> ";"]
defp csp_string do
scheme = Config.get([Pleroma.Web.Endpoint, :url])[:scheme]
scheme = @config_impl.get([Pleroma.Web.Endpoint, :url])[:scheme]
static_url = Pleroma.Web.Endpoint.static_url()
websocket_url = Pleroma.Web.Endpoint.websocket_url()
report_uri = Config.get([:http_security, :report_uri])
report_uri = @config_impl.get([:http_security, :report_uri])
img_src = "img-src 'self' data: blob:"
media_src = "media-src 'self'"
@ -97,8 +98,8 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do
# Strict multimedia CSP enforcement only when MediaProxy is enabled
{img_src, media_src, connect_src} =
if Config.get([:media_proxy, :enabled]) &&
!Config.get([:media_proxy, :proxy_opts, :redirect_on_failure]) do
if @config_impl.get([:media_proxy, :enabled]) &&
!@config_impl.get([:media_proxy, :proxy_opts, :redirect_on_failure]) do
sources = build_csp_multimedia_source_list()
{
@ -115,17 +116,21 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do
end
connect_src =
if Config.get(:env) == :dev do
if @config_impl.get([:env]) == :dev do
[connect_src, " http://localhost:3035/"]
else
connect_src
end
script_src =
if Config.get(:env) == :dev do
"script-src 'self' 'unsafe-eval'"
if @config_impl.get([:http_security, :allow_unsafe_eval]) do
if @config_impl.get([:env]) == :dev do
"script-src 'self' 'unsafe-eval'"
else
"script-src 'self' 'wasm-unsafe-eval'"
end
else
"script-src 'self' 'wasm-unsafe-eval'"
"script-src 'self'"
end
report = if report_uri, do: ["report-uri ", report_uri, ";report-to csp-endpoint"]
@ -161,11 +166,11 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do
defp build_csp_multimedia_source_list do
media_proxy_whitelist =
[:media_proxy, :whitelist]
|> Config.get()
|> @config_impl.get()
|> build_csp_from_whitelist([])
captcha_method = Config.get([Pleroma.Captcha, :method])
captcha_endpoint = Config.get([captcha_method, :endpoint])
captcha_method = @config_impl.get([Pleroma.Captcha, :method])
captcha_endpoint = @config_impl.get([captcha_method, :endpoint])
base_endpoints =
[
@ -173,7 +178,7 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do
[Pleroma.Upload, :base_url],
[Pleroma.Uploaders.S3, :public_endpoint]
]
|> Enum.map(&Config.get/1)
|> Enum.map(&@config_impl.get/1)
[captcha_endpoint | base_endpoints]
|> Enum.map(&build_csp_param/1)
@ -200,7 +205,7 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do
end
def warn_if_disabled do
unless Config.get([:http_security, :enabled]) do
unless Pleroma.Config.get([:http_security, :enabled]) do
Logger.warning("
.i;;;;i.
iYcviii;vXY:
@ -245,8 +250,8 @@ your instance and your users via malicious posts:
end
defp maybe_send_sts_header(conn, true) do
max_age_sts = Config.get([:http_security, :sts_max_age])
max_age_ct = Config.get([:http_security, :ct_max_age])
max_age_sts = @config_impl.get([:http_security, :sts_max_age])
max_age_ct = @config_impl.get([:http_security, :ct_max_age])
merge_resp_headers(conn, [
{"strict-transport-security", "max-age=#{max_age_sts}; includeSubDomains"},

71
lib/pleroma/web/plugs/http_signature_plug.ex
View file

@ -3,10 +3,22 @@
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.Plugs.HTTPSignaturePlug do
alias Pleroma.Helpers.InetHelper
import Plug.Conn
import Phoenix.Controller, only: [get_format: 1, text: 2]
alias Pleroma.Web.ActivityPub.MRF
require Logger
@config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config)
@http_signatures_impl Application.compile_env(
:pleroma,
[__MODULE__, :http_signatures_impl],
HTTPSignatures
)
def init(options) do
options
end
@ -19,7 +31,9 @@ defmodule Pleroma.Web.Plugs.HTTPSignaturePlug do
if get_format(conn) in ["json", "activity+json"] do
conn
|> maybe_assign_valid_signature()
|> maybe_assign_actor_id()
|> maybe_require_signature()
|> maybe_filter_requests()
else
conn
end
@ -33,7 +47,7 @@ defmodule Pleroma.Web.Plugs.HTTPSignaturePlug do
|> put_req_header("(request-target)", request_target)
|> put_req_header("@request-target", request_target)
HTTPSignatures.validate_conn(conn)
@http_signatures_impl.validate_conn(conn)
end
defp validate_signature(conn) do
@ -83,20 +97,63 @@ defmodule Pleroma.Web.Plugs.HTTPSignaturePlug do
end
end
defp maybe_assign_actor_id(%{assigns: %{valid_signature: true}} = conn) do
adapter = Application.get_env(:http_signatures, :adapter)
{:ok, actor_id} = adapter.get_actor_id(conn)
assign(conn, :actor_id, actor_id)
end
defp maybe_assign_actor_id(conn), do: conn
defp has_signature_header?(conn) do
conn |> get_req_header("signature") |> Enum.at(0, false)
end
defp maybe_require_signature(%{assigns: %{valid_signature: true}} = conn), do: conn
defp maybe_require_signature(conn) do
if Pleroma.Config.get([:activitypub, :authorized_fetch_mode], false) do
conn
|> put_status(:unauthorized)
|> text("Request not signed")
|> halt()
defp maybe_require_signature(%{remote_ip: remote_ip} = conn) do
if @config_impl.get([:activitypub, :authorized_fetch_mode], false) do
exceptions =
@config_impl.get([:activitypub, :authorized_fetch_mode_exceptions], [])
|> Enum.map(&InetHelper.parse_cidr/1)
if Enum.any?(exceptions, fn x -> InetCidr.contains?(x, remote_ip) end) do
conn
else
conn
|> put_status(:unauthorized)
|> text("Request not signed")
|> halt()
end
else
conn
end
end
defp maybe_filter_requests(%{halted: true} = conn), do: conn
defp maybe_filter_requests(conn) do
if @config_impl.get([:activitypub, :authorized_fetch_mode], false) and
conn.assigns[:actor_id] do
%{host: host} = URI.parse(conn.assigns.actor_id)
if MRF.subdomain_match?(rejected_domains(), host) do
conn
|> put_status(:unauthorized)
|> halt()
else
conn
end
else
conn
end
end
defp rejected_domains do
@config_impl.get([:instance, :rejected_instances])
|> Pleroma.Web.ActivityPub.MRF.instance_list_from_tuples()
|> Pleroma.Web.ActivityPub.MRF.subdomains_regex()
end
end

12
lib/pleroma/web/plugs/logger_metadata_path.ex Normal file
View file

@ -0,0 +1,12 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.Plugs.LoggerMetadataPath do
def init(opts), do: opts
def call(conn, _) do
Logger.metadata(path: conn.request_path)
conn
end
end

18
lib/pleroma/web/plugs/logger_metadata_user.ex Normal file
View file

@ -0,0 +1,18 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.Plugs.LoggerMetadataUser do
alias Pleroma.User
def init(opts), do: opts
def call(%{assigns: %{user: user = %User{}}} = conn, _) do
Logger.metadata(user: user.nickname)
conn
end
def call(conn, _) do
conn
end
end

14
lib/pleroma/web/plugs/remote_ip.ex
View file

@ -8,6 +8,7 @@ defmodule Pleroma.Web.Plugs.RemoteIp do
"""
alias Pleroma.Config
alias Pleroma.Helpers.InetHelper
import Plug.Conn
@behaviour Plug
@ -30,19 +31,8 @@ defmodule Pleroma.Web.Plugs.RemoteIp do
proxies =
Config.get([__MODULE__, :proxies], [])
|> Enum.concat(reserved)
|> Enum.map(&maybe_add_cidr/1)
|> Enum.map(&InetHelper.parse_cidr/1)
{headers, proxies}
end
defp maybe_add_cidr(proxy) when is_binary(proxy) do
proxy =
cond do
"/" in String.codepoints(proxy) -> proxy
InetCidr.v4?(InetCidr.parse_address!(proxy)) -> proxy <> "/32"
InetCidr.v6?(InetCidr.parse_address!(proxy)) -> proxy <> "/128"
end
InetCidr.parse_cidr!(proxy, true)
end
end

12
lib/pleroma/web/push.ex
View file

@ -20,17 +20,13 @@ defmodule Pleroma.Web.Push do
end
def vapid_config do
Application.get_env(:web_push_encryption, :vapid_details, [])
Application.get_env(:web_push_encryption, :vapid_details, nil)
end
def enabled do
case vapid_config() do
[] -> false
list when is_list(list) -> true
_ -> false
end
end
def enabled, do: match?([subject: _, public_key: _, private_key: _], vapid_config())
@spec send(Pleroma.Notification.t()) ::
{:ok, Oban.Job.t()} | {:error, Oban.Job.changeset() | term()}
def send(notification) do
WebPusherWorker.enqueue("web_push", %{"notification_id" => notification.id})
end

194
lib/pleroma/web/push/impl.ex
View file

@ -16,62 +16,70 @@ defmodule Pleroma.Web.Push.Impl do
require Logger
import Ecto.Query
@body_chars 140
@types ["Create", "Follow", "Announce", "Like", "Move", "EmojiReact", "Update"]
@doc "Performs sending notifications for user subscriptions"
@spec perform(Notification.t()) :: list(any) | :error | {:error, :unknown_type}
def perform(
@doc "Builds webpush notification payloads for the subscriptions enabled by the receiving user"
@spec build(Notification.t()) ::
list(%{content: map(), subscription: Subscription.t()}) | []
def build(
%{
activity: %{data: %{"type" => activity_type}} = activity,
user: %User{id: user_id}
user_id: user_id
} = notification
)
when activity_type in @types do
actor = User.get_cached_by_ap_id(notification.activity.data["actor"])
notification_actor = User.get_cached_by_ap_id(notification.activity.data["actor"])
avatar_url = User.avatar_url(notification_actor)
mastodon_type = notification.type
gcm_api_key = Application.get_env(:web_push_encryption, :gcm_api_key)
avatar_url = User.avatar_url(actor)
object = Object.normalize(activity, fetch: false)
user = User.get_cached_by_id(user_id)
direct_conversation_id = Activity.direct_conversation_id(activity, user)
for subscription <- fetch_subscriptions(user_id),
Subscription.enabled?(subscription, mastodon_type) do
%{
access_token: subscription.token.token,
notification_id: notification.id,
notification_type: mastodon_type,
icon: avatar_url,
preferred_locale: "en",
pleroma: %{
activity_id: notification.activity.id,
direct_conversation_id: direct_conversation_id
subscriptions = fetch_subscriptions(user_id)
subscriptions
|> Enum.filter(&Subscription.enabled?(&1, notification.type))
|> Enum.map(fn subscription ->
payload =
%{
access_token: subscription.token.token,
notification_id: notification.id,
notification_type: notification.type,
icon: avatar_url,
preferred_locale: "en",
pleroma: %{
activity_id: notification.activity.id,
direct_conversation_id: direct_conversation_id
}
}
}
|> Map.merge(build_content(notification, actor, object, mastodon_type))
|> Jason.encode!()
|> push_message(build_sub(subscription), gcm_api_key, subscription)
end
|> (&{:ok, &1}).()
|> Map.merge(build_content(notification, notification_actor, object))
|> Jason.encode!()
%{payload: payload, subscription: subscription}
end)
end
def perform(_) do
Logger.warning("Unknown notification type")
{:error, :unknown_type}
def build(notif) do
Logger.warning("WebPush: unknown activity type: #{inspect(notif)}")
[]
end
@doc "Push message to web"
def push_message(body, sub, api_key, subscription) do
case WebPushEncryption.send_web_push(body, sub, api_key) do
@doc "Deliver push notification to the provided webpush subscription"
@spec deliver(%{payload: String.t(), subscription: Subscription.t()}) :: :ok | :error
def deliver(%{payload: payload, subscription: subscription}) do
gcm_api_key = Application.get_env(:web_push_encryption, :gcm_api_key)
formatted_subscription = build_sub(subscription)
case WebPushEncryption.send_web_push(payload, formatted_subscription, gcm_api_key) do
{:ok, %{status: code}} when code in 200..299 ->
:ok
{:ok, %{status: code}} when code in 400..499 ->
Logger.debug("Removing subscription record")
Repo.delete!(subscription)
:ok
{:ok, %{status: code}} when code in 200..299 ->
:ok
{:ok, %{status: code}} ->
Logger.error("Web Push Notification failed with code: #{code}")
:error
@ -100,106 +108,106 @@ defmodule Pleroma.Web.Push.Impl do
}
end
def build_content(notification, actor, object, mastodon_type \\ nil)
def build_content(
%{
user: %{notification_settings: %{hide_notification_contents: true}}
} = notification,
_actor,
_object,
mastodon_type
_user,
_object
) do
%{body: format_title(notification, mastodon_type)}
%{body: format_title(notification)}
end
def build_content(notification, actor, object, mastodon_type) do
mastodon_type = mastodon_type || notification.type
def build_content(notification, user, object) do
%{
title: format_title(notification, mastodon_type),
body: format_body(notification, actor, object, mastodon_type)
title: format_title(notification),
body: format_body(notification, user, object)
}
end
def format_body(activity, actor, object, mastodon_type \\ nil)
def format_body(_activity, actor, %{data: %{"type" => "ChatMessage"} = data}, _) do
case data["content"] do
nil -> "@#{actor.nickname}: (Attachment)"
content -> "@#{actor.nickname}: #{Utils.scrub_html_and_truncate(content, 80)}"
@spec format_body(Notification.t(), User.t(), Object.t()) :: String.t()
def format_body(_notification, user, %{data: %{"type" => "ChatMessage"} = object}) do
case object["content"] do
nil -> "@#{user.nickname}: (Attachment)"
content -> "@#{user.nickname}: #{Utils.scrub_html_and_truncate(content, @body_chars)}"
end
end
def format_body(
%{activity: %{data: %{"type" => "Create"}}},
actor,
%{data: %{"content" => content}},
_mastodon_type
%{type: "poll"} = _notification,
_user,
%{data: %{"content" => content} = data} = _object
) do
"@#{actor.nickname}: #{Utils.scrub_html_and_truncate(content, 80)}"
options = Map.get(data, "anyOf") || Map.get(data, "oneOf")
content_text = content <> "\n"
options_text = Enum.map_join(options, "\n", fn x -> "○ #{x["name"]}" end)
[content_text, options_text]
|> Enum.join("\n")
|> Utils.scrub_html_and_truncate(@body_chars)
end
def format_body(
%{activity: %{data: %{"type" => "Create"}}},
user,
%{data: %{"content" => content}}
) do
"@#{user.nickname}: #{Utils.scrub_html_and_truncate(content, @body_chars)}"
end
def format_body(
%{activity: %{data: %{"type" => "Announce"}}},
actor,
%{data: %{"content" => content}},
_mastodon_type
user,
%{data: %{"content" => content}}
) do
"@#{actor.nickname} repeated: #{Utils.scrub_html_and_truncate(content, 80)}"
"@#{user.nickname} repeated: #{Utils.scrub_html_and_truncate(content, @body_chars)}"
end
def format_body(
%{activity: %{data: %{"type" => "EmojiReact", "content" => content}}},
actor,
_object,
_mastodon_type
user,
_object
) do
"@#{actor.nickname} reacted with #{content}"
"@#{user.nickname} reacted with #{content}"
end
def format_body(
%{activity: %{data: %{"type" => type}}} = notification,
actor,
_object,
mastodon_type
user,
_object
)
when type in ["Follow", "Like"] do
mastodon_type = mastodon_type || notification.type
case mastodon_type do
"follow" -> "@#{actor.nickname} has followed you"
"follow_request" -> "@#{actor.nickname} has requested to follow you"
"favourite" -> "@#{actor.nickname} has favorited your post"
case notification.type do
"follow" -> "@#{user.nickname} has followed you"
"follow_request" -> "@#{user.nickname} has requested to follow you"
"favourite" -> "@#{user.nickname} has favorited your post"
end
end
def format_body(
%{activity: %{data: %{"type" => "Update"}}},
actor,
_object,
_mastodon_type
user,
_object
) do
"@#{actor.nickname} edited a status"
"@#{user.nickname} edited a status"
end
def format_title(activity, mastodon_type \\ nil)
def format_title(%{activity: %{data: %{"directMessage" => true}}}, _mastodon_type) do
@spec format_title(Notification.t()) :: String.t()
def format_title(%{activity: %{data: %{"directMessage" => true}}}) do
"New Direct Message"
end
def format_title(%{type: type}, mastodon_type) do
case mastodon_type || type do
"mention" -> "New Mention"
"follow" -> "New Follower"
"follow_request" -> "New Follow Request"
"reblog" -> "New Repeat"
"favourite" -> "New Favorite"
"update" -> "New Update"
"pleroma:chat_mention" -> "New Chat Message"
"pleroma:emoji_reaction" -> "New Reaction"
type -> "New #{String.capitalize(type || "event")}"
end
end
def format_title(%{type: "mention"}), do: "New Mention"
def format_title(%{type: "status"}), do: "New Status"
def format_title(%{type: "follow"}), do: "New Follower"
def format_title(%{type: "follow_request"}), do: "New Follow Request"
def format_title(%{type: "reblog"}), do: "New Repeat"
def format_title(%{type: "favourite"}), do: "New Favorite"
def format_title(%{type: "update"}), do: "New Update"
def format_title(%{type: "pleroma:chat_mention"}), do: "New Chat Message"
def format_title(%{type: "pleroma:emoji_reaction"}), do: "New Reaction"
def format_title(%{type: "poll"}), do: "Poll Results"
def format_title(%{type: type}), do: "New #{String.capitalize(type || "event")}"
end

25
lib/pleroma/web/rich_media/card.ex
View file

@ -77,19 +77,23 @@ defmodule Pleroma.Web.RichMedia.Card do
@spec get_or_backfill_by_url(String.t(), map()) :: t() | nil
def get_or_backfill_by_url(url, backfill_opts \\ %{}) do
case get_by_url(url) do
%__MODULE__{} = card ->
card
if @config_impl.get([:rich_media, :enabled]) do
case get_by_url(url) do
%__MODULE__{} = card ->
card
nil ->
backfill_opts = Map.put(backfill_opts, :url, url)
nil ->
backfill_opts = Map.put(backfill_opts, :url, url)
Backfill.start(backfill_opts)
Backfill.start(backfill_opts)
nil
nil
:error ->
nil
:error ->
nil
end
else
nil
end
end
@ -104,7 +108,8 @@ defmodule Pleroma.Web.RichMedia.Card do
@spec get_by_activity(Activity.t()) :: t() | nil | :error
# Fake/Draft activity
def get_by_activity(%Activity{id: "pleroma:fakeid"} = activity) do
with %Object{} = object <- Object.normalize(activity, fetch: false),
with {_, true} <- {:config, @config_impl.get([:rich_media, :enabled])},
%Object{} = object <- Object.normalize(activity, fetch: false),
url when not is_nil(url) <- HTML.extract_first_external_url_from_object(object) do
case get_by_url(url) do
# Cache hit

2
lib/pleroma/web/rich_media/helpers.ex
View file

@ -58,7 +58,7 @@ defmodule Pleroma.Web.RichMedia.Helpers do
defp http_options do
[
pool: :media,
pool: :rich_media,
max_body: Config.get([:rich_media, :max_body], 5_000_000)
]
end

6
lib/pleroma/web/rich_media/parser.ex
View file

@ -15,10 +15,14 @@ defmodule Pleroma.Web.RichMedia.Parser do
@spec parse(String.t()) :: {:ok, map()} | {:error, any()}
def parse(url) do
with :ok <- validate_page_url(url),
with {_, true} <- {:config, @config_impl.get([:rich_media, :enabled])},
:ok <- validate_page_url(url),
{:ok, data} <- parse_url(url) do
data = Map.put(data, "url", url)
{:ok, data}
else
{:config, _} -> {:error, :rich_media_disabled}
e -> e
end
end

2
lib/pleroma/web/rich_media/parser/ttl/aws_signed_url.ex
View file

@ -23,7 +23,7 @@ defmodule Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl do
%URI{host: host, query: query} = URI.parse(image)
is_binary(host) and String.contains?(host, "amazonaws.com") and
String.contains?(query, "X-Amz-Expires")
is_binary(query) and String.contains?(query, "X-Amz-Expires")
end
defp aws_signed_url?(_), do: nil

9
lib/pleroma/web/router.ex
View file

@ -29,6 +29,7 @@ defmodule Pleroma.Web.Router do
pipeline :browser do
plug(:accepts, ["html"])
plug(:fetch_session)
plug(Pleroma.Web.Plugs.LoggerMetadataUser)
end
pipeline :oauth do
@ -67,12 +68,14 @@ defmodule Pleroma.Web.Router do
plug(:fetch_session)
plug(:authenticate)
plug(OpenApiSpex.Plug.PutApiSpec, module: Pleroma.Web.ApiSpec)
plug(Pleroma.Web.Plugs.LoggerMetadataUser)
end
pipeline :no_auth_or_privacy_expectations_api do
plug(:base_api)
plug(:after_auth)
plug(Pleroma.Web.Plugs.IdempotencyPlug)
plug(Pleroma.Web.Plugs.LoggerMetadataUser)
end
# Pipeline for app-related endpoints (no user auth checks — app-bound tokens must be supported)
@ -83,12 +86,14 @@ defmodule Pleroma.Web.Router do
pipeline :api do
plug(:expect_public_instance_or_user_authentication)
plug(:no_auth_or_privacy_expectations_api)
plug(Pleroma.Web.Plugs.LoggerMetadataUser)
end
pipeline :authenticated_api do
plug(:expect_user_authentication)
plug(:no_auth_or_privacy_expectations_api)
plug(Pleroma.Web.Plugs.EnsureAuthenticatedPlug)
plug(Pleroma.Web.Plugs.LoggerMetadataUser)
end
pipeline :admin_api do
@ -99,6 +104,7 @@ defmodule Pleroma.Web.Router do
plug(Pleroma.Web.Plugs.EnsureAuthenticatedPlug)
plug(Pleroma.Web.Plugs.UserIsStaffPlug)
plug(Pleroma.Web.Plugs.IdempotencyPlug)
plug(Pleroma.Web.Plugs.LoggerMetadataUser)
end
pipeline :require_admin do
@ -179,6 +185,7 @@ defmodule Pleroma.Web.Router do
plug(:browser)
plug(:authenticate)
plug(Pleroma.Web.Plugs.EnsureUserTokenAssignsPlug)
plug(Pleroma.Web.Plugs.LoggerMetadataUser)
end
pipeline :well_known do
@ -193,6 +200,7 @@ defmodule Pleroma.Web.Router do
pipeline :pleroma_api do
plug(:accepts, ["html", "json"])
plug(OpenApiSpex.Plug.PutApiSpec, module: Pleroma.Web.ApiSpec)
plug(Pleroma.Web.Plugs.LoggerMetadataUser)
end
pipeline :mailbox_preview do
@ -638,6 +646,7 @@ defmodule Pleroma.Web.Router do
patch("/accounts/update_credentials", AccountController, :update_credentials)
get("/accounts/relationships", AccountController, :relationships)
get("/accounts/familiar_followers", AccountController, :familiar_followers)
get("/accounts/:id/lists", AccountController, :lists)
get("/accounts/:id/identity_proofs", AccountController, :identity_proofs)
get("/endorsements", AccountController, :endorsements)

38
lib/pleroma/web/web_finger.ex
View file

@ -155,7 +155,16 @@ defmodule Pleroma.Web.WebFinger do
end
end
@cachex Pleroma.Config.get([:cachex, :provider], Cachex)
def find_lrdd_template(domain) do
@cachex.fetch!(:host_meta_cache, domain, fn _ ->
{:commit, fetch_lrdd_template(domain)}
end)
rescue
e -> {:error, "Cachex error: #{inspect(e)}"}
end
defp fetch_lrdd_template(domain) do
# WebFinger is restricted to HTTPS - https://tools.ietf.org/html/rfc7033#section-9.1
meta_url = "https://#{domain}/.well-known/host-meta"
@ -168,7 +177,7 @@ defmodule Pleroma.Web.WebFinger do
end
end
defp get_address_from_domain(domain, encoded_account) when is_binary(domain) do
defp get_address_from_domain(domain, "acct:" <> _ = encoded_account) when is_binary(domain) do
case find_lrdd_template(domain) do
{:ok, template} ->
String.replace(template, "{uri}", encoded_account)
@ -178,6 +187,11 @@ defmodule Pleroma.Web.WebFinger do
end
end
defp get_address_from_domain(domain, account) when is_binary(domain) do
encoded_account = URI.encode("acct:#{account}")
get_address_from_domain(domain, encoded_account)
end
defp get_address_from_domain(_, _), do: {:error, :webfinger_no_domain}
@spec finger(String.t()) :: {:ok, map()} | {:error, any()}
@ -192,9 +206,7 @@ defmodule Pleroma.Web.WebFinger do
URI.parse(account).host
end
encoded_account = URI.encode("acct:#{account}")
with address when is_binary(address) <- get_address_from_domain(domain, encoded_account),
with address when is_binary(address) <- get_address_from_domain(domain, account),
{:ok, %{status: status, body: body, headers: headers}} when status in 200..299 <-
HTTP.get(
address,
@ -216,10 +228,28 @@ defmodule Pleroma.Web.WebFinger do
_ ->
{:error, {:content_type, nil}}
end
|> case do
{:ok, data} -> validate_webfinger(address, data)
error -> error
end
else
error ->
Logger.debug("Couldn't finger #{account}: #{inspect(error)}")
error
end
end
defp validate_webfinger(request_url, %{"subject" => "acct:" <> acct = subject} = data) do
with [_name, acct_host] <- String.split(acct, "@"),
{_, url} <- {:address, get_address_from_domain(acct_host, subject)},
%URI{host: request_host} <- URI.parse(request_url),
%URI{host: acct_host} <- URI.parse(url),
{_, true} <- {:hosts_match, acct_host == request_host} do
{:ok, data}
else
_ -> {:error, {:webfinger_invalid, request_url, data}}
end
end
defp validate_webfinger(url, data), do: {:error, {:webfinger_invalid, url, data}}
end

2
lib/pleroma/workers/attachments_cleanup_worker.ex
View file

@ -8,7 +8,7 @@ defmodule Pleroma.Workers.AttachmentsCleanupWorker do
alias Pleroma.Object
alias Pleroma.Repo
use Pleroma.Workers.WorkerHelper, queue: "attachments_cleanup"
use Pleroma.Workers.WorkerHelper, queue: "slow"
@impl Oban.Worker
def perform(%Job{

2
lib/pleroma/workers/backup_worker.ex
View file

@ -3,7 +3,7 @@
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Workers.BackupWorker do
use Oban.Worker, queue: :backup, max_attempts: 1
use Oban.Worker, queue: :slow, max_attempts: 1
alias Oban.Job
alias Pleroma.User.Backup

2
lib/pleroma/workers/cron/new_users_digest_worker.ex
View file

@ -9,7 +9,7 @@ defmodule Pleroma.Workers.Cron.NewUsersDigestWorker do
import Ecto.Query
use Pleroma.Workers.WorkerHelper, queue: "mailer"
use Pleroma.Workers.WorkerHelper, queue: "background"
@impl Oban.Worker
def perform(_job) do

2
lib/pleroma/workers/mailer_worker.ex
View file

@ -3,7 +3,7 @@
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Workers.MailerWorker do
use Pleroma.Workers.WorkerHelper, queue: "mailer"
use Pleroma.Workers.WorkerHelper, queue: "background"
@impl Oban.Worker
def perform(%Job{args: %{"op" => "email", "encoded_email" => encoded_email, "config" => config}}) do

2
lib/pleroma/workers/mute_expire_worker.ex
View file

@ -3,7 +3,7 @@
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Workers.MuteExpireWorker do
use Pleroma.Workers.WorkerHelper, queue: "mute_expire"
use Pleroma.Workers.WorkerHelper, queue: "background"
@impl Oban.Worker
def perform(%Job{args: %{"op" => "unmute_user", "muter_id" => muter_id, "mutee_id" => mutee_id}}) do

7
lib/pleroma/workers/poll_worker.ex
View file

@ -6,7 +6,7 @@ defmodule Pleroma.Workers.PollWorker do
@moduledoc """
Generates notifications when a poll ends.
"""
use Pleroma.Workers.WorkerHelper, queue: "poll_notifications"
use Pleroma.Workers.WorkerHelper, queue: "background"
alias Pleroma.Activity
alias Pleroma.Notification
@ -14,8 +14,9 @@ defmodule Pleroma.Workers.PollWorker do
@impl Oban.Worker
def perform(%Job{args: %{"op" => "poll_end", "activity_id" => activity_id}}) do
with %Activity{} = activity <- find_poll_activity(activity_id) do
Notification.create_poll_notifications(activity)
with %Activity{} = activity <- find_poll_activity(activity_id),
{:ok, notifications} <- Notification.create_poll_notifications(activity) do
Notification.stream(notifications)
end
end

4
lib/pleroma/workers/purge_expired_activity.ex
View file

@ -7,7 +7,7 @@ defmodule Pleroma.Workers.PurgeExpiredActivity do
Worker which purges expired activity.
"""
use Oban.Worker, queue: :activity_expiration, max_attempts: 1, unique: [period: :infinity]
use Oban.Worker, queue: :slow, max_attempts: 1, unique: [period: :infinity]
import Ecto.Query
@ -59,7 +59,7 @@ defmodule Pleroma.Workers.PurgeExpiredActivity do
def get_expiration(id) do
from(j in Oban.Job,
where: j.state == "scheduled",
where: j.queue == "activity_expiration",
where: j.queue == "slow",
where: fragment("?->>'activity_id' = ?", j.args, ^id)
)
|> Pleroma.Repo.one()

4
lib/pleroma/workers/purge_expired_filter.ex
View file

@ -7,7 +7,7 @@ defmodule Pleroma.Workers.PurgeExpiredFilter do
Worker which purges expired filters
"""
use Oban.Worker, queue: :filter_expiration, max_attempts: 1, unique: [period: :infinity]
use Oban.Worker, queue: :background, max_attempts: 1, unique: [period: :infinity]
import Ecto.Query
@ -38,7 +38,7 @@ defmodule Pleroma.Workers.PurgeExpiredFilter do
def get_expiration(id) do
from(j in Job,
where: j.state == "scheduled",
where: j.queue == "filter_expiration",
where: j.queue == "background",
where: fragment("?->'filter_id' = ?", j.args, ^id)
)
|> Repo.one()

2
lib/pleroma/workers/purge_expired_token.ex
View file

@ -7,7 +7,7 @@ defmodule Pleroma.Workers.PurgeExpiredToken do
Worker which purges expired OAuth tokens
"""
use Oban.Worker, queue: :token_expiration, max_attempts: 1
use Oban.Worker, queue: :background, max_attempts: 1
@spec enqueue(%{token_id: integer(), valid_until: DateTime.t(), mod: module()}) ::
{:ok, Oban.Job.t()} | {:error, Ecto.Changeset.t()}

2
lib/pleroma/workers/remote_fetcher_worker.ex
View file

@ -5,7 +5,7 @@
defmodule Pleroma.Workers.RemoteFetcherWorker do
alias Pleroma.Object.Fetcher
use Pleroma.Workers.WorkerHelper, queue: "remote_fetcher"
use Pleroma.Workers.WorkerHelper, queue: "background"
@impl Oban.Worker
def perform(%Job{args: %{"op" => "fetch_remote", "id" => id} = args}) do

2
lib/pleroma/workers/rich_media_expiration_worker.ex
View file

@ -6,7 +6,7 @@ defmodule Pleroma.Workers.RichMediaExpirationWorker do
alias Pleroma.Web.RichMedia.Card
use Oban.Worker,
queue: :rich_media_expiration
queue: :background
@impl Oban.Worker
def perform(%Job{args: %{"url" => url} = _args}) do

2
lib/pleroma/workers/scheduled_activity_worker.ex
View file

@ -7,7 +7,7 @@ defmodule Pleroma.Workers.ScheduledActivityWorker do
The worker to post scheduled activity.
"""
use Pleroma.Workers.WorkerHelper, queue: "scheduled_activities"
use Pleroma.Workers.WorkerHelper, queue: "federator_outgoing"
alias Pleroma.Repo
alias Pleroma.ScheduledActivity

4
lib/pleroma/workers/web_pusher_worker.ex
View file

@ -5,6 +5,7 @@
defmodule Pleroma.Workers.WebPusherWorker do
alias Pleroma.Notification
alias Pleroma.Repo
alias Pleroma.Web.Push.Impl
use Pleroma.Workers.WorkerHelper, queue: "web_push"
@ -15,7 +16,8 @@ defmodule Pleroma.Workers.WebPusherWorker do
|> Repo.get(notification_id)
|> Repo.preload([:activity, :user])
Pleroma.Web.Push.Impl.perform(notification)
Impl.build(notification)
|> Enum.each(&Impl.deliver(&1))
end
@impl Oban.Worker