hj/pleroma
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge remote-tracking branch 'origin/develop' into language-detection

Browse source
This commit is contained in:
mkljczk 2025-02-22 14:05:53 +01:00
commit 2b1ef1bbdf
1005 changed files with 13890 additions and 4733 deletions
Download patch file Download diff file Expand all files Collapse all files

3
lib/mix/pleroma.ex
View file

@ -14,7 +14,8 @@ defmodule Mix.Pleroma do
:swoosh,
:timex,
:fast_html,
:oban
:oban,
:logger_backends
]
@cachex_children ["object", "user", "scrubber", "web_resp"]
@doc "Common functions to be reused in mix tasks"

29
lib/mix/tasks/pleroma/config.ex
View file

@ -205,6 +205,35 @@ defmodule Mix.Tasks.Pleroma.Config do
end
end
# Removes any policies that are not a real module
# as they will prevent the server from starting
def run(["fix_mrf_policies"]) do
check_configdb(fn ->
start_pleroma()
group = :pleroma
key = :mrf
%{value: value} =
group
|> ConfigDB.get_by_group_and_key(key)
policies =
Keyword.get(value, :policies, [])
|> Enum.filter(&is_atom(&1))
|> Enum.filter(fn mrf ->
case Code.ensure_compiled(mrf) do
{:module, _} -> true
{:error, _} -> false
end
end)
value = Keyword.put(value, :policies, policies)
ConfigDB.update_or_create(%{group: group, key: key, value: value})
end)
end
@spec migrate_to_db(Path.t() | nil) :: any()
def migrate_to_db(file_path \\ nil) do
with :ok <- Pleroma.Config.DeprecationWarnings.warn() do

177
lib/mix/tasks/pleroma/database.ex
View file

@ -67,43 +67,168 @@ defmodule Mix.Tasks.Pleroma.Database do
OptionParser.parse(
args,
strict: [
vacuum: :boolean
vacuum: :boolean,
keep_threads: :boolean,
keep_non_public: :boolean,
prune_orphaned_activities: :boolean
]
)
start_pleroma()
deadline = Pleroma.Config.get([:instance, :remote_post_retention_days])
time_deadline = NaiveDateTime.utc_now() |> NaiveDateTime.add(-(deadline * 86_400))
Logger.info("Pruning objects older than #{deadline} days")
log_message = "Pruning objects older than #{deadline} days"
time_deadline =
NaiveDateTime.utc_now()
|> NaiveDateTime.add(-(deadline * 86_400))
log_message =
if Keyword.get(options, :keep_non_public) do
log_message <> ", keeping non public posts"
else
log_message
end
from(o in Object,
where:
fragment(
"?->'to' \\? ? OR ?->'cc' \\? ?",
o.data,
^Pleroma.Constants.as_public(),
o.data,
^Pleroma.Constants.as_public()
),
where: o.inserted_at < ^time_deadline,
where:
log_message =
if Keyword.get(options, :keep_threads) do
log_message <> ", keeping threads intact"
else
log_message
end
log_message =
if Keyword.get(options, :prune_orphaned_activities) do
log_message <> ", pruning orphaned activities"
else
log_message
end
log_message =
if Keyword.get(options, :vacuum) do
log_message <>
", doing a full vacuum (you shouldn't do this as a recurring maintanance task)"
else
log_message
end
Logger.info(log_message)
if Keyword.get(options, :keep_threads) do
# We want to delete objects from threads where
# 1. the newest post is still old
# 2. none of the activities is local
# 3. none of the activities is bookmarked
# 4. optionally none of the posts is non-public
deletable_context =
if Keyword.get(options, :keep_non_public) do
Pleroma.Activity
|> join(:left, [a], b in Pleroma.Bookmark, on: a.id == b.activity_id)
|> group_by([a], fragment("? ->> 'context'::text", a.data))
|> having(
[a],
not fragment(
# Posts (checked on Create Activity) is non-public
"bool_or((not(?->'to' \\? ? OR ?->'cc' \\? ?)) and ? ->> 'type' = 'Create')",
a.data,
^Pleroma.Constants.as_public(),
a.data,
^Pleroma.Constants.as_public(),
a.data
)
)
else
Pleroma.Activity
|> join(:left, [a], b in Pleroma.Bookmark, on: a.id == b.activity_id)
|> group_by([a], fragment("? ->> 'context'::text", a.data))
end
|> having([a], max(a.updated_at) < ^time_deadline)
|> having([a], not fragment("bool_or(?)", a.local))
|> having([_, b], fragment("max(?::text) is null", b.id))
|> select([a], fragment("? ->> 'context'::text", a.data))
Pleroma.Object
|> where([o], fragment("? ->> 'context'::text", o.data) in subquery(deletable_context))
else
if Keyword.get(options, :keep_non_public) do
Pleroma.Object
|> where(
[o],
fragment(
"?->'to' \\? ? OR ?->'cc' \\? ?",
o.data,
^Pleroma.Constants.as_public(),
o.data,
^Pleroma.Constants.as_public()
)
)
else
Pleroma.Object
end
|> where([o], o.updated_at < ^time_deadline)
|> where(
[o],
fragment("split_part(?->>'actor', '/', 3) != ?", o.data, ^Pleroma.Web.Endpoint.host())
)
)
end
|> Repo.delete_all(timeout: :infinity)
prune_hashtags_query = """
if !Keyword.get(options, :keep_threads) do
# Without the --keep-threads option, it's possible that bookmarked
# objects have been deleted. We remove the corresponding bookmarks.
"""
delete from public.bookmarks
where id in (
select b.id from public.bookmarks b
left join public.activities a on b.activity_id = a.id
left join public.objects o on a."data" ->> 'object' = o.data ->> 'id'
where o.id is null
)
"""
|> Repo.query([], timeout: :infinity)
end
if Keyword.get(options, :prune_orphaned_activities) do
# Prune activities who link to a single object
"""
delete from public.activities
where id in (
select a.id from public.activities a
left join public.objects o on a.data ->> 'object' = o.data ->> 'id'
left join public.activities a2 on a.data ->> 'object' = a2.data ->> 'id'
left join public.users u on a.data ->> 'object' = u.ap_id
where not a.local
and jsonb_typeof(a."data" -> 'object') = 'string'
and o.id is null
and a2.id is null
and u.id is null
)
"""
|> Repo.query([], timeout: :infinity)
# Prune activities who link to an array of objects
"""
delete from public.activities
where id in (
select a.id from public.activities a
join json_array_elements_text((a."data" -> 'object')::json) as j on jsonb_typeof(a."data" -> 'object') = 'array'
left join public.objects o on j.value = o.data ->> 'id'
left join public.activities a2 on j.value = a2.data ->> 'id'
left join public.users u on j.value = u.ap_id
group by a.id
having max(o.data ->> 'id') is null
and max(a2.data ->> 'id') is null
and max(u.ap_id) is null
)
"""
|> Repo.query([], timeout: :infinity)
end
"""
DELETE FROM hashtags AS ht
WHERE NOT EXISTS (
SELECT 1 FROM hashtags_objects hto
WHERE ht.id = hto.hashtag_id)
"""
Repo.query(prune_hashtags_query)
|> Repo.query()
if Keyword.get(options, :vacuum) do
Maintenance.vacuum("full")
@ -170,10 +295,12 @@ defmodule Mix.Tasks.Pleroma.Database do
|> DateTime.from_naive!("Etc/UTC")
|> Timex.shift(days: days)
Pleroma.Workers.PurgeExpiredActivity.enqueue(%{
activity_id: activity.id,
expires_at: expires_at
})
Pleroma.Workers.PurgeExpiredActivity.enqueue(
%{
activity_id: activity.id
},
scheduled_at: expires_at
)
end)
end)
|> Stream.run()
@ -226,7 +353,7 @@ defmodule Mix.Tasks.Pleroma.Database do
)
end
shell_info('Done.')
shell_info(~c"Done.")
end
end

83
lib/mix/tasks/pleroma/search/indexer.ex Normal file
View file

@ -0,0 +1,83 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Mix.Tasks.Pleroma.Search.Indexer do
import Mix.Pleroma
import Ecto.Query
alias Pleroma.Workers.SearchIndexingWorker
def run(["create_index"]) do
start_pleroma()
with :ok <- Pleroma.Config.get([Pleroma.Search, :module]).create_index() do
IO.puts("Index created")
else
e -> IO.puts("Could not create index: #{inspect(e)}")
end
end
def run(["drop_index"]) do
start_pleroma()
with :ok <- Pleroma.Config.get([Pleroma.Search, :module]).drop_index() do
IO.puts("Index dropped")
else
e -> IO.puts("Could not drop index: #{inspect(e)}")
end
end
def run(["index" | options]) do
{options, [], []} =
OptionParser.parse(
options,
strict: [
chunk: :integer,
limit: :integer,
step: :integer
]
)
start_pleroma()
chunk_size = Keyword.get(options, :chunk, 100)
limit = Keyword.get(options, :limit, 100_000)
per_step = Keyword.get(options, :step, 1000)
chunks = max(div(limit, per_step), 1)
1..chunks
|> Enum.each(fn step ->
q =
from(a in Pleroma.Activity,
limit: ^per_step,
offset: ^per_step * (^step - 1),
select: [:id],
order_by: [desc: :id]
)
{:ok, ids} =
Pleroma.Repo.transaction(fn ->
Pleroma.Repo.stream(q, timeout: :infinity)
|> Enum.map(fn a ->
a.id
end)
end)
IO.puts("Got #{length(ids)} activities, adding to indexer")
ids
|> Enum.chunk_every(chunk_size)
|> Enum.each(fn chunk ->
IO.puts("Adding #{length(chunk)} activities to indexing queue")
chunk
|> Enum.map(fn id ->
SearchIndexingWorker.new(%{"op" => "add_to_index", "activity" => id})
end)
|> Oban.insert_all()
end)
end)
end
end

6
lib/mix/tasks/pleroma/search/meilisearch.ex
View file

@ -9,7 +9,7 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do
import Ecto.Query
import Pleroma.Search.Meilisearch,
only: [meili_post: 2, meili_put: 2, meili_get: 1, meili_delete: 1]
only: [meili_put: 2, meili_get: 1, meili_delete: 1]
def run(["index"]) do
start_pleroma()
@ -28,7 +28,7 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do
end
{:ok, _} =
meili_post(
meili_put(
"/indexes/objects/settings/ranking-rules",
[
"published:desc",
@ -42,7 +42,7 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do
)
{:ok, _} =
meili_post(
meili_put(
"/indexes/objects/settings/searchable-attributes",
[
"content"

25
lib/mix/tasks/pleroma/test_runner.ex Normal file
View file

@ -0,0 +1,25 @@
defmodule Mix.Tasks.Pleroma.TestRunner do
@shortdoc "Retries tests once if they fail"
use Mix.Task
def run(args \\ []) do
case System.cmd("mix", ["test"] ++ args, into: IO.stream(:stdio, :line)) do
{_, 0} ->
:ok
_ ->
retry(args)
end
end
def retry(args) do
case System.cmd("mix", ["test", "--failed"] ++ args, into: IO.stream(:stdio, :line)) do
{_, 0} ->
:ok
_ ->
exit(1)
end
end
end

15
lib/pleroma/application.ex
View file

@ -51,7 +51,11 @@ defmodule Pleroma.Application do
Pleroma.HTML.compile_scrubbers()
Pleroma.Config.Oban.warn()
Config.DeprecationWarnings.warn()
Pleroma.Web.Plugs.HTTPSecurityPlug.warn_if_disabled()
if Config.get([Pleroma.Web.Plugs.HTTPSecurityPlug, :enable], true) do
Pleroma.Web.Plugs.HTTPSecurityPlug.warn_if_disabled()
end
Pleroma.ApplicationRequirements.verify!()
load_custom_modules()
Pleroma.Docs.JSON.compile()
@ -90,6 +94,7 @@ defmodule Pleroma.Application do
children =
[
Pleroma.PromEx,
Pleroma.LDAP,
Pleroma.Repo,
Config.TransferTask,
Pleroma.Emoji,
@ -109,7 +114,8 @@ defmodule Pleroma.Application do
streamer_registry() ++
background_migrators() ++
shout_child(shout_enabled?()) ++
[Pleroma.Gopher.Server]
[Pleroma.Gopher.Server] ++
[Pleroma.Search.Healthcheck]
# See http://elixir-lang.org/docs/stable/elixir/Supervisor.html
# for other strategies and supported options
@ -162,7 +168,8 @@ defmodule Pleroma.Application do
expiration: chat_message_id_idempotency_key_expiration(),
limit: 500_000
),
build_cachex("rel_me", limit: 2500)
build_cachex("rel_me", limit: 2500),
build_cachex("host_meta", default_ttl: :timer.minutes(120), limit: 5000)
]
end
@ -285,8 +292,6 @@ defmodule Pleroma.Application do
config = Config.get(ConcurrentLimiter, [])
[
Pleroma.Web.RichMedia.Helpers,
Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy,
Pleroma.Search
]
|> Enum.each(fn module ->

7
lib/pleroma/application_requirements.ex
View file

@ -261,10 +261,9 @@ defmodule Pleroma.ApplicationRequirements do
missing_mrfs =
Enum.reduce(mrfs, [], fn x, acc ->
if Code.ensure_compiled(x) do
acc
else
acc ++ [x]
case Code.ensure_compiled(x) do
{:module, _} -> acc
{:error, _} -> acc ++ [x]
end
end)

46
lib/pleroma/config/transfer_task.ex
View file

@ -1,5 +1,5 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# Copyright © 2017-2023 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Config.TransferTask do
@ -22,7 +22,8 @@ defmodule Pleroma.Config.TransferTask do
{:pleroma, :markup},
{:pleroma, :streamer},
{:pleroma, :pools},
{:pleroma, :connections_pool}
{:pleroma, :connections_pool},
{:pleroma, :ldap}
]
defp reboot_time_subkeys,
@ -44,14 +45,9 @@ defmodule Pleroma.Config.TransferTask do
with {_, true} <- {:configurable, Config.get(:configurable_from_database)} do
# We need to restart applications for loaded settings take effect
{logger, other} =
settings =
(Repo.all(ConfigDB) ++ deleted_settings)
|> Enum.map(&merge_with_default/1)
|> Enum.split_with(fn {group, _, _, _} -> group in [:logger] end)
logger
|> Enum.sort()
|> Enum.each(&configure/1)
started_applications = Application.started_applications()
@ -64,7 +60,7 @@ defmodule Pleroma.Config.TransferTask do
[:pleroma | reject]
end
other
settings
|> Enum.map(&update/1)
|> Enum.uniq()
|> Enum.reject(&(&1 in reject))
@ -102,38 +98,6 @@ defmodule Pleroma.Config.TransferTask do
{group, key, value, merged}
end
# change logger configuration in runtime, without restart
defp configure({_, :backends, _, merged}) do
# removing current backends
Enum.each(Application.get_env(:logger, :backends), &Logger.remove_backend/1)
Enum.each(merged, &Logger.add_backend/1)
:ok = update_env(:logger, :backends, merged)
end
defp configure({_, key, _, merged}) when key in [:console, :ex_syslogger] do
merged =
if key == :console do
put_in(merged[:format], merged[:format] <> "\n")
else
merged
end
backend =
if key == :ex_syslogger,
do: {ExSyslogger, :ex_syslogger},
else: key
Logger.configure_backend(backend, merged)
:ok = update_env(:logger, key, merged)
end
defp configure({_, key, _, merged}) do
Logger.configure([{key, merged}])
:ok = update_env(:logger, key, merged)
end
defp update({group, key, value, merged}) do
try do
:ok = update_env(group, key, merged)

12
lib/pleroma/config_db.ex
View file

@ -165,8 +165,7 @@ defmodule Pleroma.ConfigDB do
{:pleroma, :ecto_repos},
{:mime, :types},
{:cors_plug, [:max_age, :methods, :expose, :headers]},
{:swarm, :node_blacklist},
{:logger, :backends}
{:swarm, :node_blacklist}
]
Enum.any?(full_key_update, fn
@ -385,7 +384,12 @@ defmodule Pleroma.ConfigDB do
@spec module_name?(String.t()) :: boolean()
def module_name?(string) do
Regex.match?(~r/^(Pleroma|Phoenix|Tesla|Ueberauth|Swoosh)\./, string) or
string in ["Oban", "Ueberauth", "ExSyslogger", "ConcurrentLimiter"]
if String.contains?(string, ".") do
[name | _] = String.split(string, ".", parts: 2)
name in ~w[Pleroma Phoenix Tesla Ueberauth Swoosh Logger LoggerBackends]
else
string in ~w[Oban Ueberauth ExSyslogger ConcurrentLimiter]
end
end
end

37
lib/pleroma/constants.ex
View file

@ -37,6 +37,7 @@ defmodule Pleroma.Constants do
"updated",
"emoji",
"content",
"contentMap",
"summary",
"sensitive",
"attachment",
@ -87,6 +88,42 @@ defmodule Pleroma.Constants do
]
)
const(activity_types,
do: [
"Block",
"Create",
"Update",
"Delete",
"Follow",
"Accept",
"Reject",
"Add",
"Remove",
"Like",
"Announce",
"Undo",
"Flag",
"EmojiReact",
"Listen"
]
)
const(allowed_activity_types_from_strangers,
do: [
"Block",
"Create",
"Flag",
"Follow",
"Like",
"EmojiReact",
"Announce"
]
)
const(object_types,
do: ~w[Event Question Answer Audio Video Image Article Note Page ChatMessage]
)
# basic regex, just there to weed out potential mistakes
# https://datatracker.ietf.org/doc/html/rfc2045#section-5.1
const(mime_regex,

2
lib/pleroma/conversation/participation.ex
View file

@ -12,6 +12,8 @@ defmodule Pleroma.Conversation.Participation do
import Ecto.Changeset
import Ecto.Query
@type t :: %__MODULE__{}
schema "conversation_participations" do
belongs_to(:user, User, type: FlakeId.Ecto.CompatType)
belongs_to(:conversation, Conversation)

8
lib/pleroma/ecto_enums.ex
View file

@ -27,11 +27,3 @@ defenum(Pleroma.DataMigration.State,
failed: 4,
manual: 5
)
defenum(Pleroma.User.Backup.State,
pending: 1,
running: 2,
complete: 3,
failed: 4,
invalid: 5
)

2
lib/pleroma/ecto_type/activity_pub/object_validators/language_code.ex
View file

@ -21,7 +21,7 @@ defmodule Pleroma.EctoType.ActivityPub.ObjectValidators.LanguageCode do
def load(data), do: {:ok, data}
def good_locale_code?(code) when is_binary(code), do: code =~ ~r<^[a-zA-Z0-9\-]+$>
def good_locale_code?(code) when is_binary(code), do: code =~ ~r<^[a-zA-Z0-9\-]+\z$>
def good_locale_code?(_code), do: false
end

3
lib/pleroma/emails/mailer.ex
View file

@ -25,7 +25,8 @@ defmodule Pleroma.Emails.Mailer do
|> :erlang.term_to_binary()
|> Base.encode64()
MailerWorker.enqueue("email", %{"encoded_email" => encoded_email, "config" => config})
MailerWorker.new(%{"op" => "email", "encoded_email" => encoded_email, "config" => config})
|> Oban.insert()
end
@doc "callback to perform send email from queue"

35
lib/pleroma/emails/user_email.ex
View file

@ -345,37 +345,22 @@ defmodule Pleroma.Emails.UserEmail do
Router.Helpers.subscription_url(Endpoint, :unsubscribe, token)
end
def backup_is_ready_email(backup, admin_user_id \\ nil) do
def backup_is_ready_email(backup) do
%{user: user} = Pleroma.Repo.preload(backup, :user)
Gettext.with_locale_or_default user.language do
download_url = Pleroma.Web.PleromaAPI.BackupView.download_url(backup)
html_body =
if is_nil(admin_user_id) do
Gettext.dpgettext(
"static_pages",
"account archive email body - self-requested",
"""
<p>You requested a full backup of your Pleroma account. It's ready for download:</p>
<p><a href="%{download_url}">%{download_url}</a></p>
""",
download_url: download_url
)
else
admin = Pleroma.Repo.get(User, admin_user_id)
Gettext.dpgettext(
"static_pages",
"account archive email body - admin requested",
"""
<p>Admin @%{admin_nickname} requested a full backup of your Pleroma account. It's ready for download:</p>
<p><a href="%{download_url}">%{download_url}</a></p>
""",
admin_nickname: admin.nickname,
download_url: download_url
)
end
Gettext.dpgettext(
"static_pages",
"account archive email body",
"""
<p>A full backup of your Pleroma account was requested. It's ready for download:</p>
<p><a href="%{download_url}">%{download_url}</a></p>
""",
download_url: download_url
)
new()
|> to(recipient(user))

6
lib/pleroma/emoji/pack.ex
View file

@ -416,10 +416,10 @@ defmodule Pleroma.Emoji.Pack do
end
defp create_archive_and_cache(pack, hash) do
files = ['pack.json' | Enum.map(pack.files, fn {_, file} -> to_charlist(file) end)]
files = [~c"pack.json" | Enum.map(pack.files, fn {_, file} -> to_charlist(file) end)]
{:ok, {_, result}} =
:zip.zip('#{pack.name}.zip', files, [:memory, cwd: to_charlist(pack.path)])
:zip.zip(~c"#{pack.name}.zip", files, [:memory, cwd: to_charlist(pack.path)])
ttl_per_file = Pleroma.Config.get!([:emoji, :shared_pack_cache_seconds_per_file])
overall_ttl = :timer.seconds(ttl_per_file * Enum.count(files))
@ -586,7 +586,7 @@ defmodule Pleroma.Emoji.Pack do
with :ok <- File.mkdir_p!(local_pack.path) do
files = Enum.map(remote_pack["files"], fn {_, path} -> to_charlist(path) end)
# Fallback cannot contain a pack.json file
files = if pack_info[:fallback], do: files, else: ['pack.json' | files]
files = if pack_info[:fallback], do: files, else: [~c"pack.json" | files]
:zip.unzip(archive, cwd: to_charlist(local_pack.path), file_list: files)
end

11
lib/pleroma/filter.ex
View file

@ -133,10 +133,13 @@ defmodule Pleroma.Filter do
defp maybe_add_expires_at(changeset, _), do: changeset
defp maybe_add_expiration_job(%{expires_at: %NaiveDateTime{} = expires_at} = filter) do
Pleroma.Workers.PurgeExpiredFilter.enqueue(%{
filter_id: filter.id,
expires_at: DateTime.from_naive!(expires_at, "Etc/UTC")
})
Pleroma.Workers.PurgeExpiredFilter.new(
%{
filter_id: filter.id
},
scheduled_at: DateTime.from_naive!(expires_at, "Etc/UTC")
)
|> Oban.insert()
end
defp maybe_add_expiration_job(_), do: {:ok, nil}

4
lib/pleroma/following_relationship.ex
View file

@ -199,8 +199,8 @@ defmodule Pleroma.FollowingRelationship do
|> preload([:follower])
|> Repo.all()
|> Enum.map(fn following_relationship ->
Pleroma.Web.CommonAPI.follow(following_relationship.follower, target)
Pleroma.Web.CommonAPI.unfollow(following_relationship.follower, origin)
Pleroma.Web.CommonAPI.follow(target, following_relationship.follower)
Pleroma.Web.CommonAPI.unfollow(origin, following_relationship.follower)
end)
|> case do
[] ->

11
lib/pleroma/frontend.ex
View file

@ -43,10 +43,6 @@ defmodule Pleroma.Frontend do
{:download_or_unzip, _} ->
Logger.info("Could not download or unzip the frontend")
{:error, "Could not download or unzip the frontend"}
_e ->
Logger.info("Could not install the frontend")
{:error, "Could not install the frontend"}
end
end
@ -78,11 +74,14 @@ defmodule Pleroma.Frontend do
new_file_path = Path.join(dest, path)
new_file_path
path
|> Path.dirname()
|> then(&Path.join(dest, &1))
|> File.mkdir_p!()
File.write!(new_file_path, data)
if not File.dir?(new_file_path) do
File.write!(new_file_path, data)
end
end)
end
end

2
lib/pleroma/gun/connection_pool/reclaimer.ex
View file

@ -9,7 +9,7 @@ defmodule Pleroma.Gun.ConnectionPool.Reclaimer do
def start_monitor do
pid =
case GenServer.start_link(__MODULE__, [], name: {:via, Registry, {registry(), "reclaimer"}}) do
case GenServer.start(__MODULE__, [], name: {:via, Registry, {registry(), "reclaimer"}}) do
{:ok, pid} ->
pid

32
lib/pleroma/gun/connection_pool/worker_supervisor.ex
View file

@ -5,6 +5,9 @@
defmodule Pleroma.Gun.ConnectionPool.WorkerSupervisor do
@moduledoc "Supervisor for pool workers. Does not do anything except enforce max connection limit"
alias Pleroma.Config
alias Pleroma.Gun.ConnectionPool.Worker
use DynamicSupervisor
def start_link(opts) do
@ -14,21 +17,28 @@ defmodule Pleroma.Gun.ConnectionPool.WorkerSupervisor do
def init(_opts) do
DynamicSupervisor.init(
strategy: :one_for_one,
max_children: Pleroma.Config.get([:connections_pool, :max_connections])
max_children: Config.get([:connections_pool, :max_connections])
)
end
def start_worker(opts, last_attempt \\ false) do
case DynamicSupervisor.start_child(__MODULE__, {Pleroma.Gun.ConnectionPool.Worker, opts}) do
{:error, :max_children} ->
funs = [fn -> last_attempt end, fn -> match?(:error, free_pool()) end]
def start_worker(opts, last_attempt \\ false)
if Enum.any?(funs, fn fun -> fun.() end) do
:telemetry.execute([:pleroma, :connection_pool, :provision_failure], %{opts: opts})
{:error, :pool_full}
else
start_worker(opts, true)
end
def start_worker(opts, true) do
case DynamicSupervisor.start_child(__MODULE__, {Worker, opts}) do
{:error, :max_children} ->
:telemetry.execute([:pleroma, :connection_pool, :provision_failure], %{opts: opts})
{:error, :pool_full}
res ->
res
end
end
def start_worker(opts, false) do
case DynamicSupervisor.start_child(__MODULE__, {Worker, opts}) do
{:error, :max_children} ->
free_pool()
start_worker(opts, true)
res ->
res

27
lib/pleroma/hashtag.ex
View file

@ -12,6 +12,7 @@ defmodule Pleroma.Hashtag do
alias Pleroma.Hashtag
alias Pleroma.Object
alias Pleroma.Repo
alias Pleroma.User.HashtagFollow
schema "hashtags" do
field(:name, :string)
@ -27,6 +28,14 @@ defmodule Pleroma.Hashtag do
|> String.trim()
end
def get_by_id(id) do
Repo.get(Hashtag, id)
end
def get_by_name(name) do
Repo.get_by(Hashtag, name: normalize_name(name))
end
def get_or_create_by_name(name) do
changeset = changeset(%Hashtag{}, %{name: name})
@ -103,4 +112,22 @@ defmodule Pleroma.Hashtag do
{:ok, deleted_count}
end
end
def get_followers(%Hashtag{id: hashtag_id}) do
from(hf in HashtagFollow)
|> where([hf], hf.hashtag_id == ^hashtag_id)
|> join(:inner, [hf], u in assoc(hf, :user))
|> select([hf, u], u.id)
|> Repo.all()
end
def get_recipients_for_activity(%Pleroma.Activity{object: %{hashtags: tags}})
when is_list(tags) do
tags
|> Enum.map(&get_followers/1)
|> List.flatten()
|> Enum.uniq()
end
def get_recipients_for_activity(_activity), do: []
end

11
lib/pleroma/helpers/inet_helper.ex
View file

@ -16,4 +16,15 @@ defmodule Pleroma.Helpers.InetHelper do
def parse_address(ip) do
:inet.parse_address(ip)
end
def parse_cidr(proxy) when is_binary(proxy) do
proxy =
cond do
"/" in String.codepoints(proxy) -> proxy
InetCidr.v4?(InetCidr.parse_address!(proxy)) -> proxy <> "/32"
InetCidr.v6?(InetCidr.parse_address!(proxy)) -> proxy <> "/128"
end
InetCidr.parse_cidr!(proxy, true)
end
end

16
lib/pleroma/helpers/media_helper.ex
View file

@ -25,7 +25,7 @@ defmodule Pleroma.Helpers.MediaHelper do
end
def image_resize(url, options) do
with {:ok, env} <- HTTP.get(url, [], pool: :media),
with {:ok, env} <- HTTP.get(url, [], http_client_opts()),
{:ok, resized} <-
Operation.thumbnail_buffer(env.body, options.max_width,
height: options.max_height,
@ -45,8 +45,8 @@ defmodule Pleroma.Helpers.MediaHelper do
@spec video_framegrab(String.t()) :: {:ok, binary()} | {:error, any()}
def video_framegrab(url) do
with executable when is_binary(executable) <- System.find_executable("ffmpeg"),
false <- @cachex.exists?(:failed_media_helper_cache, url),
{:ok, env} <- HTTP.get(url, [], pool: :media),
{:ok, false} <- @cachex.exists?(:failed_media_helper_cache, url),
{:ok, env} <- HTTP.get(url, [], http_client_opts()),
{:ok, pid} <- StringIO.open(env.body) do
body_stream = IO.binstream(pid, 1)
@ -71,17 +71,19 @@ defmodule Pleroma.Helpers.MediaHelper do
end)
case Task.yield(task, 5_000) do
nil ->
{:ok, result} ->
{:ok, result}
_ ->
Task.shutdown(task)
@cachex.put(:failed_media_helper_cache, url, nil)
{:error, {:ffmpeg, :timeout}}
result ->
{:ok, result}
end
else
nil -> {:error, {:ffmpeg, :command_not_found}}
{:error, _} = error -> error
end
end
defp http_client_opts, do: Pleroma.Config.get([:media_proxy, :proxy_opts, :http], pool: :media)
end

21
lib/pleroma/http.ex
View file

@ -37,7 +37,7 @@ defmodule Pleroma.HTTP do
See `Pleroma.HTTP.request/5`
"""
@spec post(Request.url(), String.t(), Request.headers(), keyword()) ::
@spec post(Request.url(), Tesla.Env.body(), Request.headers(), keyword()) ::
{:ok, Env.t()} | {:error, any()}
def post(url, body, headers \\ [], options \\ []),
do: request(:post, url, body, headers, options)
@ -56,7 +56,7 @@ defmodule Pleroma.HTTP do
`{:ok, %Tesla.Env{}}` or `{:error, error}`
"""
@spec request(method(), Request.url(), String.t(), Request.headers(), keyword()) ::
@spec request(method(), Request.url(), Tesla.Env.body(), Request.headers(), keyword()) ::
{:ok, Env.t()} | {:error, any()}
def request(method, url, body, headers, options) when is_binary(url) do
uri = URI.parse(url)
@ -68,7 +68,9 @@ defmodule Pleroma.HTTP do
adapter = Application.get_env(:tesla, :adapter)
client = Tesla.client(adapter_middlewares(adapter), adapter)
extra_middleware = options[:tesla_middleware] || []
client = Tesla.client(adapter_middlewares(adapter, extra_middleware), adapter)
maybe_limit(
fn ->
@ -102,20 +104,21 @@ defmodule Pleroma.HTTP do
fun.()
end
defp adapter_middlewares(Tesla.Adapter.Gun) do
[Tesla.Middleware.FollowRedirects, Pleroma.Tesla.Middleware.ConnectionPool]
defp adapter_middlewares(Tesla.Adapter.Gun, extra_middleware) do
[Tesla.Middleware.FollowRedirects, Pleroma.Tesla.Middleware.ConnectionPool] ++
extra_middleware
end
defp adapter_middlewares({Tesla.Adapter.Finch, _}) do
[Tesla.Middleware.FollowRedirects]
defp adapter_middlewares({Tesla.Adapter.Finch, _}, extra_middleware) do
[Tesla.Middleware.FollowRedirects] ++ extra_middleware
end
defp adapter_middlewares(_) do
defp adapter_middlewares(_, extra_middleware) do
if Pleroma.Config.get(:env) == :test do
# Emulate redirects in test env, which are handled by adapters in other environments
[Tesla.Middleware.FollowRedirects]
else
[]
extra_middleware
end
end
end

10
lib/pleroma/http/adapter_helper.ex
View file

@ -52,6 +52,7 @@ defmodule Pleroma.HTTP.AdapterHelper do
case adapter() do
Tesla.Adapter.Gun -> AdapterHelper.Gun
Tesla.Adapter.Hackney -> AdapterHelper.Hackney
{Tesla.Adapter.Finch, _} -> AdapterHelper.Finch
_ -> AdapterHelper.Default
end
end
@ -118,4 +119,13 @@ defmodule Pleroma.HTTP.AdapterHelper do
host_charlist
end
end
@spec can_stream? :: bool()
def can_stream? do
case Application.get_env(:tesla, :adapter) do
Tesla.Adapter.Gun -> true
{Tesla.Adapter.Finch, _} -> true
_ -> false
end
end
end

33
lib/pleroma/http/adapter_helper/finch.ex Normal file
View file

@ -0,0 +1,33 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.HTTP.AdapterHelper.Finch do
@behaviour Pleroma.HTTP.AdapterHelper
alias Pleroma.Config
alias Pleroma.HTTP.AdapterHelper
@spec options(keyword(), URI.t()) :: keyword()
def options(incoming_opts \\ [], %URI{} = _uri) do
proxy =
[:http, :proxy_url]
|> Config.get()
|> AdapterHelper.format_proxy()
config_opts = Config.get([:http, :adapter], [])
config_opts
|> Keyword.merge(incoming_opts)
|> AdapterHelper.maybe_add_proxy(proxy)
|> maybe_stream()
end
# Finch uses [response: :stream]
defp maybe_stream(opts) do
case Keyword.pop(opts, :stream, nil) do
{true, opts} -> Keyword.put(opts, :response, :stream)
{_, opts} -> opts
end
end
end

11
lib/pleroma/http/adapter_helper/gun.ex
View file

@ -15,7 +15,7 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do
retry_timeout: 1_000
]
@type pool() :: :federation | :upload | :media | :default
@type pool() :: :federation | :upload | :media | :rich_media | :default
@spec options(keyword(), URI.t()) :: keyword()
def options(incoming_opts \\ [], %URI{} = uri) do
@ -32,6 +32,7 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do
|> AdapterHelper.maybe_add_proxy(proxy)
|> Keyword.merge(incoming_opts)
|> put_timeout()
|> maybe_stream()
end
defp add_scheme_opts(opts, %{scheme: "http"}), do: opts
@ -47,6 +48,14 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do
Keyword.put(opts, :timeout, recv_timeout)
end
# Gun uses [body_as: :stream]
defp maybe_stream(opts) do
case Keyword.pop(opts, :stream, nil) do
{true, opts} -> Keyword.put(opts, :body_as, :stream)
{_, opts} -> opts
end
end
@spec pool_timeout(pool()) :: non_neg_integer()
def pool_timeout(pool) do
default = Config.get([:pools, :default, :recv_timeout], 5_000)

4
lib/pleroma/http_signatures_api.ex Normal file
View file

@ -0,0 +1,4 @@
defmodule Pleroma.HTTPSignaturesAPI do
@callback validate_conn(conn :: Plug.Conn.t()) :: boolean
@callback signature_for_conn(conn :: Plug.Conn.t()) :: map
end

5
lib/pleroma/instances/instance.ex
View file

@ -10,7 +10,7 @@ defmodule Pleroma.Instances.Instance do
alias Pleroma.Maps
alias Pleroma.Repo
alias Pleroma.User
alias Pleroma.Workers.BackgroundWorker
alias Pleroma.Workers.DeleteWorker
use Ecto.Schema
@ -297,7 +297,8 @@ defmodule Pleroma.Instances.Instance do
all of those users' activities and notifications.
"""
def delete_users_and_activities(host) when is_binary(host) do
BackgroundWorker.enqueue("delete_instance", %{"host" => host})
DeleteWorker.new(%{"op" => "delete_instance", "host" => host})
|> Oban.insert()
end
def perform(:delete_instance, host) when is_binary(host) do

271
lib/pleroma/ldap.ex Normal file
View file

@ -0,0 +1,271 @@
defmodule Pleroma.LDAP do
use GenServer
require Logger
alias Pleroma.Config
alias Pleroma.User
import Pleroma.Web.Auth.Helpers, only: [fetch_user: 1]
@connection_timeout 2_000
@search_timeout 2_000
def start_link(_) do
GenServer.start_link(__MODULE__, [], name: __MODULE__)
end
def bind_user(name, password) do
GenServer.call(__MODULE__, {:bind_user, name, password})
end
def change_password(name, password, new_password) do
GenServer.call(__MODULE__, {:change_password, name, password, new_password})
end
@impl true
def init(state) do
case {Config.get(Pleroma.Web.Auth.Authenticator), Config.get([:ldap, :enabled])} do
{Pleroma.Web.Auth.LDAPAuthenticator, true} ->
{:ok, state, {:continue, :connect}}
{Pleroma.Web.Auth.LDAPAuthenticator, false} ->
Logger.error(
"LDAP Authenticator enabled but :pleroma, :ldap is not enabled. Auth will not work."
)
{:ok, state}
{_, true} ->
Logger.warning(
":pleroma, :ldap is enabled but Pleroma.Web.Authenticator is not set to the LDAPAuthenticator. LDAP will not be used."
)
{:ok, state}
_ ->
{:ok, state}
end
end
@impl true
def handle_continue(:connect, _state), do: do_handle_connect()
@impl true
def handle_info(:connect, _state), do: do_handle_connect()
def handle_info({:bind_after_reconnect, name, password, from}, state) do
result = do_bind_user(state[:handle], name, password)
GenServer.reply(from, result)
{:noreply, state}
end
@impl true
def handle_call({:bind_user, name, password}, from, state) do
case do_bind_user(state[:handle], name, password) do
:needs_reconnect ->
Process.send(self(), {:bind_after_reconnect, name, password, from}, [])
{:noreply, state, {:continue, :connect}}
result ->
{:reply, result, state, :hibernate}
end
end
def handle_call({:change_password, name, password, new_password}, _from, state) do
result = change_password(state[:handle], name, password, new_password)
{:reply, result, state, :hibernate}
end
@impl true
def terminate(_, state) do
handle = Keyword.get(state, :handle)
if not is_nil(handle) do
:eldap.close(handle)
end
:ok
end
defp do_handle_connect do
state =
case connect() do
{:ok, handle} ->
:eldap.controlling_process(handle, self())
Process.link(handle)
[handle: handle]
_ ->
Logger.error("Failed to connect to LDAP. Retrying in 5000ms")
Process.send_after(self(), :connect, 5_000)
[]
end
{:noreply, state}
end
defp connect do
ldap = Config.get(:ldap, [])
host = Keyword.get(ldap, :host, "localhost")
port = Keyword.get(ldap, :port, 389)
ssl = Keyword.get(ldap, :ssl, false)
tls = Keyword.get(ldap, :tls, false)
cacertfile = Keyword.get(ldap, :cacertfile) || CAStore.file_path()
if ssl, do: Application.ensure_all_started(:ssl)
default_secure_opts = [
verify: :verify_peer,
cacerts: decode_certfile(cacertfile),
customize_hostname_check: [
fqdn_fun: fn _ -> to_charlist(host) end
]
]
sslopts = Keyword.merge(default_secure_opts, Keyword.get(ldap, :sslopts, []))
tlsopts = Keyword.merge(default_secure_opts, Keyword.get(ldap, :tlsopts, []))
default_options = [{:port, port}, {:ssl, ssl}, {:timeout, @connection_timeout}]
# :sslopts can only be included in :eldap.open/2 when {ssl: true}
# or the connection will fail
options =
if ssl do
default_options ++ [{:sslopts, sslopts}]
else
default_options
end
case :eldap.open([to_charlist(host)], options) do
{:ok, handle} ->
try do
cond do
tls ->
case :eldap.start_tls(
handle,
tlsopts,
@connection_timeout
) do
:ok ->
{:ok, handle}
error ->
Logger.error("Could not start TLS: #{inspect(error)}")
:eldap.close(handle)
end
true ->
{:ok, handle}
end
after
:ok
end
{:error, error} ->
Logger.error("Could not open LDAP connection: #{inspect(error)}")
{:error, {:ldap_connection_error, error}}
end
end
defp do_bind_user(handle, name, password) do
dn = make_dn(name)
case :eldap.simple_bind(handle, dn, password) do
:ok ->
case fetch_user(name) do
%User{} = user ->
user
_ ->
register_user(handle, ldap_base(), ldap_uid(), name)
end
# eldap does not inform us of socket closure
# until it is used
{:error, {:gen_tcp_error, :closed}} ->
:eldap.close(handle)
:needs_reconnect
{:error, error} = e ->
Logger.error("Could not bind LDAP user #{name}: #{inspect(error)}")
e
end
end
defp register_user(handle, base, uid, name) do
case :eldap.search(handle, [
{:base, to_charlist(base)},
{:filter, :eldap.equalityMatch(to_charlist(uid), to_charlist(name))},
{:scope, :eldap.wholeSubtree()},
{:timeout, @search_timeout}
]) do
# The :eldap_search_result record structure changed in OTP 24.3 and added a controls field
# https://github.com/erlang/otp/pull/5538
{:ok, {:eldap_search_result, [{:eldap_entry, _object, attributes}], _referrals}} ->
try_register(name, attributes)
{:ok, {:eldap_search_result, [{:eldap_entry, _object, attributes}], _referrals, _controls}} ->
try_register(name, attributes)
error ->
Logger.error("Couldn't register user because LDAP search failed: #{inspect(error)}")
{:error, {:ldap_search_error, error}}
end
end
defp try_register(name, attributes) do
mail_attribute = Config.get([:ldap, :mail])
params = %{
name: name,
nickname: name,
password: nil
}
params =
case List.keyfind(attributes, to_charlist(mail_attribute), 0) do
{_, [mail]} -> Map.put_new(params, :email, :erlang.list_to_binary(mail))
_ -> params
end
changeset = User.register_changeset_ldap(%User{}, params)
case User.register(changeset) do
{:ok, user} -> user
error -> error
end
end
defp change_password(handle, name, password, new_password) do
dn = make_dn(name)
with :ok <- :eldap.simple_bind(handle, dn, password) do
:eldap.modify_password(handle, dn, to_charlist(new_password), to_charlist(password))
end
end
defp decode_certfile(file) do
with {:ok, data} <- File.read(file) do
data
|> :public_key.pem_decode()
|> Enum.map(fn {_, b, _} -> b end)
else
_ ->
Logger.error("Unable to read certfile: #{file}")
[]
end
end
defp ldap_uid, do: to_charlist(Config.get([:ldap, :uid], "cn"))
defp ldap_base, do: to_charlist(Config.get([:ldap, :base]))
defp make_dn(name) do
uid = ldap_uid()
base = ldap_base()
~c"#{uid}=#{name},#{base}"
end
end

4
lib/pleroma/maps.ex
View file

@ -20,15 +20,13 @@ defmodule Pleroma.Maps do
end
def filter_empty_values(data) do
# TODO: Change to Map.filter in Elixir 1.13+
data
|> Enum.filter(fn
|> Map.filter(fn
{_k, nil} -> false
{_k, ""} -> false
{_k, []} -> false
{_k, %{} = v} -> Map.keys(v) != []
{_k, _v} -> true
end)
|> Map.new()
end
end

13
lib/pleroma/mfa/token.ex
View file

@ -52,11 +52,14 @@ defmodule Pleroma.MFA.Token do
@spec create(User.t(), Authorization.t() | nil) :: {:ok, t()} | {:error, Ecto.Changeset.t()}
def create(user, authorization \\ nil) do
with {:ok, token} <- do_create(user, authorization) do
Pleroma.Workers.PurgeExpiredToken.enqueue(%{
token_id: token.id,
valid_until: DateTime.from_naive!(token.valid_until, "Etc/UTC"),
mod: __MODULE__
})
Pleroma.Workers.PurgeExpiredToken.new(
%{
token_id: token.id,
mod: __MODULE__
},
scheduled_at: DateTime.from_naive!(token.valid_until, "Etc/UTC")
)
|> Oban.insert()
{:ok, token}
end

59
lib/pleroma/notification.ex
View file

@ -73,6 +73,7 @@ defmodule Pleroma.Notification do
pleroma:report
reblog
poll
status
}
def changeset(%Notification{} = notification, attrs) do
@ -280,15 +281,10 @@ defmodule Pleroma.Notification do
select: n.id
)
{:ok, %{ids: {_, notification_ids}}} =
Multi.new()
|> Multi.update_all(:ids, query, set: [seen: true, updated_at: NaiveDateTime.utc_now()])
|> Marker.multi_set_last_read_id(user, "notifications")
|> Repo.transaction()
for_user_query(user)
|> where([n], n.id in ^notification_ids)
|> Repo.all()
Multi.new()
|> Multi.update_all(:ids, query, set: [seen: true, updated_at: NaiveDateTime.utc_now()])
|> Marker.multi_set_last_read_id(user, "notifications")
|> Repo.transaction()
end
@spec read_one(User.t(), String.t()) ::
@ -299,10 +295,6 @@ defmodule Pleroma.Notification do
|> Multi.update(:update, changeset(notification, %{seen: true}))
|> Marker.multi_set_last_read_id(user, "notifications")
|> Repo.transaction()
|> case do
{:ok, %{update: notification}} -> {:ok, notification}
{:error, :update, changeset, _} -> {:error, changeset}
end
end
end
@ -384,10 +376,15 @@ defmodule Pleroma.Notification do
defp do_create_notifications(%Activity{} = activity) do
enabled_receivers = get_notified_from_activity(activity)
enabled_subscribers = get_notified_subscribers_from_activity(activity)
notifications =
Enum.map(enabled_receivers, fn user ->
create_notification(activity, user)
end)
(Enum.map(enabled_receivers, fn user ->
create_notification(activity, user)
end) ++
Enum.map(enabled_subscribers -- enabled_receivers, fn user ->
create_notification(activity, user, type: "status")
end))
|> Enum.reject(&is_nil/1)
{:ok, notifications}
@ -492,7 +489,7 @@ defmodule Pleroma.Notification do
NOTE: might be called for FAKE Activities, see ActivityPub.Utils.get_notified_from_object/1
"""
@spec get_notified_from_activity(Activity.t(), boolean()) :: {list(User.t()), list(User.t())}
@spec get_notified_from_activity(Activity.t(), boolean()) :: list(User.t())
def get_notified_from_activity(activity, local_only \\ true)
def get_notified_from_activity(%Activity{data: %{"type" => type}} = activity, local_only)
@ -520,7 +517,25 @@ defmodule Pleroma.Notification do
Enum.filter(potential_receivers, fn u -> u.ap_id in notification_enabled_ap_ids end)
end
def get_notified_from_activity(_, _local_only), do: {[], []}
def get_notified_from_activity(_, _local_only), do: []
def get_notified_subscribers_from_activity(activity, local_only \\ true)
def get_notified_subscribers_from_activity(
%Activity{data: %{"type" => "Create"}} = activity,
local_only
) do
notification_enabled_ap_ids =
[]
|> Utils.maybe_notify_subscribers(activity)
potential_receivers =
User.get_users_from_set(notification_enabled_ap_ids, local_only: local_only)
Enum.filter(potential_receivers, fn u -> u.ap_id in notification_enabled_ap_ids end)
end
def get_notified_subscribers_from_activity(_, _), do: []
# For some activities, only notify the author of the object
def get_potential_receiver_ap_ids(%{data: %{"type" => type, "object" => object_id}})
@ -563,7 +578,6 @@ defmodule Pleroma.Notification do
[]
|> Utils.maybe_notify_to_recipients(activity)
|> Utils.maybe_notify_mentioned_recipients(activity)
|> Utils.maybe_notify_subscribers(activity)
|> Utils.maybe_notify_followers(activity)
|> Enum.uniq()
end
@ -720,7 +734,7 @@ defmodule Pleroma.Notification do
def mark_as_read?(activity, target_user) do
user = Activity.user_actor(activity)
User.mutes_user?(target_user, user) || CommonAPI.thread_muted?(target_user, activity)
User.mutes_user?(target_user, user) || CommonAPI.thread_muted?(activity, target_user)
end
def for_user_and_activity(user, activity) do
@ -743,8 +757,9 @@ defmodule Pleroma.Notification do
|> Repo.update_all(set: [seen: true])
end
@spec send(list(Notification.t())) :: :ok
def send(notifications) do
@doc "Streams a list of notifications over websockets and web push"
@spec stream(list(Notification.t())) :: :ok
def stream(notifications) do
Enum.each(notifications, fn notification ->
Streamer.stream(["user", "user:notification"], notification)
Push.send(notification)

21
lib/pleroma/object.ex
View file

@ -99,24 +99,6 @@ defmodule Pleroma.Object do
def get_by_id(nil), do: nil
def get_by_id(id), do: Repo.get(Object, id)
def get_by_id_and_maybe_refetch(id, opts \\ []) do
%{updated_at: updated_at} = object = get_by_id(id)
if opts[:interval] &&
NaiveDateTime.diff(NaiveDateTime.utc_now(), updated_at) > opts[:interval] do
case Fetcher.refetch_object(object) do
{:ok, %Object{} = object} ->
object
e ->
Logger.error("Couldn't refresh #{object.data["id"]}:\n#{inspect(e)}")
object
end
else
object
end
end
def get_by_ap_id(nil), do: nil
def get_by_ap_id(ap_id) do
@ -252,7 +234,8 @@ defmodule Pleroma.Object do
@spec cleanup_attachments(boolean(), Object.t()) ::
{:ok, Oban.Job.t() | nil}
def cleanup_attachments(true, %Object{} = object) do
AttachmentsCleanupWorker.enqueue("cleanup_attachments", %{"object" => object})
AttachmentsCleanupWorker.new(%{"op" => "cleanup_attachments", "object" => object})
|> Oban.insert()
end
def cleanup_attachments(_, _), do: {:ok, nil}

47
lib/pleroma/object/fetcher.ex
View file

@ -58,7 +58,12 @@ defmodule Pleroma.Object.Fetcher do
end
end
@typep fetcher_errors ::
:error | :reject | :allowed_depth | :fetch | :containment | :transmogrifier
# Note: will create a Create activity, which we need internally at the moment.
@spec fetch_object_from_id(String.t(), list()) ::
{:ok, Object.t()} | {fetcher_errors(), any()} | Pipeline.errors()
def fetch_object_from_id(id, options \\ []) do
with {_, nil} <- {:fetch_object, Object.get_cached_by_ap_id(id)},
{_, true} <- {:allowed_depth, Federator.allowed_thread_distance?(options[:depth])},
@ -72,50 +77,22 @@ defmodule Pleroma.Object.Fetcher do
{:object, data, Object.normalize(activity, fetch: false)} do
{:ok, object}
else
{:allowed_depth, false} = e ->
log_fetch_error(id, e)
{:error, :allowed_depth}
{:containment, reason} = e ->
log_fetch_error(id, e)
{:error, reason}
{:transmogrifier, {:error, {:reject, reason}}} = e ->
log_fetch_error(id, e)
{:reject, reason}
{:transmogrifier, {:reject, reason}} = e ->
log_fetch_error(id, e)
{:reject, reason}
{:transmogrifier, reason} = e ->
log_fetch_error(id, e)
{:error, reason}
{:object, data, nil} ->
reinject_object(%Object{}, data)
{:normalize, object = %Object{}} ->
{:ok, object}
{:fetch_object, %Object{} = object} ->
{:ok, object}
{:fetch, {:error, reason}} = e ->
log_fetch_error(id, e)
{:error, reason}
{:object, data, nil} ->
reinject_object(%Object{}, data)
e ->
log_fetch_error(id, e)
{:error, e}
Logger.metadata(object: id)
Logger.error("Object rejected while fetching #{id} #{inspect(e)}")
e
end
end
defp log_fetch_error(id, error) do
Logger.metadata(object: id)
Logger.error("Object rejected while fetching #{id} #{inspect(error)}")
end
defp prepare_activity_params(data) do
%{
"type" => "Create",
@ -168,6 +145,7 @@ defmodule Pleroma.Object.Fetcher do
Logger.debug("Fetching object #{id} via AP")
with {:scheme, true} <- {:scheme, String.starts_with?(id, "http")},
{_, true} <- {:mrf, MRF.id_filter(id)},
{:ok, body} <- get_object(id),
{:ok, data} <- safe_json_decode(body),
:ok <- Containment.contain_origin_from_id(id, data) do
@ -183,6 +161,9 @@ defmodule Pleroma.Object.Fetcher do
{:error, e} ->
{:error, e}
{:mrf, false} ->
{:error, {:reject, "Filtered by id"}}
e ->
{:error, e}
end

5
lib/pleroma/object/updater.ex
View file

@ -134,7 +134,10 @@ defmodule Pleroma.Object.Updater do
else
%{updated_object: updated_data} =
updated_data
|> maybe_update_history(original_data, updated: updated, use_history_in_new_object?: false)
|> maybe_update_history(original_data,
updated: updated,
use_history_in_new_object?: false
)
updated_data
|> Map.put("updated", date)

6
lib/pleroma/pagination.ex
View file

@ -89,9 +89,9 @@ defmodule Pleroma.Pagination do
defp cast_params(params) do
param_types = %{
min_id: :string,
since_id: :string,
max_id: :string,
min_id: params[:id_type] || :string,
since_id: params[:id_type] || :string,
max_id: params[:id_type] || :string,
offset: :integer,
limit: :integer,
skip_extra_order: :boolean,

23
lib/pleroma/release_tasks.ex
View file

@ -16,17 +16,24 @@ defmodule Pleroma.ReleaseTasks do
end
end
def find_module(task) do
module_name =
task
|> String.split(".")
|> Enum.map(&String.capitalize/1)
|> then(fn x -> [Mix, Tasks, Pleroma] ++ x end)
|> Module.concat()
case Code.ensure_loaded(module_name) do
{:module, _} -> module_name
_ -> nil
end
end
defp mix_task(task, args) do
Application.load(:pleroma)
{:ok, modules} = :application.get_key(:pleroma, :modules)
module =
Enum.find(modules, fn module ->
module = Module.split(module)
match?(["Mix", "Tasks", "Pleroma" | _], module) and
String.downcase(List.last(module)) == task
end)
module = find_module(task)
if module do
module.run(args)

16
lib/pleroma/reverse_proxy.ex
View file

@ -8,7 +8,7 @@ defmodule Pleroma.ReverseProxy do
~w(if-unmodified-since if-none-match) ++ @range_headers
@resp_cache_headers ~w(etag date last-modified)
@keep_resp_headers @resp_cache_headers ++
~w(content-type content-disposition content-encoding) ++
~w(content-length content-type content-disposition content-encoding) ++
~w(content-range accept-ranges vary)
@default_cache_control_header "public, max-age=1209600"
@valid_resp_codes [200, 206, 304]
@ -180,6 +180,7 @@ defmodule Pleroma.ReverseProxy do
result =
conn
|> put_resp_headers(build_resp_headers(headers, opts))
|> streaming_compat
|> send_chunked(status)
|> chunk_reply(client, opts)
@ -417,4 +418,17 @@ defmodule Pleroma.ReverseProxy do
@cachex.put(:failed_proxy_url_cache, url, true, ttl: ttl)
end
# When Cowboy handles a chunked response with a content-length header it streams
# over HTTP 1.1 instead of chunking. Bandit cannot stream over HTTP 1.1 so the header
# must be stripped or it breaks RFC compliance for Transfer Encoding: Chunked. RFC9112§6.2
#
# HTTP2 is always streamed for all adapters.
defp streaming_compat(conn) do
with Phoenix.Endpoint.Cowboy2Adapter <- Pleroma.Web.Endpoint.config(:adapter) do
conn
else
_ -> delete_resp_header(conn, "content-length")
end
end
end

2
lib/pleroma/scheduled_activity.ex
View file

@ -204,7 +204,7 @@ defmodule Pleroma.ScheduledActivity do
def job_query(scheduled_activity_id) do
from(j in Oban.Job,
where: j.queue == "scheduled_activities",
where: j.queue == "federator_outgoing",
where: fragment("args ->> 'activity_id' = ?::text", ^to_string(scheduled_activity_id))
)
end

14
lib/pleroma/search.ex
View file

@ -2,16 +2,22 @@ defmodule Pleroma.Search do
alias Pleroma.Workers.SearchIndexingWorker
def add_to_index(%Pleroma.Activity{id: activity_id}) do
SearchIndexingWorker.enqueue("add_to_index", %{"activity" => activity_id})
SearchIndexingWorker.new(%{"op" => "add_to_index", "activity" => activity_id})
|> Oban.insert()
end
def remove_from_index(%Pleroma.Object{id: object_id}) do
SearchIndexingWorker.enqueue("remove_from_index", %{"object" => object_id})
SearchIndexingWorker.new(%{"op" => "remove_from_index", "object" => object_id})
|> Oban.insert()
end
def search(query, options) do
search_module = Pleroma.Config.get([Pleroma.Search, :module], Pleroma.Activity)
search_module = Pleroma.Config.get([Pleroma.Search, :module])
search_module.search(options[:for_user], query, options)
end
def healthcheck_endpoints do
search_module = Pleroma.Config.get([Pleroma.Search, :module])
search_module.healthcheck_endpoints()
end
end

45
lib/pleroma/search/database_search.ex
View file

@ -28,7 +28,7 @@ defmodule Pleroma.Search.DatabaseSearch do
|> Activity.with_preloaded_object()
|> Activity.restrict_deactivated_users()
|> restrict_public(user)
|> query_with(index_type, search_query, :websearch)
|> query_with(index_type, search_query)
|> maybe_restrict_local(user)
|> maybe_restrict_author(author)
|> maybe_restrict_blocked(user)
@ -48,6 +48,15 @@ defmodule Pleroma.Search.DatabaseSearch do
@impl true
def remove_from_index(_object), do: :ok
@impl true
def create_index, do: :ok
@impl true
def drop_index, do: :ok
@impl true
def healthcheck_endpoints, do: nil
def maybe_restrict_author(query, %User{} = author) do
Activity.Queries.by_author(query, author)
end
@ -79,25 +88,7 @@ defmodule Pleroma.Search.DatabaseSearch do
)
end
defp query_with(q, :gin, search_query, :plain) do
%{rows: [[tsc]]} =
Ecto.Adapters.SQL.query!(
Pleroma.Repo,
"select current_setting('default_text_search_config')::regconfig::oid;"
)
from([a, o] in q,
where:
fragment(
"to_tsvector(?::oid::regconfig, ?->>'content') @@ plainto_tsquery(?)",
^tsc,
o.data,
^search_query
)
)
end
defp query_with(q, :gin, search_query, :websearch) do
defp query_with(q, :gin, search_query) do
%{rows: [[tsc]]} =
Ecto.Adapters.SQL.query!(
Pleroma.Repo,
@ -115,19 +106,7 @@ defmodule Pleroma.Search.DatabaseSearch do
)
end
defp query_with(q, :rum, search_query, :plain) do
from([a, o] in q,
where:
fragment(
"? @@ plainto_tsquery(?)",
o.fts_content,
^search_query
),
order_by: [fragment("? <=> now()::date", o.inserted_at)]
)
end
defp query_with(q, :rum, search_query, :websearch) do
defp query_with(q, :rum, search_query) do
from([a, o] in q,
where:
fragment(

86
lib/pleroma/search/healthcheck.ex Normal file
View file

@ -0,0 +1,86 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2024 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Search.Healthcheck do
@doc """
Monitors health of search backend to control processing of events based on health and availability.
"""
use GenServer
require Logger
@queue :search_indexing
@tick :timer.seconds(5)
@timeout :timer.seconds(2)
def start_link(_) do
GenServer.start_link(__MODULE__, [], name: __MODULE__)
end
@impl true
def init(_) do
state = %{healthy: false}
{:ok, state, {:continue, :start}}
end
@impl true
def handle_continue(:start, state) do
tick()
{:noreply, state}
end
@impl true
def handle_info(:check, state) do
urls = Pleroma.Search.healthcheck_endpoints()
new_state =
if check(urls) do
Oban.resume_queue(queue: @queue)
Map.put(state, :healthy, true)
else
Oban.pause_queue(queue: @queue)
Map.put(state, :healthy, false)
end
maybe_log_state_change(state, new_state)
tick()
{:noreply, new_state}
end
@impl true
def handle_call(:state, _from, state) do
{:reply, state, state, :hibernate}
end
def state, do: GenServer.call(__MODULE__, :state)
def check([]), do: true
def check(urls) when is_list(urls) do
Enum.all?(
urls,
fn url ->
case Pleroma.HTTP.get(url, [], recv_timeout: @timeout) do
{:ok, %{status: 200}} -> true
_ -> false
end
end
)
end
def check(_), do: true
defp tick do
Process.send_after(self(), :check, @tick)
end
defp maybe_log_state_change(%{healthy: true}, %{healthy: false}) do
Logger.error("Pausing Oban queue #{@queue} due to search backend healthcheck failure")
end
defp maybe_log_state_change(%{healthy: false}, %{healthy: true}) do
Logger.info("Resuming Oban queue #{@queue} due to search backend healthcheck pass")
end
defp maybe_log_state_change(_, _), do: :ok
end

18
lib/pleroma/search/meilisearch.ex
View file

@ -10,6 +10,12 @@ defmodule Pleroma.Search.Meilisearch do
@behaviour Pleroma.Search.SearchBackend
@impl true
def create_index, do: :ok
@impl true
def drop_index, do: :ok
defp meili_headers do
private_key = Config.get([Pleroma.Search.Meilisearch, :private_key])
@ -116,6 +122,7 @@ defmodule Pleroma.Search.Meilisearch do
# Only index public or unlisted Notes
if not is_nil(object) and object.data["type"] == "Note" and
not is_nil(object.data["content"]) and
not is_nil(object.data["published"]) and
(Pleroma.Constants.as_public() in object.data["to"] or
Pleroma.Constants.as_public() in object.data["cc"]) and
object.data["content"] not in ["", "."] do
@ -178,4 +185,15 @@ defmodule Pleroma.Search.Meilisearch do
def remove_from_index(object) do
meili_delete("/indexes/objects/documents/#{object.id}")
end
@impl true
def healthcheck_endpoints do
endpoint =
Config.get([Pleroma.Search.Meilisearch, :url])
|> URI.parse()
|> Map.put(:path, "/health")
|> URI.to_string()
[endpoint]
end
end

182
lib/pleroma/search/qdrant_search.ex Normal file
View file

@ -0,0 +1,182 @@
defmodule Pleroma.Search.QdrantSearch do
@behaviour Pleroma.Search.SearchBackend
import Ecto.Query
alias Pleroma.Activity
alias Pleroma.Config.Getting, as: Config
alias __MODULE__.OpenAIClient
alias __MODULE__.QdrantClient
import Pleroma.Search.Meilisearch, only: [object_to_search_data: 1]
import Pleroma.Search.DatabaseSearch, only: [maybe_fetch: 3]
@impl true
def create_index do
payload = Config.get([Pleroma.Search.QdrantSearch, :qdrant_index_configuration])
with {:ok, %{status: 200}} <- QdrantClient.put("/collections/posts", payload) do
:ok
else
e -> {:error, e}
end
end
@impl true
def drop_index do
with {:ok, %{status: 200}} <- QdrantClient.delete("/collections/posts") do
:ok
else
e -> {:error, e}
end
end
def get_embedding(text) do
with {:ok, %{body: %{"data" => [%{"embedding" => embedding}]}}} <-
OpenAIClient.post("/v1/embeddings", %{
input: text,
model: Config.get([Pleroma.Search.QdrantSearch, :openai_model])
}) do
{:ok, embedding}
else
_ ->
{:error, "Failed to get embedding"}
end
end
defp actor_from_activity(%{data: %{"actor" => actor}}) do
actor
end
defp actor_from_activity(_), do: nil
defp build_index_payload(activity, embedding) do
actor = actor_from_activity(activity)
published_at = activity.data["published"]
%{
points: [
%{
id: activity.id |> FlakeId.from_string() |> Ecto.UUID.cast!(),
vector: embedding,
payload: %{actor: actor, published_at: published_at}
}
]
}
end
defp build_search_payload(embedding, options) do
base = %{
vector: embedding,
limit: options[:limit] || 20,
offset: options[:offset] || 0
}
if author = options[:author] do
Map.put(base, :filter, %{
must: [%{key: "actor", match: %{value: author.ap_id}}]
})
else
base
end
end
@impl true
def add_to_index(activity) do
# This will only index public or unlisted notes
maybe_search_data = object_to_search_data(activity.object)
if activity.data["type"] == "Create" and maybe_search_data do
with {:ok, embedding} <- get_embedding(maybe_search_data.content),
{:ok, %{status: 200}} <-
QdrantClient.put(
"/collections/posts/points",
build_index_payload(activity, embedding)
) do
:ok
else
e -> {:error, e}
end
else
:ok
end
end
@impl true
def remove_from_index(object) do
activity = Activity.get_by_object_ap_id_with_object(object.data["id"])
id = activity.id |> FlakeId.from_string() |> Ecto.UUID.cast!()
with {:ok, %{status: 200}} <-
QdrantClient.post("/collections/posts/points/delete", %{"points" => [id]}) do
:ok
else
e -> {:error, e}
end
end
@impl true
def search(user, original_query, options) do
query = "Represent this sentence for searching relevant passages: #{original_query}"
with {:ok, embedding} <- get_embedding(query),
{:ok, %{body: %{"result" => result}}} <-
QdrantClient.post(
"/collections/posts/points/search",
build_search_payload(embedding, options)
) do
ids =
Enum.map(result, fn %{"id" => id} ->
Ecto.UUID.dump!(id)
end)
from(a in Activity, where: a.id in ^ids)
|> Activity.with_preloaded_object()
|> Activity.restrict_deactivated_users()
|> Ecto.Query.order_by([a], fragment("array_position(?, ?)", ^ids, a.id))
|> Pleroma.Repo.all()
|> maybe_fetch(user, original_query)
else
_ ->
[]
end
end
@impl true
def healthcheck_endpoints do
qdrant_health =
Config.get([Pleroma.Search.QdrantSearch, :qdrant_url])
|> URI.parse()
|> Map.put(:path, "/healthz")
|> URI.to_string()
openai_health = Config.get([Pleroma.Search.QdrantSearch, :openai_healthcheck_url])
[qdrant_health, openai_health] |> Enum.filter(& &1)
end
end
defmodule Pleroma.Search.QdrantSearch.OpenAIClient do
use Tesla
alias Pleroma.Config.Getting, as: Config
plug(Tesla.Middleware.BaseUrl, Config.get([Pleroma.Search.QdrantSearch, :openai_url]))
plug(Tesla.Middleware.JSON)
plug(Tesla.Middleware.Headers, [
{"Authorization",
"Bearer #{Pleroma.Config.get([Pleroma.Search.QdrantSearch, :openai_api_key])}"}
])
end
defmodule Pleroma.Search.QdrantSearch.QdrantClient do
use Tesla
alias Pleroma.Config.Getting, as: Config
plug(Tesla.Middleware.BaseUrl, Config.get([Pleroma.Search.QdrantSearch, :qdrant_url]))
plug(Tesla.Middleware.JSON)
plug(Tesla.Middleware.Headers, [
{"api-key", Pleroma.Config.get([Pleroma.Search.QdrantSearch, :qdrant_api_key])}
])
end

18
lib/pleroma/search/search_backend.ex
View file

@ -21,4 +21,22 @@ defmodule Pleroma.Search.SearchBackend do
from index.
"""
@callback remove_from_index(object :: Pleroma.Object.t()) :: :ok | {:error, any()}
@doc """
Create the index
"""
@callback create_index() :: :ok | {:error, any()}
@doc """
Drop the index
"""
@callback drop_index() :: :ok | {:error, any()}
@doc """
Healthcheck endpoints of search backend infrastructure to monitor for controlling
processing of jobs in the Oban queue.
It is expected a 200 response is healthy and other responses are unhealthy.
"""
@callback healthcheck_endpoints :: list() | nil
end

68
lib/pleroma/signature.ex
View file

@ -10,6 +10,14 @@ defmodule Pleroma.Signature do
alias Pleroma.User
alias Pleroma.Web.ActivityPub.ActivityPub
import Plug.Conn, only: [put_req_header: 3]
@http_signatures_impl Application.compile_env(
:pleroma,
[__MODULE__, :http_signatures_impl],
HTTPSignatures
)
@known_suffixes ["/publickey", "/main-key"]
def key_id_to_actor_id(key_id) do
@ -44,8 +52,7 @@ defmodule Pleroma.Signature do
defp remove_suffix(uri, []), do: uri
def fetch_public_key(conn) do
with %{"keyId" => kid} <- HTTPSignatures.signature_for_conn(conn),
{:ok, actor_id} <- key_id_to_actor_id(kid),
with {:ok, actor_id} <- get_actor_id(conn),
{:ok, public_key} <- User.get_public_key_for_ap_id(actor_id) do
{:ok, public_key}
else
@ -55,8 +62,7 @@ defmodule Pleroma.Signature do
end
def refetch_public_key(conn) do
with %{"keyId" => kid} <- HTTPSignatures.signature_for_conn(conn),
{:ok, actor_id} <- key_id_to_actor_id(kid),
with {:ok, actor_id} <- get_actor_id(conn),
{:ok, _user} <- ActivityPub.make_user_from_ap_id(actor_id),
{:ok, public_key} <- User.get_public_key_for_ap_id(actor_id) do
{:ok, public_key}
@ -66,6 +72,16 @@ defmodule Pleroma.Signature do
end
end
def get_actor_id(conn) do
with %{"keyId" => kid} <- HTTPSignatures.signature_for_conn(conn),
{:ok, actor_id} <- key_id_to_actor_id(kid) do
{:ok, actor_id}
else
e ->
{:error, e}
end
end
def sign(%User{keys: keys} = user, headers) do
with {:ok, private_key, _} <- Keys.keys_from_pem(keys) do
HTTPSignatures.sign(private_key, user.ap_id <> "#main-key", headers)
@ -77,4 +93,48 @@ defmodule Pleroma.Signature do
def signed_date(%NaiveDateTime{} = date) do
Timex.format!(date, "{WDshort}, {0D} {Mshort} {YYYY} {h24}:{m}:{s} GMT")
end
@spec validate_signature(Plug.Conn.t(), String.t()) :: boolean()
def validate_signature(%Plug.Conn{} = conn, request_target) do
# Newer drafts for HTTP signatures now use @request-target instead of the
# old (request-target). We'll now support both for incoming signatures.
conn =
conn
|> put_req_header("(request-target)", request_target)
|> put_req_header("@request-target", request_target)
@http_signatures_impl.validate_conn(conn)
end
@spec validate_signature(Plug.Conn.t()) :: boolean()
def validate_signature(%Plug.Conn{} = conn) do
# This (request-target) is non-standard, but many implementations do it
# this way due to a misinterpretation of
# https://datatracker.ietf.org/doc/html/draft-cavage-http-signatures-06
# "path" was interpreted as not having the query, though later examples
# show that it must be the absolute path + query. This behavior is kept to
# make sure most software (Pleroma itself, Mastodon, and probably others)
# do not break.
request_target = Enum.join([String.downcase(conn.method), conn.request_path], " ")
# This is the proper way to build the @request-target, as expected by
# many HTTP signature libraries, clarified in the following draft:
# https://www.ietf.org/archive/id/draft-ietf-httpbis-message-signatures-11.html#section-2.2.6
# It is the same as before, but containing the query part as well.
proper_target = Enum.join([request_target, "?", conn.query_string], "")
cond do
# Normal, non-standard behavior but expected by Pleroma and more.
validate_signature(conn, request_target) ->
true
# Has query string and the previous one failed: let's try the standard.
conn.query_string != "" ->
validate_signature(conn, proper_target)
# If there's no query string and signature fails, it's rotten.
true ->
false
end
end
end

4
lib/pleroma/telemetry/logger.ex
View file

@ -39,7 +39,7 @@ defmodule Pleroma.Telemetry.Logger do
_,
_
) do
Logger.error(fn ->
Logger.debug(fn ->
"Connection pool failed to reclaim any connections due to all of them being in use. It will have to drop requests for opening connections to new hosts"
end)
end
@ -70,7 +70,7 @@ defmodule Pleroma.Telemetry.Logger do
%{key: key},
_
) do
Logger.warning(fn ->
Logger.debug(fn ->
"Pool worker for #{key}: Client #{inspect(client_pid)} died before releasing the connection with #{inspect(reason)}"
end)
end

26
lib/pleroma/upload.ex
View file

@ -239,20 +239,26 @@ defmodule Pleroma.Upload do
""
end
[base_url, path]
|> Path.join()
if String.contains?(base_url, Pleroma.Uploaders.IPFS.placeholder()) do
String.replace(base_url, Pleroma.Uploaders.IPFS.placeholder(), path)
else
[base_url, path]
|> Path.join()
end
end
defp url_from_spec(_upload, _base_url, {:url, url}), do: url
@spec base_url() :: binary
def base_url do
uploader = @config_impl.get([Pleroma.Upload, :uploader])
upload_base_url = @config_impl.get([Pleroma.Upload, :base_url])
upload_fallback_url = Pleroma.Web.Endpoint.url() <> "/media/"
upload_base_url = @config_impl.get([Pleroma.Upload, :base_url]) || upload_fallback_url
public_endpoint = @config_impl.get([uploader, :public_endpoint])
case uploader do
Pleroma.Uploaders.Local ->
upload_base_url || Pleroma.Web.Endpoint.url() <> "/media/"
upload_base_url
Pleroma.Uploaders.S3 ->
bucket = @config_impl.get([Pleroma.Uploaders.S3, :bucket])
@ -264,11 +270,14 @@ defmodule Pleroma.Upload do
!is_nil(truncated_namespace) ->
truncated_namespace
!is_nil(namespace) ->
!is_nil(namespace) and !is_nil(bucket) ->
namespace <> ":" <> bucket
true ->
!is_nil(bucket) ->
bucket
true ->
""
end
if public_endpoint do
@ -277,8 +286,11 @@ defmodule Pleroma.Upload do
Path.join([upload_base_url, bucket_with_namespace])
end
Pleroma.Uploaders.IPFS ->
@config_impl.get([Pleroma.Uploaders.IPFS, :get_gateway_url])
_ ->
public_endpoint || upload_base_url || Pleroma.Web.Endpoint.url() <> "/media/"
public_endpoint || upload_base_url
end
end
end

10
lib/pleroma/upload/filter/analyze_metadata.ex
View file

@ -90,9 +90,13 @@ defmodule Pleroma.Upload.Filter.AnalyzeMetadata do
{:ok, rgb} =
if Image.has_alpha?(resized_image) do
# remove alpha channel
resized_image
|> Operation.extract_band!(0, n: 3)
|> Image.write_to_binary()
case Operation.extract_band(resized_image, 0, n: 3) do
{:ok, data} ->
Image.write_to_binary(data)
_ ->
Image.write_to_binary(resized_image)
end
else
Image.write_to_binary(resized_image)
end

10
lib/pleroma/upload/filter/dedupe.ex
View file

@ -17,8 +17,16 @@ defmodule Pleroma.Upload.Filter.Dedupe do
|> Base.encode16(case: :lower)
filename = shasum <> "." <> extension
{:ok, :filtered, %Upload{upload | id: shasum, path: filename}}
{:ok, :filtered, %Upload{upload | id: shasum, path: shard_path(filename)}}
end
def filter(_), do: {:ok, :noop}
@spec shard_path(String.t()) :: String.t()
def shard_path(
<<a::binary-size(2), b::binary-size(2), c::binary-size(2), _::binary>> = filename
) do
Path.join([a, b, c, filename])
end
end

6
lib/pleroma/upload/filter/exiftool/strip_location.ex
View file

@ -9,8 +9,6 @@ defmodule Pleroma.Upload.Filter.Exiftool.StripLocation do
"""
@behaviour Pleroma.Upload.Filter
@spec filter(Pleroma.Upload.t()) :: {:ok, any()} | {:error, String.t()}
# Formats not compatible with exiftool at this time
def filter(%Pleroma.Upload{content_type: "image/heic"}), do: {:ok, :noop}
def filter(%Pleroma.Upload{content_type: "image/webp"}), do: {:ok, :noop}
@ -18,7 +16,9 @@ defmodule Pleroma.Upload.Filter.Exiftool.StripLocation do
def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do
try do
case System.cmd("exiftool", ["-overwrite_original", "-gps:all=", file], parallelism: true) do
case System.cmd("exiftool", ["-overwrite_original", "-gps:all=", "-png:all=", file],
parallelism: true
) do
{_response, 0} -> {:ok, :filtered}
{error, 1} -> {:error, error}
end

1
lib/pleroma/upload/filter/mogrifun.ex
View file

@ -38,7 +38,6 @@ defmodule Pleroma.Upload.Filter.Mogrifun do
[{"fill", "yellow"}, {"tint", "40"}]
]
@spec filter(Pleroma.Upload.t()) :: {:ok, atom()} | {:error, String.t()}
def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do
try do
Filter.Mogrify.do_filter(file, [Enum.random(@filters)])

1
lib/pleroma/upload/filter/mogrify.ex
View file

@ -8,7 +8,6 @@ defmodule Pleroma.Upload.Filter.Mogrify do
@type conversion :: action :: String.t() | {action :: String.t(), opts :: String.t()}
@type conversions :: conversion() | [conversion()]
@spec filter(Pleroma.Upload.t()) :: {:ok, :atom} | {:error, String.t()}
def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do
try do
do_filter(file, Pleroma.Config.get!([__MODULE__, :args]))

72
lib/pleroma/uploaders/ipfs.ex Normal file
View file

@ -0,0 +1,72 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Uploaders.IPFS do
@behaviour Pleroma.Uploaders.Uploader
require Logger
alias Tesla.Multipart
@api_add "/api/v0/add"
@api_delete "/api/v0/files/rm"
@config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config)
@placeholder "{CID}"
def placeholder, do: @placeholder
@impl true
def get_file(file) do
b_url = Pleroma.Upload.base_url()
if String.contains?(b_url, @placeholder) do
{:ok, {:url, String.replace(b_url, @placeholder, URI.decode(file))}}
else
{:error, "IPFS Get URL doesn't contain 'cid' placeholder"}
end
end
@impl true
def put_file(%Pleroma.Upload{tempfile: tempfile}) do
mp =
Multipart.new()
|> Multipart.add_content_type_param("charset=utf-8")
|> Multipart.add_file(tempfile)
endpoint = ipfs_endpoint(@api_add)
with {:ok, %{body: body}} when is_binary(body) <-
Pleroma.HTTP.post(endpoint, mp, [], params: ["cid-version": "1"], pool: :upload),
{_, {:ok, decoded}} <- {:json, Jason.decode(body)},
{_, true} <- {:hash, Map.has_key?(decoded, "Hash")} do
{:ok, {:file, decoded["Hash"]}}
else
{:hash, false} ->
{:error, "JSON doesn't contain Hash key"}
{:json, error} ->
Logger.error("#{__MODULE__}: #{inspect(error)}")
{:error, "JSON decode failed"}
error ->
Logger.error("#{__MODULE__}: #{inspect(error)}")
{:error, "IPFS Gateway upload failed"}
end
end
@impl true
def delete_file(file) do
endpoint = ipfs_endpoint(@api_delete)
case Pleroma.HTTP.post(endpoint, "", [], params: [arg: file]) do
{:ok, %{status: 204}} -> :ok
error -> {:error, inspect(error)}
end
end
defp ipfs_endpoint(path) do
URI.parse(@config_impl.get([__MODULE__, :post_gateway_url]))
|> Map.put(:path, path)
|> URI.to_string()
end
end

195
lib/pleroma/user.ex
View file

@ -19,6 +19,7 @@ defmodule Pleroma.User do
alias Pleroma.Emoji
alias Pleroma.FollowingRelationship
alias Pleroma.Formatter
alias Pleroma.Hashtag
alias Pleroma.HTML
alias Pleroma.Keys
alias Pleroma.MFA
@ -27,6 +28,7 @@ defmodule Pleroma.User do
alias Pleroma.Registration
alias Pleroma.Repo
alias Pleroma.User
alias Pleroma.User.HashtagFollow
alias Pleroma.UserRelationship
alias Pleroma.Web.ActivityPub.ActivityPub
alias Pleroma.Web.ActivityPub.Builder
@ -38,6 +40,8 @@ defmodule Pleroma.User do
alias Pleroma.Web.OAuth
alias Pleroma.Web.RelMe
alias Pleroma.Workers.BackgroundWorker
alias Pleroma.Workers.DeleteWorker
alias Pleroma.Workers.UserRefreshWorker
require Logger
require Pleroma.Constants
@ -172,6 +176,12 @@ defmodule Pleroma.User do
has_many(:outgoing_relationships, UserRelationship, foreign_key: :source_id)
has_many(:incoming_relationships, UserRelationship, foreign_key: :target_id)
many_to_many(:followed_hashtags, Hashtag,
on_replace: :delete,
on_delete: :delete_all,
join_through: HashtagFollow
)
for {relationship_type,
[
{outgoing_relation, outgoing_relation_target},
@ -417,6 +427,11 @@ defmodule Pleroma.User do
end
end
def image_description(image, default \\ "")
def image_description(%{"name" => name}, _default), do: name
def image_description(_, default), do: default
# Should probably be renamed or removed
@spec ap_id(User.t()) :: String.t()
def ap_id(%User{nickname: nickname}), do: "#{Endpoint.url()}/users/#{nickname}"
@ -461,6 +476,7 @@ defmodule Pleroma.User do
def remote_user_changeset(struct \\ %User{local: false}, params) do
bio_limit = Config.get([:instance, :user_bio_length], 5000)
name_limit = Config.get([:instance, :user_name_length], 100)
fields_limit = Config.get([:instance, :max_remote_account_fields], 0)
name =
case params[:name] do
@ -474,6 +490,7 @@ defmodule Pleroma.User do
|> Map.put_new(:last_refreshed_at, NaiveDateTime.utc_now())
|> truncate_if_exists(:name, name_limit)
|> truncate_if_exists(:bio, bio_limit)
|> Map.update(:fields, [], &Enum.take(&1, fields_limit))
|> truncate_fields_param()
|> fix_follower_address()
@ -582,16 +599,26 @@ defmodule Pleroma.User do
|> validate_length(:bio, max: bio_limit)
|> validate_length(:name, min: 1, max: name_limit)
|> validate_inclusion(:actor_type, Pleroma.Constants.allowed_user_actor_types())
|> validate_image_description(:avatar_description, params)
|> validate_image_description(:header_description, params)
|> put_fields()
|> put_emoji()
|> put_change_if_present(:bio, &{:ok, parse_bio(&1, struct)})
|> put_change_if_present(:avatar, &put_upload(&1, :avatar))
|> put_change_if_present(:banner, &put_upload(&1, :banner))
|> put_change_if_present(
:avatar,
&put_upload(&1, :avatar, Map.get(params, :avatar_description))
)
|> put_change_if_present(
:banner,
&put_upload(&1, :banner, Map.get(params, :header_description))
)
|> put_change_if_present(:background, &put_upload(&1, :background))
|> put_change_if_present(
:pleroma_settings_store,
&{:ok, Map.merge(struct.pleroma_settings_store, &1)}
)
|> maybe_update_image_description(:avatar, Map.get(params, :avatar_description))
|> maybe_update_image_description(:banner, Map.get(params, :header_description))
|> validate_fields(false)
end
@ -670,13 +697,41 @@ defmodule Pleroma.User do
end
end
defp put_upload(value, type) do
defp put_upload(value, type, description \\ nil) do
with %Plug.Upload{} <- value,
{:ok, object} <- ActivityPub.upload(value, type: type) do
{:ok, object} <- ActivityPub.upload(value, type: type, description: description) do
{:ok, object.data}
end
end
defp validate_image_description(changeset, key, params) do
description_limit = Config.get([:instance, :description_limit], 5_000)
description = Map.get(params, key)
if is_binary(description) and String.length(description) > description_limit do
changeset
|> add_error(key, "#{key} is too long")
else
changeset
end
end
defp maybe_update_image_description(changeset, image_field, description)
when is_binary(description) do
with {:image_missing, true} <- {:image_missing, not changed?(changeset, image_field)},
{:existing_image, %{"id" => id}} <-
{:existing_image, Map.get(changeset.data, image_field)},
{:object, %Object{} = object} <- {:object, Object.get_by_ap_id(id)},
{:ok, object} <- Object.update_data(object, %{"name" => description}) do
put_change(changeset, image_field, object.data)
else
{:description_too_long, true} -> {:error}
_ -> changeset
end
end
defp maybe_update_image_description(changeset, _, _), do: changeset
def update_as_admin_changeset(struct, params) do
struct
|> update_changeset(params)
@ -734,7 +789,8 @@ defmodule Pleroma.User do
end
def force_password_reset_async(user) do
BackgroundWorker.enqueue("force_password_reset", %{"user_id" => user.id})
BackgroundWorker.new(%{"op" => "force_password_reset", "user_id" => user.id})
|> Oban.insert()
end
@spec force_password_reset(User.t()) :: {:ok, User.t()} | {:error, Ecto.Changeset.t()}
@ -1216,7 +1272,8 @@ defmodule Pleroma.User do
def update_and_set_cache(changeset) do
with {:ok, user} <- Repo.update(changeset, stale_error_field: :id) do
if get_change(changeset, :raw_fields) do
BackgroundWorker.enqueue("verify_fields_links", %{"user_id" => user.id})
BackgroundWorker.new(%{"op" => "verify_fields_links", "user_id" => user.id})
|> Oban.insert()
end
set_cache(user)
@ -1404,6 +1461,40 @@ defmodule Pleroma.User do
|> Repo.all()
end
@spec get_familiar_followers_query(User.t(), User.t(), pos_integer() | nil) :: Ecto.Query.t()
def get_familiar_followers_query(%User{} = user, %User{} = current_user, nil) do
friends =
get_friends_query(current_user)
|> where([u], not u.hide_follows)
|> select([u], u.id)
User.Query.build(%{is_active: true})
|> where([u], u.id not in ^[user.id, current_user.id])
|> join(:inner, [u], r in FollowingRelationship,
as: :followers_relationships,
on: r.following_id == ^user.id and r.follower_id == u.id
)
|> where([followers_relationships: r], r.state == ^:follow_accept)
|> where([followers_relationships: r], r.follower_id in subquery(friends))
end
def get_familiar_followers_query(%User{} = user, %User{} = current_user, page) do
user
|> get_familiar_followers_query(current_user, nil)
|> User.Query.paginate(page, 20)
end
@spec get_familiar_followers_query(User.t(), User.t()) :: Ecto.Query.t()
def get_familiar_followers_query(%User{} = user, %User{} = current_user),
do: get_familiar_followers_query(user, current_user, nil)
@spec get_familiar_followers(User.t(), User.t(), pos_integer() | nil) :: {:ok, list(User.t())}
def get_familiar_followers(%User{} = user, %User{} = current_user, page \\ nil) do
user
|> get_familiar_followers_query(current_user, page)
|> Repo.all()
end
def increase_note_count(%User{} = user) do
User
|> where(id: ^user.id)
@ -1553,11 +1644,11 @@ defmodule Pleroma.User do
)) ||
{:ok, nil} do
if duration > 0 do
Pleroma.Workers.MuteExpireWorker.enqueue(
"unmute_user",
%{"muter_id" => muter.id, "mutee_id" => mutee.id},
Pleroma.Workers.MuteExpireWorker.new(
%{"op" => "unmute_user", "muter_id" => muter.id, "mutee_id" => mutee.id},
scheduled_at: expires_at
)
|> Oban.insert()
end
@cachex.del(:user_cache, "muted_users_ap_ids:#{muter.ap_id}")
@ -1800,7 +1891,8 @@ defmodule Pleroma.User do
defp maybe_filter_on_ap_id(query, _ap_ids), do: query
def set_activation_async(user, status \\ true) do
BackgroundWorker.enqueue("user_activation", %{"user_id" => user.id, "status" => status})
BackgroundWorker.new(%{"op" => "user_activation", "user_id" => user.id, "status" => status})
|> Oban.insert()
end
@spec set_activation([User.t()], boolean()) :: {:ok, User.t()} | {:error, Ecto.Changeset.t()}
@ -1947,7 +2039,9 @@ defmodule Pleroma.User do
def delete(%User{} = user) do
# Purge the user immediately
purge(user)
BackgroundWorker.enqueue("delete_user", %{"user_id" => user.id})
DeleteWorker.new(%{"op" => "delete_user", "user_id" => user.id})
|> Oban.insert()
end
# *Actually* delete the user from the DB
@ -2019,7 +2113,8 @@ defmodule Pleroma.User do
%{scheme: scheme, userinfo: nil, host: host}
when not_empty_string(host) and scheme in ["http", "https"] <-
URI.parse(value),
{:not_idn, true} <- {:not_idn, to_string(:idna.encode(host)) == host},
{:not_idn, true} <-
{:not_idn, match?(^host, to_string(:idna.encode(to_charlist(host))))},
"me" <- Pleroma.Web.RelMe.maybe_put_rel_me(value, profile_urls) do
CommonUtils.to_masto_date(NaiveDateTime.utc_now())
else
@ -2119,20 +2214,20 @@ defmodule Pleroma.User do
def fetch_by_ap_id(ap_id), do: ActivityPub.make_user_from_ap_id(ap_id)
@spec get_or_fetch_by_ap_id(String.t()) :: {:ok, User.t()} | {:error, any()}
def get_or_fetch_by_ap_id(ap_id) do
cached_user = get_cached_by_ap_id(ap_id)
with cached_user = %User{} <- get_cached_by_ap_id(ap_id),
_ <- maybe_refresh(cached_user) do
{:ok, cached_user}
else
_ -> fetch_by_ap_id(ap_id)
end
end
maybe_fetched_user = needs_update?(cached_user) && fetch_by_ap_id(ap_id)
case {cached_user, maybe_fetched_user} do
{_, {:ok, %User{} = user}} ->
{:ok, user}
{%User{} = user, _} ->
{:ok, user}
_ ->
{:error, :not_found}
defp maybe_refresh(user) do
if needs_update?(user) do
UserRefreshWorker.new(%{"ap_id" => user.ap_id})
|> Oban.insert()
end
end
@ -2693,7 +2788,7 @@ defmodule Pleroma.User do
end
end
@spec add_to_block(User.t(), User.t()) ::
@spec remove_from_block(User.t(), User.t()) ::
{:ok, UserRelationship.t()} | {:ok, nil} | {:error, Ecto.Changeset.t()}
defp remove_from_block(%User{} = user, %User{} = blocked) do
with {:ok, relationship} <- UserRelationship.delete_block(user, blocked) do
@ -2774,4 +2869,54 @@ defmodule Pleroma.User do
birthday_month: month
})
end
defp maybe_load_followed_hashtags(%User{followed_hashtags: follows} = user)
when is_list(follows),
do: user
defp maybe_load_followed_hashtags(%User{} = user) do
followed_hashtags = HashtagFollow.get_by_user(user)
%{user | followed_hashtags: followed_hashtags}
end
def followed_hashtags(%User{followed_hashtags: follows})
when is_list(follows),
do: follows
def followed_hashtags(%User{} = user) do
{:ok, user} =
user
|> maybe_load_followed_hashtags()
|> set_cache()
user.followed_hashtags
end
def follow_hashtag(%User{} = user, %Hashtag{} = hashtag) do
Logger.debug("Follow hashtag #{hashtag.name} for user #{user.nickname}")
user = maybe_load_followed_hashtags(user)
with {:ok, _} <- HashtagFollow.new(user, hashtag),
follows <- HashtagFollow.get_by_user(user),
%User{} = user <- user |> Map.put(:followed_hashtags, follows) do
user
|> set_cache()
end
end
def unfollow_hashtag(%User{} = user, %Hashtag{} = hashtag) do
Logger.debug("Unfollow hashtag #{hashtag.name} for user #{user.nickname}")
user = maybe_load_followed_hashtags(user)
with {:ok, _} <- HashtagFollow.delete(user, hashtag),
follows <- HashtagFollow.get_by_user(user),
%User{} = user <- user |> Map.put(:followed_hashtags, follows) do
user
|> set_cache()
end
end
def following_hashtag?(%User{} = user, %Hashtag{} = hashtag) do
not is_nil(HashtagFollow.get(user, hashtag))
end
end

377
lib/pleroma/user/backup.ex
View file

@ -14,9 +14,10 @@ defmodule Pleroma.User.Backup do
alias Pleroma.Activity
alias Pleroma.Bookmark
alias Pleroma.Config
alias Pleroma.Repo
alias Pleroma.Uploaders.Uploader
alias Pleroma.User
alias Pleroma.User.Backup.State
alias Pleroma.Web.ActivityPub.ActivityPub
alias Pleroma.Web.ActivityPub.Transmogrifier
alias Pleroma.Web.ActivityPub.UserView
@ -29,71 +30,107 @@ defmodule Pleroma.User.Backup do
field(:file_name, :string)
field(:file_size, :integer, default: 0)
field(:processed, :boolean, default: false)
field(:state, State, default: :invalid)
field(:processed_number, :integer, default: 0)
field(:tempdir, :string)
belongs_to(:user, User, type: FlakeId.Ecto.CompatType)
timestamps()
end
@config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config)
@doc """
Schedules a job to backup a user if the number of backup requests has not exceeded the limit.
def create(user, admin_id \\ nil) do
with :ok <- validate_limit(user, admin_id),
{:ok, backup} <- user |> new() |> Repo.insert() do
BackupWorker.process(backup, admin_id)
Admins can directly call new/1 and schedule_backup/1 to bypass the limit.
"""
@spec user(User.t()) :: {:ok, t()} | {:error, any()}
def user(user) do
days = Config.get([__MODULE__, :limit_days])
with true <- permitted?(user),
%__MODULE__{} = backup <- new(user),
{:ok, inserted_backup} <- Repo.insert(backup),
{:ok, %Oban.Job{}} <- schedule_backup(inserted_backup) do
{:ok, inserted_backup}
else
false ->
{:error,
dngettext(
"errors",
"Last export was less than a day ago",
"Last export was less than %{days} days ago",
days,
days: days
)}
e ->
{:error, e}
end
end
@doc "Generates a %Backup{} for a user with a random file name"
@spec new(User.t()) :: t()
def new(user) do
rand_str = :crypto.strong_rand_bytes(32) |> Base.url_encode64(padding: false)
datetime = Calendar.NaiveDateTime.Format.iso8601_basic(NaiveDateTime.utc_now())
name = "archive-#{user.nickname}-#{datetime}-#{rand_str}.zip"
%__MODULE__{
user_id: user.id,
content_type: "application/zip",
file_name: name,
state: :pending
tempdir: tempdir(),
user: user
}
end
def delete(backup) do
uploader = Pleroma.Config.get([Pleroma.Upload, :uploader])
@doc "Schedules the execution of the provided backup"
@spec schedule_backup(t()) :: {:ok, Oban.Job.t()} | {:error, any()}
def schedule_backup(backup) do
with false <- is_nil(backup.id) do
%{"op" => "process", "backup_id" => backup.id}
|> BackupWorker.new()
|> Oban.insert()
else
true ->
{:error, "Backup is missing id. Please insert it into the Repo first."}
end
end
@doc "Deletes the backup archive file and removes the database record"
@spec delete_archive(t()) :: {:ok, Ecto.Schema.t()} | {:error, Ecto.Changeset.t()}
def delete_archive(backup) do
uploader = Config.get([Pleroma.Upload, :uploader])
with :ok <- uploader.delete_file(Path.join("backups", backup.file_name)) do
Repo.delete(backup)
end
end
defp validate_limit(_user, admin_id) when is_binary(admin_id), do: :ok
@doc "Schedules a job to delete the backup archive"
@spec schedule_delete(t()) :: {:ok, Oban.Job.t()} | {:error, any()}
def schedule_delete(backup) do
days = Config.get([__MODULE__, :purge_after_days])
time = 60 * 60 * 24 * days
scheduled_at = Calendar.NaiveDateTime.add!(backup.inserted_at, time)
defp validate_limit(user, nil) do
case get_last(user.id) do
%__MODULE__{inserted_at: inserted_at} ->
days = Pleroma.Config.get([__MODULE__, :limit_days])
diff = Timex.diff(NaiveDateTime.utc_now(), inserted_at, :days)
%{"op" => "delete", "backup_id" => backup.id}
|> BackupWorker.new(scheduled_at: scheduled_at)
|> Oban.insert()
end
if diff > days do
:ok
else
{:error,
dngettext(
"errors",
"Last export was less than a day ago",
"Last export was less than %{days} days ago",
days,
days: days
)}
end
defp permitted?(user) do
with {_, %__MODULE__{inserted_at: inserted_at}} <- {:last, get_last(user)} do
days = Config.get([__MODULE__, :limit_days])
diff = Timex.diff(NaiveDateTime.utc_now(), inserted_at, :days)
nil ->
:ok
diff > days
else
{:last, nil} -> true
end
end
def get_last(user_id) do
@doc "Returns last backup for the provided user"
@spec get_last(User.t()) :: t()
def get_last(%User{id: user_id}) do
__MODULE__
|> where(user_id: ^user_id)
|> order_by(desc: :id)
@ -101,6 +138,8 @@ defmodule Pleroma.User.Backup do
|> Repo.one()
end
@doc "Lists all existing backups for a user"
@spec list(User.t()) :: [Ecto.Schema.t() | term()]
def list(%User{id: user_id}) do
__MODULE__
|> where(user_id: ^user_id)
@ -108,149 +147,113 @@ defmodule Pleroma.User.Backup do
|> Repo.all()
end
def remove_outdated(%__MODULE__{id: latest_id, user_id: user_id}) do
__MODULE__
|> where(user_id: ^user_id)
|> where([b], b.id != ^latest_id)
|> Repo.all()
|> Enum.each(&BackupWorker.delete/1)
@doc "Schedules deletion of all but the the most recent backup"
@spec remove_outdated(User.t()) :: :ok
def remove_outdated(user) do
with %__MODULE__{} = latest_backup <- get_last(user) do
__MODULE__
|> where(user_id: ^user.id)
|> where([b], b.id != ^latest_backup.id)
|> Repo.all()
|> Enum.each(&schedule_delete/1)
else
_ -> :ok
end
end
def get(id), do: Repo.get(__MODULE__, id)
defp set_state(backup, state, processed_number \\ nil) do
struct =
%{state: state}
|> Pleroma.Maps.put_if_present(:processed_number, processed_number)
def get_by_id(id), do: Repo.get(__MODULE__, id)
@doc "Generates changeset for %Pleroma.User.Backup{}"
@spec changeset(%__MODULE__{}, map()) :: %Ecto.Changeset{}
def changeset(backup \\ %__MODULE__{}, attrs) do
backup
|> cast(struct, [:state, :processed_number])
|> cast(attrs, [:content_type, :file_name, :file_size, :processed, :tempdir])
end
@doc "Updates the backup record"
@spec update_record(%__MODULE__{}, map()) :: {:ok, %__MODULE__{}} | {:error, %Ecto.Changeset{}}
def update_record(%__MODULE__{} = backup, attrs) do
backup
|> changeset(attrs)
|> Repo.update()
end
def process(
%__MODULE__{} = backup,
processor_module \\ __MODULE__.Processor
) do
set_state(backup, :running, 0)
current_pid = self()
task =
Task.Supervisor.async_nolink(
Pleroma.TaskSupervisor,
processor_module,
:do_process,
[backup, current_pid]
)
wait_backup(backup, backup.processed_number, task)
end
defp wait_backup(backup, current_processed, task) do
wait_time = @config_impl.get([__MODULE__, :process_wait_time])
receive do
{:progress, new_processed} ->
total_processed = current_processed + new_processed
set_state(backup, :running, total_processed)
wait_backup(backup, total_processed, task)
{:DOWN, _ref, _proc, _pid, reason} ->
backup = get(backup.id)
if reason != :normal do
Logger.error("Backup #{backup.id} process ended abnormally: #{inspect(reason)}")
{:ok, backup} = set_state(backup, :failed)
cleanup(backup)
{:error,
%{
backup: backup,
reason: :exit,
details: reason
}}
else
{:ok, backup}
end
after
wait_time ->
Logger.error(
"Backup #{backup.id} timed out after no response for #{wait_time}ms, terminating"
)
Task.Supervisor.terminate_child(Pleroma.TaskSupervisor, task.pid)
{:ok, backup} = set_state(backup, :failed)
cleanup(backup)
{:error,
%{
backup: backup,
reason: :timeout
}}
end
end
@files [
'actor.json',
'outbox.json',
'likes.json',
'bookmarks.json',
'followers.json',
'following.json'
~c"actor.json",
~c"outbox.json",
~c"likes.json",
~c"bookmarks.json",
~c"followers.json",
~c"following.json"
]
@spec export(Pleroma.User.Backup.t(), pid()) :: {:ok, String.t()} | :error
def export(%__MODULE__{} = backup, caller_pid) do
backup = Repo.preload(backup, :user)
dir = backup_tempdir(backup)
with :ok <- File.mkdir(dir),
:ok <- actor(dir, backup.user, caller_pid),
:ok <- statuses(dir, backup.user, caller_pid),
:ok <- likes(dir, backup.user, caller_pid),
:ok <- bookmarks(dir, backup.user, caller_pid),
:ok <- followers(dir, backup.user, caller_pid),
:ok <- following(dir, backup.user, caller_pid),
{:ok, zip_path} <- :zip.create(backup.file_name, @files, cwd: dir),
{:ok, _} <- File.rm_rf(dir) do
{:ok, zip_path}
@spec run(t()) :: {:ok, t()} | {:error, :failed}
def run(%__MODULE__{} = backup) do
backup = Repo.preload(backup, :user)
tempfile = Path.join([backup.tempdir, backup.file_name])
with {_, :ok} <- {:mkdir, File.mkdir_p(backup.tempdir)},
{_, :ok} <- {:actor, actor(backup.tempdir, backup.user)},
{_, :ok} <- {:statuses, statuses(backup.tempdir, backup.user)},
{_, :ok} <- {:likes, likes(backup.tempdir, backup.user)},
{_, :ok} <- {:bookmarks, bookmarks(backup.tempdir, backup.user)},
{_, :ok} <- {:followers, followers(backup.tempdir, backup.user)},
{_, :ok} <- {:following, following(backup.tempdir, backup.user)},
{_, {:ok, _zip_path}} <-
{:zip, :zip.create(to_charlist(tempfile), @files, cwd: to_charlist(backup.tempdir))},
{_, {:ok, %File.Stat{size: zip_size}}} <- {:filestat, File.stat(tempfile)},
{:ok, updated_backup} <- update_record(backup, %{file_size: zip_size}) do
{:ok, updated_backup}
else
_ -> :error
_ ->
File.rm_rf(backup.tempdir)
{:error, :failed}
end
end
def dir(name) do
dir = Pleroma.Config.get([__MODULE__, :dir]) || System.tmp_dir!()
Path.join(dir, name)
defp tempdir do
rand = :crypto.strong_rand_bytes(8) |> Base.url_encode64(padding: false)
subdir = "backup-#{rand}"
case Config.get([__MODULE__, :tempdir]) do
nil ->
Path.join([System.tmp_dir!(), subdir])
path ->
Path.join([path, subdir])
end
end
def upload(%__MODULE__{} = backup, zip_path) do
uploader = Pleroma.Config.get([Pleroma.Upload, :uploader])
@doc "Uploads the completed backup and marks it as processed"
@spec upload(t()) :: {:ok, t()}
def upload(%__MODULE__{tempdir: tempdir} = backup) when is_binary(tempdir) do
uploader = Config.get([Pleroma.Upload, :uploader])
upload = %Pleroma.Upload{
name: backup.file_name,
tempfile: zip_path,
tempfile: Path.join([tempdir, backup.file_name]),
content_type: backup.content_type,
path: Path.join("backups", backup.file_name)
}
with {:ok, _} <- Pleroma.Uploaders.Uploader.put_file(uploader, upload),
:ok <- File.rm(zip_path) do
{:ok, upload}
with {:ok, _} <- Uploader.put_file(uploader, upload),
{:ok, uploaded_backup} <- update_record(backup, %{processed: true}),
{:ok, _} <- File.rm_rf(tempdir) do
{:ok, uploaded_backup}
end
end
defp actor(dir, user, caller_pid) do
defp actor(dir, user) do
with {:ok, json} <-
UserView.render("user.json", %{user: user})
|> Map.merge(%{"likes" => "likes.json", "bookmarks" => "bookmarks.json"})
|> Map.merge(%{
"bookmarks" => "bookmarks.json",
"likes" => "likes.json",
"outbox" => "outbox.json",
"followers" => "followers.json",
"following" => "following.json"
})
|> Jason.encode() do
send(caller_pid, {:progress, 1})
File.write(Path.join(dir, "actor.json"), json)
end
end
@ -269,22 +272,10 @@ defmodule Pleroma.User.Backup do
)
end
defp should_report?(num, chunk_size), do: rem(num, chunk_size) == 0
defp backup_tempdir(backup) do
name = String.trim_trailing(backup.file_name, ".zip")
dir(name)
end
defp cleanup(backup) do
dir = backup_tempdir(backup)
File.rm_rf(dir)
end
defp write(query, dir, name, fun, caller_pid) do
defp write(query, dir, name, fun) do
path = Path.join(dir, "#{name}.json")
chunk_size = Pleroma.Config.get([__MODULE__, :process_chunk_size])
chunk_size = Config.get([__MODULE__, :process_chunk_size])
with {:ok, file} <- File.open(path, [:write, :utf8]),
:ok <- write_header(file, name) do
@ -300,10 +291,6 @@ defmodule Pleroma.User.Backup do
end),
{:ok, str} <- Jason.encode(data),
:ok <- IO.write(file, str <> ",\n") do
if should_report?(acc + 1, chunk_size) do
send(caller_pid, {:progress, chunk_size})
end
acc + 1
else
{:error, e} ->
@ -312,37 +299,32 @@ defmodule Pleroma.User.Backup do
)
acc
_ ->
acc
end
end)
send(caller_pid, {:progress, rem(total, chunk_size)})
with :ok <- :file.pwrite(file, {:eof, -2}, "\n],\n \"totalItems\": #{total}}") do
File.close(file)
end
end
end
defp bookmarks(dir, %{id: user_id} = _user, caller_pid) do
defp bookmarks(dir, %{id: user_id} = _user) do
Bookmark
|> where(user_id: ^user_id)
|> join(:inner, [b], activity in assoc(b, :activity))
|> select([b, a], %{id: b.id, object: fragment("(?)->>'object'", a.data)})
|> write(dir, "bookmarks", fn a -> {:ok, a.object} end, caller_pid)
|> write(dir, "bookmarks", fn a -> {:ok, a.object} end)
end
defp likes(dir, user, caller_pid) do
defp likes(dir, user) do
user.ap_id
|> Activity.Queries.by_actor()
|> Activity.Queries.by_type("Like")
|> select([like], %{id: like.id, object: fragment("(?)->>'object'", like.data)})
|> write(dir, "likes", fn a -> {:ok, a.object} end, caller_pid)
|> write(dir, "likes", fn a -> {:ok, a.object} end)
end
defp statuses(dir, user, caller_pid) do
defp statuses(dir, user) do
opts =
%{}
|> Map.put(:type, ["Create", "Announce"])
@ -362,52 +344,17 @@ defmodule Pleroma.User.Backup do
with {:ok, activity} <- Transmogrifier.prepare_outgoing(a.data) do
{:ok, Map.delete(activity, "@context")}
end
end,
caller_pid
end
)
end
defp followers(dir, user, caller_pid) do
defp followers(dir, user) do
User.get_followers_query(user)
|> write(dir, "followers", fn a -> {:ok, a.ap_id} end, caller_pid)
|> write(dir, "followers", fn a -> {:ok, a.ap_id} end)
end
defp following(dir, user, caller_pid) do
defp following(dir, user) do
User.get_friends_query(user)
|> write(dir, "following", fn a -> {:ok, a.ap_id} end, caller_pid)
end
end
defmodule Pleroma.User.Backup.ProcessorAPI do
@callback do_process(%Pleroma.User.Backup{}, pid()) ::
{:ok, %Pleroma.User.Backup{}} | {:error, any()}
end
defmodule Pleroma.User.Backup.Processor do
@behaviour Pleroma.User.Backup.ProcessorAPI
alias Pleroma.Repo
alias Pleroma.User.Backup
import Ecto.Changeset
@impl true
def do_process(backup, current_pid) do
with {:ok, zip_file} <- Backup.export(backup, current_pid),
{:ok, %{size: size}} <- File.stat(zip_file),
{:ok, _upload} <- Backup.upload(backup, zip_file) do
backup
|> cast(
%{
file_size: size,
processed: true,
state: :complete
},
[:file_size, :processed, :state]
)
|> Repo.update()
else
e -> {:error, e}
end
|> write(dir, "following", fn a -> {:ok, a.ap_id} end)
end
end

55
lib/pleroma/user/hashtag_follow.ex Normal file
View file

@ -0,0 +1,55 @@
defmodule Pleroma.User.HashtagFollow do
use Ecto.Schema
import Ecto.Query
import Ecto.Changeset
alias Pleroma.Hashtag
alias Pleroma.Repo
alias Pleroma.User
schema "user_follows_hashtag" do
belongs_to(:user, User, type: FlakeId.Ecto.CompatType)
belongs_to(:hashtag, Hashtag)
end
def changeset(%__MODULE__{} = user_hashtag_follow, attrs) do
user_hashtag_follow
|> cast(attrs, [:user_id, :hashtag_id])
|> unique_constraint(:hashtag_id,
name: :user_hashtag_follows_user_id_hashtag_id_index,
message: "already following"
)
|> validate_required([:user_id, :hashtag_id])
end
def new(%User{} = user, %Hashtag{} = hashtag) do
%__MODULE__{}
|> changeset(%{user_id: user.id, hashtag_id: hashtag.id})
|> Repo.insert(on_conflict: :nothing)
end
def delete(%User{} = user, %Hashtag{} = hashtag) do
with %__MODULE__{} = user_hashtag_follow <- get(user, hashtag) do
Repo.delete(user_hashtag_follow)
else
_ -> {:ok, nil}
end
end
def get(%User{} = user, %Hashtag{} = hashtag) do
from(hf in __MODULE__)
|> where([hf], hf.user_id == ^user.id and hf.hashtag_id == ^hashtag.id)
|> Repo.one()
end
def get_by_user(%User{} = user) do
user
|> followed_hashtags_query()
|> Repo.all()
end
def followed_hashtags_query(%User{} = user) do
Ecto.assoc(user, :followed_hashtags)
|> Ecto.Query.order_by([h], desc: h.id)
end
end

138
lib/pleroma/user/import.ex
View file

@ -5,81 +5,107 @@
defmodule Pleroma.User.Import do
use Ecto.Schema
alias Pleroma.Repo
alias Pleroma.User
alias Pleroma.Web.CommonAPI
alias Pleroma.Workers.BackgroundWorker
require Logger
@spec perform(atom(), User.t(), list()) :: :ok | list() | {:error, any()}
def perform(:mutes_import, %User{} = user, [_ | _] = identifiers) do
Enum.map(
identifiers,
fn identifier ->
with {:ok, %User{} = muted_user} <- User.get_or_fetch(identifier),
{:ok, _} <- User.mute(user, muted_user) do
muted_user
else
error -> handle_error(:mutes_import, identifier, error)
end
end
)
@spec perform(atom(), User.t(), String.t()) :: :ok | {:error, any()}
def perform(:mute_import, %User{} = user, actor) do
with {:ok, %User{} = muted_user} <- User.get_or_fetch(actor),
{_, false} <- {:existing_mute, User.mutes_user?(user, muted_user)},
{:ok, _} <- User.mute(user, muted_user) do
{:ok, muted_user}
else
{:existing_mute, true} -> :ok
error -> handle_error(:mutes_import, actor, error)
end
end
def perform(:blocks_import, %User{} = blocker, [_ | _] = identifiers) do
Enum.map(
identifiers,
fn identifier ->
with {:ok, %User{} = blocked} <- User.get_or_fetch(identifier),
{:ok, _block} <- CommonAPI.block(blocker, blocked) do
blocked
else
error -> handle_error(:blocks_import, identifier, error)
end
end
)
def perform(:block_import, %User{} = user, actor) do
with {:ok, %User{} = blocked} <- User.get_or_fetch(actor),
{_, false} <- {:existing_block, User.blocks_user?(user, blocked)},
{:ok, _block} <- CommonAPI.block(blocked, user) do
{:ok, blocked}
else
{:existing_block, true} -> :ok
error -> handle_error(:blocks_import, actor, error)
end
end
def perform(:follow_import, %User{} = follower, [_ | _] = identifiers) do
Enum.map(
identifiers,
fn identifier ->
with {:ok, %User{} = followed} <- User.get_or_fetch(identifier),
{:ok, follower, followed} <- User.maybe_direct_follow(follower, followed),
{:ok, _, _, _} <- CommonAPI.follow(follower, followed) do
followed
else
error -> handle_error(:follow_import, identifier, error)
end
end
)
def perform(:follow_import, %User{} = user, actor) do
with {:ok, %User{} = followed} <- User.get_or_fetch(actor),
{_, false} <- {:existing_follow, User.following?(user, followed)},
{:ok, user, followed} <- User.maybe_direct_follow(user, followed),
{:ok, _, _, _} <- CommonAPI.follow(followed, user) do
{:ok, followed}
else
{:existing_follow, true} -> :ok
error -> handle_error(:follow_import, actor, error)
end
end
def perform(_, _, _), do: :ok
defp handle_error(op, user_id, error) do
Logger.debug("#{op} failed for #{user_id} with: #{inspect(error)}")
error
{:error, error}
end
def blocks_import(%User{} = blocker, [_ | _] = identifiers) do
BackgroundWorker.enqueue(
"blocks_import",
%{"user_id" => blocker.id, "identifiers" => identifiers}
)
def blocks_import(%User{} = user, [_ | _] = actors) do
jobs =
Repo.checkout(fn ->
Enum.reduce(actors, [], fn actor, acc ->
{:ok, job} =
BackgroundWorker.new(%{
"op" => "block_import",
"user_id" => user.id,
"actor" => actor
})
|> Oban.insert()
acc ++ [job]
end)
end)
{:ok, jobs}
end
def follow_import(%User{} = follower, [_ | _] = identifiers) do
BackgroundWorker.enqueue(
"follow_import",
%{"user_id" => follower.id, "identifiers" => identifiers}
)
def follows_import(%User{} = user, [_ | _] = actors) do
jobs =
Repo.checkout(fn ->
Enum.reduce(actors, [], fn actor, acc ->
{:ok, job} =
BackgroundWorker.new(%{
"op" => "follow_import",
"user_id" => user.id,
"actor" => actor
})
|> Oban.insert()
acc ++ [job]
end)
end)
{:ok, jobs}
end
def mutes_import(%User{} = user, [_ | _] = identifiers) do
BackgroundWorker.enqueue(
"mutes_import",
%{"user_id" => user.id, "identifiers" => identifiers}
)
def mutes_import(%User{} = user, [_ | _] = actors) do
jobs =
Repo.checkout(fn ->
Enum.reduce(actors, [], fn actor, acc ->
{:ok, job} =
BackgroundWorker.new(%{
"op" => "mute_import",
"user_id" => user.id,
"actor" => actor
})
|> Oban.insert()
acc ++ [job]
end)
end)
{:ok, jobs}
end
end

2
lib/pleroma/web.ex
View file

@ -163,7 +163,7 @@ defmodule Pleroma.Web do
"""
def safe_render_many(collection, view, template, assigns \\ %{}) do
Enum.map(collection, fn resource ->
as = Map.get(assigns, :as) || view.__resource__
as = Map.get(assigns, :as) || view.__resource__()
assigns = Map.put(assigns, as, resource)
safe_render(view, template, assigns)
end)

85
lib/pleroma/web/activity_pub/activity_pub.ex
View file

@ -201,7 +201,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
def notify_and_stream(activity) do
{:ok, notifications} = Notification.create_notifications(activity)
Notification.send(notifications)
Notification.stream(notifications)
original_activity =
case activity do
@ -222,10 +222,12 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
%{data: %{"expires_at" => %DateTime{} = expires_at}} = activity
) do
with {:ok, _job} <-
Pleroma.Workers.PurgeExpiredActivity.enqueue(%{
activity_id: activity.id,
expires_at: expires_at
}) do
Pleroma.Workers.PurgeExpiredActivity.enqueue(
%{
activity_id: activity.id
},
scheduled_at: expires_at
) do
{:ok, activity}
end
end
@ -446,10 +448,12 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
_ <- notify_and_stream(activity) do
maybe_federate(activity)
BackgroundWorker.enqueue("move_following", %{
BackgroundWorker.new(%{
"op" => "move_following",
"origin_id" => origin.id,
"target_id" => target.id
})
|> Oban.insert()
{:ok, activity}
else
@ -920,6 +924,31 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
)
end
# Essentially, either look for activities addressed to `recipients`, _OR_ ones
# that reference a hashtag that the user follows
# Firstly, two fallbacks in case there's no hashtag constraint, or the user doesn't
# follow any
defp restrict_recipients_or_hashtags(query, recipients, user, nil) do
restrict_recipients(query, recipients, user)
end
defp restrict_recipients_or_hashtags(query, recipients, user, []) do
restrict_recipients(query, recipients, user)
end
defp restrict_recipients_or_hashtags(query, recipients, _user, hashtag_ids) do
from([activity, object] in query)
|> join(:left, [activity, object], hto in "hashtags_objects",
on: hto.object_id == object.id,
as: :hto
)
|> where(
[activity, object, hto: hto],
(hto.hashtag_id in ^hashtag_ids and ^Constants.as_public() in activity.recipients) or
fragment("? && ?", ^recipients, activity.recipients)
)
end
defp restrict_local(query, %{local_only: true}) do
from(activity in query, where: activity.local == true)
end
@ -979,8 +1008,9 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
defp restrict_replies(query, %{exclude_replies: true}) do
from(
[_activity, object] in query,
where: fragment("?->>'inReplyTo' is null", object.data)
[activity, object] in query,
where:
fragment("?->>'inReplyTo' is null or ?->>'type' = 'Announce'", object.data, activity.data)
)
end
@ -1409,7 +1439,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
|> maybe_preload_report_notes(opts)
|> maybe_set_thread_muted_field(opts)
|> maybe_order(opts)
|> restrict_recipients(recipients, opts[:user])
|> restrict_recipients_or_hashtags(recipients, opts[:user], opts[:followed_hashtags])
|> restrict_replies(opts)
|> restrict_since(opts)
|> restrict_local(opts)
@ -1537,16 +1567,23 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
defp get_actor_url(_url), do: nil
defp normalize_image(%{"url" => url}) do
defp normalize_image(%{"url" => url} = data) do
%{
"type" => "Image",
"url" => [%{"href" => url}]
}
|> maybe_put_description(data)
end
defp normalize_image(urls) when is_list(urls), do: urls |> List.first() |> normalize_image()
defp normalize_image(_), do: nil
defp maybe_put_description(map, %{"name" => description}) when is_binary(description) do
Map.put(map, "name", description)
end
defp maybe_put_description(map, _), do: map
defp object_to_user_data(data, additional) do
fields =
data
@ -1660,7 +1697,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
}}
else
{:error, _} = e -> e
e -> {:error, e}
end
end
@ -1793,24 +1829,27 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
end
end
def pinned_fetch_task(nil), do: nil
def pinned_fetch_task(%{pinned_objects: pins}) do
if Enum.all?(pins, fn {ap_id, _} ->
Object.get_cached_by_ap_id(ap_id) ||
match?({:ok, _object}, Fetcher.fetch_object_from_id(ap_id))
end) do
:ok
else
:error
end
def enqueue_pin_fetches(%{pinned_objects: pins}) do
# enqueue a task to fetch all pinned objects
Enum.each(pins, fn {ap_id, _} ->
if is_nil(Object.get_cached_by_ap_id(ap_id)) do
Pleroma.Workers.RemoteFetcherWorker.new(%{
"op" => "fetch_remote",
"id" => ap_id,
"depth" => 1
})
|> Oban.insert()
end
end)
end
def enqueue_pin_fetches(_), do: nil
def make_user_from_ap_id(ap_id, additional \\ []) do
user = User.get_cached_by_ap_id(ap_id)
with {:ok, data} <- fetch_and_prepare_user_from_ap_id(ap_id, additional) do
{:ok, _pid} = Task.start(fn -> pinned_fetch_task(data) end)
enqueue_pin_fetches(data)
if user do
user

23
lib/pleroma/web/activity_pub/activity_pub_controller.ex
View file

@ -52,6 +52,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
when action in [:activity, :object]
)
plug(:log_inbox_metadata when action in [:inbox])
plug(:set_requester_reachable when action in [:inbox])
plug(:relay_active? when action in [:relay])
@ -292,8 +293,15 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
json(conn, "ok")
end
def inbox(%{assigns: %{valid_signature: false}, req_headers: req_headers} = conn, params) do
Federator.incoming_ap_doc(%{req_headers: req_headers, params: params})
def inbox(%{assigns: %{valid_signature: false}} = conn, params) do
Federator.incoming_ap_doc(%{
method: conn.method,
req_headers: conn.req_headers,
request_path: conn.request_path,
params: params,
query_string: conn.query_string
})
json(conn, "ok")
end
@ -303,7 +311,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
post_inbox_relayed_create(conn, params)
else
conn
|> put_status(:bad_request)
|> put_status(403)
|> json("Not federating")
end
end
@ -474,7 +482,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
|> put_status(:forbidden)
|> json(message)
{:error, message} ->
{:error, message} when is_binary(message) ->
conn
|> put_status(:bad_request)
|> json(message)
@ -521,6 +529,13 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
conn
end
defp log_inbox_metadata(%{params: %{"actor" => actor, "type" => type}} = conn, _) do
Logger.metadata(actor: actor, type: type)
conn
end
defp log_inbox_metadata(conn, _), do: conn
def upload_media(%{assigns: %{user: %User{} = user}} = conn, %{"file" => file} = data) do
with {:ok, object} <-
ActivityPub.upload(

10
lib/pleroma/web/activity_pub/mrf.ex
View file

@ -108,6 +108,14 @@ defmodule Pleroma.Web.ActivityPub.MRF do
def filter(%{} = object), do: get_policies() |> filter(object)
def id_filter(policies, id) when is_binary(id) do
policies
|> Enum.filter(&function_exported?(&1, :id_filter, 1))
|> Enum.all?(& &1.id_filter(id))
end
def id_filter(id) when is_binary(id), do: get_policies() |> id_filter(id)
@impl true
def pipeline_filter(%{} = message, meta) do
object = meta[:object_data]
@ -204,7 +212,7 @@ defmodule Pleroma.Web.ActivityPub.MRF do
if function_exported?(policy, :config_description, 0) do
description =
@default_description
|> Map.merge(policy.config_description)
|> Map.merge(policy.config_description())
|> Map.put(:group, :pleroma)
|> Map.put(:tab, :mrf)
|> Map.put(:type, :group)

8
lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex
View file

@ -63,20 +63,20 @@ defmodule Pleroma.Web.ActivityPub.MRF.AntiFollowbotPolicy do
end
@impl true
def filter(%{"type" => "Follow", "actor" => actor_id} = message) do
def filter(%{"type" => "Follow", "actor" => actor_id} = activity) do
%User{} = actor = normalize_by_ap_id(actor_id)
score = determine_if_followbot(actor)
if score < 0.8 || bot_allowed?(message, actor) do
{:ok, message}
if score < 0.8 || bot_allowed?(activity, actor) do
{:ok, activity}
else
{:reject, "[AntiFollowbotPolicy] Scored #{actor_id} as #{score}"}
end
end
@impl true
def filter(message), do: {:ok, message}
def filter(activity), do: {:ok, activity}
@impl true
def describe, do: {:ok, %{}}

10
lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex
View file

@ -29,17 +29,17 @@ defmodule Pleroma.Web.ActivityPub.MRF.AntiLinkSpamPolicy do
defp contains_links?(_), do: false
@impl true
def filter(%{"type" => "Create", "actor" => actor, "object" => object} = message) do
def filter(%{"type" => "Create", "actor" => actor, "object" => object} = activity) do
with {:ok, %User{local: false} = u} <- User.get_or_fetch_by_ap_id(actor),
{:contains_links, true} <- {:contains_links, contains_links?(object)},
{:old_user, true} <- {:old_user, old_user?(u)} do
{:ok, message}
{:ok, activity}
else
{:ok, %User{local: true}} ->
{:ok, message}
{:ok, activity}
{:contains_links, false} ->
{:ok, message}
{:ok, activity}
{:old_user, false} ->
{:reject, "[AntiLinkSpamPolicy] User has no posts nor followers"}
@ -53,7 +53,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.AntiLinkSpamPolicy do
end
# in all other cases, pass through
def filter(message), do: {:ok, message}
def filter(activity), do: {:ok, activity}
@impl true
def describe, do: {:ok, %{}}

87
lib/pleroma/web/activity_pub/mrf/anti_mention_spam_policy.ex Normal file
View file

@ -0,0 +1,87 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.ActivityPub.MRF.AntiMentionSpamPolicy do
alias Pleroma.Config
alias Pleroma.User
require Pleroma.Constants
@behaviour Pleroma.Web.ActivityPub.MRF.Policy
defp user_has_posted?(%User{} = u), do: u.note_count > 0
defp user_has_age?(%User{} = u) do
user_age_limit = Config.get([:mrf_antimentionspam, :user_age_limit], 30_000)
diff = NaiveDateTime.utc_now() |> NaiveDateTime.diff(u.inserted_at, :millisecond)
diff >= user_age_limit
end
defp good_reputation?(%User{} = u) do
user_has_age?(u) and user_has_posted?(u)
end
# copied from HellthreadPolicy
defp get_recipient_count(activity) do
recipients = (activity["to"] || []) ++ (activity["cc"] || [])
follower_collection =
User.get_cached_by_ap_id(activity["actor"] || activity["attributedTo"]).follower_address
if Enum.member?(recipients, Pleroma.Constants.as_public()) do
recipients =
recipients
|> List.delete(Pleroma.Constants.as_public())
|> List.delete(follower_collection)
{:public, length(recipients)}
else
recipients =
recipients
|> List.delete(follower_collection)
{:not_public, length(recipients)}
end
end
defp object_has_recipients?(%{"object" => object} = activity) do
{_, object_count} = get_recipient_count(object)
{_, activity_count} = get_recipient_count(activity)
object_count + activity_count > 0
end
defp object_has_recipients?(object) do
{_, count} = get_recipient_count(object)
count > 0
end
@impl true
def filter(%{"type" => "Create", "actor" => actor} = activity) do
with {:ok, %User{local: false} = u} <- User.get_or_fetch_by_ap_id(actor),
{:has_mentions, true} <- {:has_mentions, object_has_recipients?(activity)},
{:good_reputation, true} <- {:good_reputation, good_reputation?(u)} do
{:ok, activity}
else
{:ok, %User{local: true}} ->
{:ok, activity}
{:has_mentions, false} ->
{:ok, activity}
{:good_reputation, false} ->
{:reject, "[AntiMentionSpamPolicy] User rejected"}
{:error, _} ->
{:reject, "[AntiMentionSpamPolicy] Failed to get or fetch user by ap_id"}
e ->
{:reject, "[AntiMentionSpamPolicy] Unhandled error #{inspect(e)}"}
end
end
# in all other cases, pass through
def filter(activity), do: {:ok, activity}
@impl true
def describe, do: {:ok, %{}}
end

146
lib/pleroma/web/activity_pub/mrf/dnsrbl_policy.ex Normal file
View file

@ -0,0 +1,146 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2024 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.ActivityPub.MRF.DNSRBLPolicy do
@moduledoc """
Dynamic activity filtering based on an RBL database
This MRF makes queries to a custom DNS server which will
respond with values indicating the classification of the domain
the activity originated from. This method has been widely used
in the email anti-spam industry for very fast reputation checks.
e.g., if the DNS response is 127.0.0.1 or empty, the domain is OK
Other values such as 127.0.0.2 may be used for specific classifications.
Information for why the host is blocked can be stored in a corresponding TXT record.
This method is fail-open so if the queries fail the activites are accepted.
An example of software meant for this purpsoe is rbldnsd which can be found
at http://www.corpit.ru/mjt/rbldnsd.html or mirrored at
https://git.pleroma.social/feld/rbldnsd
It is highly recommended that you run your own copy of rbldnsd and use an
external mechanism to sync/share the contents of the zone file. This is
important to keep the latency on the queries as low as possible and prevent
your DNS server from being attacked so it fails and content is permitted.
"""
@behaviour Pleroma.Web.ActivityPub.MRF.Policy
alias Pleroma.Config
require Logger
@query_retries 1
@query_timeout 500
@impl true
def filter(%{"actor" => actor} = activity) do
actor_info = URI.parse(actor)
with {:ok, activity} <- check_rbl(actor_info, activity) do
{:ok, activity}
else
_ -> {:reject, "[DNSRBLPolicy]"}
end
end
@impl true
def filter(activity), do: {:ok, activity}
@impl true
def describe do
mrf_dnsrbl =
Config.get(:mrf_dnsrbl)
|> Enum.into(%{})
{:ok, %{mrf_dnsrbl: mrf_dnsrbl}}
end
@impl true
def config_description do
%{
key: :mrf_dnsrbl,
related_policy: "Pleroma.Web.ActivityPub.MRF.DNSRBLPolicy",
label: "MRF DNSRBL",
description: "DNS RealTime Blackhole Policy",
children: [
%{
key: :nameserver,
type: {:string},
description: "DNSRBL Nameserver to Query (IP or hostame)",
suggestions: ["127.0.0.1"]
},
%{
key: :port,
type: {:string},
description: "Nameserver port",
suggestions: ["53"]
},
%{
key: :zone,
type: {:string},
description: "Root zone for querying",
suggestions: ["bl.pleroma.com"]
}
]
}
end
defp check_rbl(%{host: actor_host}, activity) do
with false <- match?(^actor_host, Pleroma.Web.Endpoint.host()),
zone when not is_nil(zone) <- Keyword.get(Config.get([:mrf_dnsrbl]), :zone) do
query =
Enum.join([actor_host, zone], ".")
|> String.to_charlist()
rbl_response = rblquery(query)
if Enum.empty?(rbl_response) do
{:ok, activity}
else
Task.start(fn ->
reason =
case rblquery(query, :txt) do
[[result]] -> result
_ -> "undefined"
end
Logger.warning(
"DNSRBL Rejected activity from #{actor_host} for reason: #{inspect(reason)}"
)
end)
:error
end
else
_ -> {:ok, activity}
end
end
defp get_rblhost_ip(rblhost) do
case rblhost |> String.to_charlist() |> :inet_parse.address() do
{:ok, _} -> rblhost |> String.to_charlist() |> :inet_parse.address()
_ -> {:ok, rblhost |> String.to_charlist() |> :inet_res.lookup(:in, :a) |> Enum.random()}
end
end
defp rblquery(query, type \\ :a) do
config = Config.get([:mrf_dnsrbl])
case get_rblhost_ip(config[:nameserver]) do
{:ok, rblnsip} ->
:inet_res.lookup(query, :in, type,
nameservers: [{rblnsip, config[:port]}],
timeout: @query_timeout,
retry: @query_retries
)
_ ->
[]
end
end
end

12
lib/pleroma/web/activity_pub/mrf/drop_policy.ex
View file

@ -8,9 +8,15 @@ defmodule Pleroma.Web.ActivityPub.MRF.DropPolicy do
@behaviour Pleroma.Web.ActivityPub.MRF.Policy
@impl true
def filter(object) do
Logger.debug("REJECTING #{inspect(object)}")
{:reject, object}
def filter(activity) do
Logger.debug("REJECTING #{inspect(activity)}")
{:reject, activity}
end
@impl true
def id_filter(id) do
Logger.debug("REJECTING #{id}")
false
end
@impl true

30
lib/pleroma/web/activity_pub/mrf/emoji_policy.ex
View file

@ -28,11 +28,11 @@ defmodule Pleroma.Web.ActivityPub.MRF.EmojiPolicy do
Pleroma.Config.get([:mrf_emoji, :federated_timeline_removal_shortcode], [])
end
@impl Pleroma.Web.ActivityPub.MRF.Policy
@impl true
def history_awareness, do: :manual
@impl Pleroma.Web.ActivityPub.MRF.Policy
def filter(%{"type" => type, "object" => %{"type" => objtype} = object} = message)
@impl true
def filter(%{"type" => type, "object" => %{"type" => objtype} = object} = activity)
when type in ["Create", "Update"] and objtype in Pleroma.Constants.status_object_types() do
with {:ok, object} <-
Updater.do_with_history(object, fn object ->
@ -42,13 +42,13 @@ defmodule Pleroma.Web.ActivityPub.MRF.EmojiPolicy do
Updater.do_with_history(object, fn object ->
{:ok, process_remove(object, :shortcode, config_remove_shortcode())}
end),
activity <- Map.put(message, "object", object),
activity <- Map.put(activity, "object", object),
activity <- maybe_delist(activity) do
{:ok, activity}
end
end
@impl Pleroma.Web.ActivityPub.MRF.Policy
@impl true
def filter(%{"type" => type} = object) when type in Pleroma.Constants.actor_types() do
with object <- process_remove(object, :url, config_remove_url()),
object <- process_remove(object, :shortcode, config_remove_shortcode()) do
@ -56,7 +56,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.EmojiPolicy do
end
end
@impl Pleroma.Web.ActivityPub.MRF.Policy
@impl true
def filter(%{"type" => "EmojiReact"} = object) do
with {:ok, _} <-
matched_emoji_checker(config_remove_url(), config_remove_shortcode()).(object) do
@ -67,9 +67,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.EmojiPolicy do
end
end
@impl Pleroma.Web.ActivityPub.MRF.Policy
def filter(message) do
{:ok, message}
@impl true
def filter(activity) do
{:ok, activity}
end
defp match_string?(string, pattern) when is_binary(pattern) do
@ -214,7 +214,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.EmojiPolicy do
)
end
@impl Pleroma.Web.ActivityPub.MRF.Policy
@impl true
def describe do
mrf_emoji =
Pleroma.Config.get(:mrf_emoji, [])
@ -226,7 +226,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.EmojiPolicy do
{:ok, %{mrf_emoji: mrf_emoji}}
end
@impl Pleroma.Web.ActivityPub.MRF.Policy
@impl true
def config_description do
%{
key: :mrf_emoji,
@ -239,7 +239,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.EmojiPolicy do
key: :remove_url,
type: {:list, :string},
description: """
A list of patterns which result in emoji whose URL matches being removed from the message. This will apply to statuses, emoji reactions, and user profiles.
A list of patterns which result in emoji whose URL matches being removed from the activity. This will apply to statuses, emoji reactions, and user profiles.
Each pattern can be a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`.
""",
@ -249,7 +249,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.EmojiPolicy do
key: :remove_shortcode,
type: {:list, :string},
description: """
A list of patterns which result in emoji whose shortcode matches being removed from the message. This will apply to statuses, emoji reactions, and user profiles.
A list of patterns which result in emoji whose shortcode matches being removed from the activity. This will apply to statuses, emoji reactions, and user profiles.
Each pattern can be a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`.
""",
@ -259,7 +259,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.EmojiPolicy do
key: :federated_timeline_removal_url,
type: {:list, :string},
description: """
A list of patterns which result in message with emojis whose URLs match being removed from federated timelines (a.k.a unlisted). This will apply only to statuses.
A list of patterns which result in activity with emojis whose URLs match being removed from federated timelines (a.k.a unlisted). This will apply only to statuses.
Each pattern can be a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`.
""",
@ -269,7 +269,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.EmojiPolicy do
key: :federated_timeline_removal_shortcode,
type: {:list, :string},
description: """
A list of patterns which result in message with emojis whose shortcodes match being removed from federated timelines (a.k.a unlisted). This will apply only to statuses.
A list of patterns which result in activities with emojis whose shortcodes match being removed from federated timelines (a.k.a unlisted). This will apply only to statuses.
Each pattern can be a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`.
""",

14
lib/pleroma/web/activity_pub/mrf/ensure_re_prepended.ex
View file

@ -29,19 +29,19 @@ defmodule Pleroma.Web.ActivityPub.MRF.EnsureRePrepended do
def filter_by_summary(_in_reply_to, child), do: child
def filter(%{"type" => type, "object" => child_object} = object)
when type in ["Create", "Update"] and is_map(child_object) do
def filter(%{"type" => type, "object" => object} = activity)
when type in ["Create", "Update"] and is_map(object) do
child =
child_object["inReplyTo"]
object["inReplyTo"]
|> Object.normalize(fetch: false)
|> filter_by_summary(child_object)
|> filter_by_summary(object)
object = Map.put(object, "object", child)
activity = Map.put(activity, "object", child)
{:ok, object}
{:ok, activity}
end
def filter(object), do: {:ok, object}
def filter(activity), do: {:ok, activity}
def describe, do: {:ok, %{}}
end

53
lib/pleroma/web/activity_pub/mrf/fo_direct_reply.ex Normal file
View file

@ -0,0 +1,53 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2024 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.ActivityPub.MRF.FODirectReply do
@moduledoc """
FODirectReply alters the scope of replies to activities which are Followers Only to be Direct. The purpose of this policy is to prevent broken threads for followers of the reply author because their response was to a user that they are not also following.
"""
alias Pleroma.Object
alias Pleroma.User
alias Pleroma.Web.ActivityPub.Visibility
@behaviour Pleroma.Web.ActivityPub.MRF.Policy
@impl true
def filter(
%{
"type" => "Create",
"to" => to,
"object" => %{
"actor" => actor,
"type" => "Note",
"inReplyTo" => in_reply_to
}
} = activity
) do
with true <- is_binary(in_reply_to),
%User{follower_address: followers_collection, local: true} <- User.get_by_ap_id(actor),
%Object{} = in_reply_to_object <- Object.get_by_ap_id(in_reply_to),
"private" <- Visibility.get_visibility(in_reply_to_object) do
direct_to = to -- [followers_collection]
updated_activity =
activity
|> Map.put("cc", [])
|> Map.put("to", direct_to)
|> Map.put("directMessage", true)
|> put_in(["object", "cc"], [])
|> put_in(["object", "to"], direct_to)
{:ok, updated_activity}
else
_ -> {:ok, activity}
end
end
@impl true
def filter(activity), do: {:ok, activity}
@impl true
def describe, do: {:ok, %{}}
end

22
lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex
View file

@ -11,12 +11,12 @@ defmodule Pleroma.Web.ActivityPub.MRF.FollowBotPolicy do
require Logger
@impl true
def filter(message) do
def filter(activity) do
with follower_nickname <- Config.get([:mrf_follow_bot, :follower_nickname]),
%User{actor_type: "Service"} = follower <-
User.get_cached_by_nickname(follower_nickname),
%{"type" => "Create", "object" => %{"type" => "Note"}} <- message do
try_follow(follower, message)
%{"type" => "Create", "object" => %{"type" => "Note"}} <- activity do
try_follow(follower, activity)
else
nil ->
Logger.warning(
@ -24,17 +24,17 @@ defmodule Pleroma.Web.ActivityPub.MRF.FollowBotPolicy do
account does not exist, or the account is not correctly configured as a bot."
)
{:ok, message}
{:ok, activity}
_ ->
{:ok, message}
{:ok, activity}
end
end
defp try_follow(follower, message) do
to = Map.get(message, "to", [])
cc = Map.get(message, "cc", [])
actor = [message["actor"]]
defp try_follow(follower, activity) do
to = Map.get(activity, "to", [])
cc = Map.get(activity, "cc", [])
actor = [activity["actor"]]
Enum.concat([to, cc, actor])
|> List.flatten()
@ -49,11 +49,11 @@ defmodule Pleroma.Web.ActivityPub.MRF.FollowBotPolicy do
"#{__MODULE__}: Follow request from #{follower.nickname} to #{user.nickname}"
)
CommonAPI.follow(follower, user)
CommonAPI.follow(user, follower)
end
end)
{:ok, message}
{:ok, activity}
end
@impl true

12
lib/pleroma/web/activity_pub/mrf/force_bot_unlisted_policy.ex
View file

@ -22,7 +22,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.ForceBotUnlistedPolicy do
"cc" => cc,
"actor" => actor,
"object" => object
} = message
} = activity
) do
user = User.get_cached_by_ap_id(actor)
isbot = check_if_bot(user)
@ -36,20 +36,20 @@ defmodule Pleroma.Web.ActivityPub.MRF.ForceBotUnlistedPolicy do
|> Map.put("to", to)
|> Map.put("cc", cc)
message =
message
activity =
activity
|> Map.put("to", to)
|> Map.put("cc", cc)
|> Map.put("object", object)
{:ok, message}
{:ok, activity}
else
{:ok, message}
{:ok, activity}
end
end
@impl true
def filter(message), do: {:ok, message}
def filter(activity), do: {:ok, activity}
@impl true
def describe, do: {:ok, %{}}

2
lib/pleroma/web/activity_pub/mrf/force_mention.ex
View file

@ -52,7 +52,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.ForceMention do
end
@impl true
def filter(object), do: {:ok, object}
def filter(activity), do: {:ok, activity}
@impl true
def describe, do: {:ok, %{}}

12
lib/pleroma/web/activity_pub/mrf/force_mentions_in_content.ex
View file

@ -79,18 +79,18 @@ defmodule Pleroma.Web.ActivityPub.MRF.ForceMentionsInContent do
%{
"type" => type,
"object" => %{"type" => "Note", "to" => to, "inReplyTo" => in_reply_to}
} = object
} = activity
)
when type in ["Create", "Update"] and is_list(to) and is_binary(in_reply_to) do
# image-only posts from pleroma apparently reach this MRF without the content field
content = object["object"]["content"] || ""
content = activity["object"]["content"] || ""
# Get the replied-to user for sorting
replied_to_user = get_replied_to_user(object["object"])
replied_to_user = get_replied_to_user(activity["object"])
mention_users =
to
|> clean_recipients(object)
|> clean_recipients(activity)
|> Enum.map(&User.get_cached_by_ap_id/1)
|> Enum.reject(&is_nil/1)
|> sort_replied_user(replied_to_user)
@ -126,11 +126,11 @@ defmodule Pleroma.Web.ActivityPub.MRF.ForceMentionsInContent do
content
end
{:ok, put_in(object["object"]["content"], content)}
{:ok, put_in(activity["object"]["content"], content)}
end
@impl true
def filter(object), do: {:ok, object}
def filter(activity), do: {:ok, activity}
@impl true
def describe, do: {:ok, %{}}

51
lib/pleroma/web/activity_pub/mrf/hashtag_policy.ex
View file

@ -9,7 +9,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.HashtagPolicy do
alias Pleroma.Object
@moduledoc """
Reject, TWKN-remove or Set-Sensitive messages with specific hashtags (without the leading #)
Reject, TWKN-remove or Set-Sensitive activities with specific hashtags (without the leading #)
Note: This MRF Policy is always enabled, if you want to disable it you have to set empty lists.
"""
@ -19,40 +19,40 @@ defmodule Pleroma.Web.ActivityPub.MRF.HashtagPolicy do
@impl true
def history_awareness, do: :manual
defp check_reject(message, hashtags) do
defp check_reject(activity, hashtags) do
if Enum.any?(Config.get([:mrf_hashtag, :reject]), fn match -> match in hashtags end) do
{:reject, "[HashtagPolicy] Matches with rejected keyword"}
else
{:ok, message}
{:ok, activity}
end
end
defp check_ftl_removal(%{"to" => to} = message, hashtags) do
defp check_ftl_removal(%{"to" => to} = activity, hashtags) do
if Pleroma.Constants.as_public() in to and
Enum.any?(Config.get([:mrf_hashtag, :federated_timeline_removal]), fn match ->
match in hashtags
end) do
to = List.delete(to, Pleroma.Constants.as_public())
cc = [Pleroma.Constants.as_public() | message["cc"] || []]
cc = [Pleroma.Constants.as_public() | activity["cc"] || []]
message =
message
activity =
activity
|> Map.put("to", to)
|> Map.put("cc", cc)
|> Kernel.put_in(["object", "to"], to)
|> Kernel.put_in(["object", "cc"], cc)
{:ok, message}
{:ok, activity}
else
{:ok, message}
{:ok, activity}
end
end
defp check_ftl_removal(message, _hashtags), do: {:ok, message}
defp check_ftl_removal(activity, _hashtags), do: {:ok, activity}
defp check_sensitive(message) do
defp check_sensitive(activity) do
{:ok, new_object} =
Object.Updater.do_with_history(message["object"], fn object ->
Object.Updater.do_with_history(activity["object"], fn object ->
hashtags = Object.hashtags(%Object{data: object})
if Enum.any?(Config.get([:mrf_hashtag, :sensitive]), fn match -> match in hashtags end) do
@ -62,11 +62,12 @@ defmodule Pleroma.Web.ActivityPub.MRF.HashtagPolicy do
end
end)
{:ok, Map.put(message, "object", new_object)}
{:ok, Map.put(activity, "object", new_object)}
end
@impl true
def filter(%{"type" => type, "object" => object} = message) when type in ["Create", "Update"] do
def filter(%{"type" => type, "object" => object} = activity)
when type in ["Create", "Update"] do
history_items =
with %{"formerRepresentations" => %{"orderedItems" => items}} <- object do
items
@ -82,23 +83,23 @@ defmodule Pleroma.Web.ActivityPub.MRF.HashtagPolicy do
hashtags = Object.hashtags(%Object{data: object}) ++ historical_hashtags
if hashtags != [] do
with {:ok, message} <- check_reject(message, hashtags),
{:ok, message} <-
with {:ok, activity} <- check_reject(activity, hashtags),
{:ok, activity} <-
(if type == "Create" do
check_ftl_removal(message, hashtags)
check_ftl_removal(activity, hashtags)
else
{:ok, message}
{:ok, activity}
end),
{:ok, message} <- check_sensitive(message) do
{:ok, message}
{:ok, activity} <- check_sensitive(activity) do
{:ok, activity}
end
else
{:ok, message}
{:ok, activity}
end
end
@impl true
def filter(message), do: {:ok, message}
def filter(activity), do: {:ok, activity}
@impl true
def describe do
@ -120,21 +121,21 @@ defmodule Pleroma.Web.ActivityPub.MRF.HashtagPolicy do
%{
key: :reject,
type: {:list, :string},
description: "A list of hashtags which result in message being rejected.",
description: "A list of hashtags which result in the activity being rejected.",
suggestions: ["foo"]
},
%{
key: :federated_timeline_removal,
type: {:list, :string},
description:
"A list of hashtags which result in message being removed from federated timelines (a.k.a unlisted).",
"A list of hashtags which result in the activity being removed from federated timelines (a.k.a unlisted).",
suggestions: ["foo"]
},
%{
key: :sensitive,
type: {:list, :string},
description:
"A list of hashtags which result in message being set as sensitive (a.k.a NSFW/R-18)",
"A list of hashtags which result in the activity being set as sensitive (a.k.a NSFW/R-18)",
suggestions: ["nsfw", "r18"]
}
]

54
lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex
View file

@ -7,54 +7,54 @@ defmodule Pleroma.Web.ActivityPub.MRF.HellthreadPolicy do
require Pleroma.Constants
@moduledoc "Block messages with too much mentions (configurable)"
@moduledoc "Block activities with too much mentions (configurable)"
@behaviour Pleroma.Web.ActivityPub.MRF.Policy
defp delist_message(message, threshold) when threshold > 0 do
follower_collection = User.get_cached_by_ap_id(message["actor"]).follower_address
to = message["to"] || []
cc = message["cc"] || []
defp delist_activity(activity, threshold) when threshold > 0 do
follower_collection = User.get_cached_by_ap_id(activity["actor"]).follower_address
to = activity["to"] || []
cc = activity["cc"] || []
follower_collection? = Enum.member?(to ++ cc, follower_collection)
message =
case get_recipient_count(message) do
activity =
case get_recipient_count(activity) do
{:public, recipients}
when follower_collection? and recipients > threshold ->
message
activity
|> Map.put("to", [follower_collection])
|> Map.put("cc", [Pleroma.Constants.as_public()])
{:public, recipients} when recipients > threshold ->
message
activity
|> Map.put("to", [])
|> Map.put("cc", [Pleroma.Constants.as_public()])
_ ->
message
activity
end
{:ok, message}
{:ok, activity}
end
defp delist_message(message, _threshold), do: {:ok, message}
defp delist_activity(activity, _threshold), do: {:ok, activity}
defp reject_message(message, threshold) when threshold > 0 do
with {_, recipients} <- get_recipient_count(message) do
defp reject_activity(activity, threshold) when threshold > 0 do
with {_, recipients} <- get_recipient_count(activity) do
if recipients > threshold do
{:reject, "[HellthreadPolicy] #{recipients} recipients is over the limit of #{threshold}"}
else
{:ok, message}
{:ok, activity}
end
end
end
defp reject_message(message, _threshold), do: {:ok, message}
defp reject_activity(activity, _threshold), do: {:ok, activity}
defp get_recipient_count(message) do
recipients = (message["to"] || []) ++ (message["cc"] || [])
follower_collection = User.get_cached_by_ap_id(message["actor"]).follower_address
defp get_recipient_count(activity) do
recipients = (activity["to"] || []) ++ (activity["cc"] || [])
follower_collection = User.get_cached_by_ap_id(activity["actor"]).follower_address
if Enum.member?(recipients, Pleroma.Constants.as_public()) do
recipients =
@ -73,7 +73,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.HellthreadPolicy do
end
@impl true
def filter(%{"type" => "Create", "object" => %{"type" => object_type}} = message)
def filter(%{"type" => "Create", "object" => %{"type" => object_type}} = activity)
when object_type in ~w{Note Article} do
reject_threshold =
Pleroma.Config.get(
@ -83,16 +83,16 @@ defmodule Pleroma.Web.ActivityPub.MRF.HellthreadPolicy do
delist_threshold = Pleroma.Config.get([:mrf_hellthread, :delist_threshold])
with {:ok, message} <- reject_message(message, reject_threshold),
{:ok, message} <- delist_message(message, delist_threshold) do
{:ok, message}
with {:ok, activity} <- reject_activity(activity, reject_threshold),
{:ok, activity} <- delist_activity(activity, delist_threshold) do
{:ok, activity}
else
e -> e
end
end
@impl true
def filter(message), do: {:ok, message}
def filter(activity), do: {:ok, activity}
@impl true
def describe,
@ -104,13 +104,13 @@ defmodule Pleroma.Web.ActivityPub.MRF.HellthreadPolicy do
key: :mrf_hellthread,
related_policy: "Pleroma.Web.ActivityPub.MRF.HellthreadPolicy",
label: "MRF Hellthread",
description: "Block messages with excessive user mentions",
description: "Block activities with excessive user mentions",
children: [
%{
key: :delist_threshold,
type: :integer,
description:
"Number of mentioned users after which the message gets removed from timelines and" <>
"Number of mentioned users after which the activity gets removed from timelines and" <>
"disables notifications. Set to 0 to disable.",
suggestions: [10]
},
@ -118,7 +118,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.HellthreadPolicy do
key: :reject_threshold,
type: :integer,
description:
"Number of mentioned users after which the messaged gets rejected. Set to 0 to disable.",
"Number of mentioned users after which the activity gets rejected. Set to 0 to disable.",
suggestions: [20]
}
]

4
lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex
View file

@ -48,12 +48,12 @@ defmodule Pleroma.Web.ActivityPub.MRF.InlineQuotePolicy do
end
@impl true
def filter(object), do: {:ok, object}
def filter(activity), do: {:ok, activity}
@impl true
def describe, do: {:ok, %{}}
@impl Pleroma.Web.ActivityPub.MRF.Policy
@impl true
def history_awareness, do: :auto
@impl true

48
lib/pleroma/web/activity_pub/mrf/keyword_policy.ex
View file

@ -7,7 +7,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do
alias Pleroma.Web.ActivityPub.MRF.Utils
@moduledoc "Reject or Word-Replace messages with a keyword or regex"
@moduledoc "Reject or Word-Replace activities with a keyword or regex"
@behaviour Pleroma.Web.ActivityPub.MRF.Policy
@ -25,7 +25,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do
|> Enum.join("\n")
end
defp check_reject(%{"object" => %{} = object} = message) do
defp check_reject(%{"object" => %{} = object} = activity) do
with {:ok, _new_object} <-
Pleroma.Object.Updater.do_with_history(object, fn object ->
payload = object_payload(object)
@ -35,16 +35,16 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do
end) do
{:reject, "[KeywordPolicy] Matches with rejected keyword"}
else
{:ok, message}
{:ok, activity}
end
end) do
{:ok, message}
{:ok, activity}
else
e -> e
end
end
defp check_ftl_removal(%{"type" => "Create", "to" => to, "object" => %{} = object} = message) do
defp check_ftl_removal(%{"type" => "Create", "to" => to, "object" => %{} = object} = activity) do
check_keyword = fn object ->
payload = object_payload(object)
@ -67,24 +67,24 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do
if Pleroma.Constants.as_public() in to and should_delist?.(object) do
to = List.delete(to, Pleroma.Constants.as_public())
cc = [Pleroma.Constants.as_public() | message["cc"] || []]
cc = [Pleroma.Constants.as_public() | activity["cc"] || []]
message =
message
activity =
activity
|> Map.put("to", to)
|> Map.put("cc", cc)
{:ok, message}
{:ok, activity}
else
{:ok, message}
{:ok, activity}
end
end
defp check_ftl_removal(message) do
{:ok, message}
defp check_ftl_removal(activity) do
{:ok, activity}
end
defp check_replace(%{"object" => %{} = object} = message) do
defp check_replace(%{"object" => %{} = object} = activity) do
replace_kw = fn object ->
["content", "name", "summary"]
|> Enum.filter(fn field -> Map.has_key?(object, field) && object[field] end)
@ -103,18 +103,18 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do
{:ok, object} = Pleroma.Object.Updater.do_with_history(object, replace_kw)
message = Map.put(message, "object", object)
activity = Map.put(activity, "object", object)
{:ok, message}
{:ok, activity}
end
@impl true
def filter(%{"type" => type, "object" => %{"content" => _content}} = message)
def filter(%{"type" => type, "object" => %{"content" => _content}} = activity)
when type in ["Create", "Update"] do
with {:ok, message} <- check_reject(message),
{:ok, message} <- check_ftl_removal(message),
{:ok, message} <- check_replace(message) do
{:ok, message}
with {:ok, activity} <- check_reject(activity),
{:ok, activity} <- check_ftl_removal(activity),
{:ok, activity} <- check_replace(activity) do
{:ok, activity}
else
{:reject, nil} -> {:reject, "[KeywordPolicy] "}
{:reject, _} = e -> e
@ -123,7 +123,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do
end
@impl true
def filter(message), do: {:ok, message}
def filter(activity), do: {:ok, activity}
@impl true
def describe do
@ -154,13 +154,13 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do
related_policy: "Pleroma.Web.ActivityPub.MRF.KeywordPolicy",
label: "MRF Keyword",
description:
"Reject or Word-Replace messages matching a keyword or [Regex](https://hexdocs.pm/elixir/Regex.html).",
"Reject or Word-Replace activities matching a keyword or [Regex](https://hexdocs.pm/elixir/Regex.html).",
children: [
%{
key: :reject,
type: {:list, :string},
description: """
A list of patterns which result in message being rejected.
A list of patterns which result in the activity being rejected.
Each pattern can be a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`.
""",
@ -170,7 +170,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do
key: :federated_timeline_removal,
type: {:list, :string},
description: """
A list of patterns which result in message being removed from federated timelines (a.k.a unlisted).
A list of patterns which result in the activity being removed from federated timelines (a.k.a unlisted).
Each pattern can be a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`.
""",

28
lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex
View file

@ -11,11 +11,6 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do
require Logger
@adapter_options [
pool: :media,
recv_timeout: 10_000
]
@impl true
def history_awareness, do: :auto
@ -27,19 +22,16 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do
Logger.debug("Prefetching #{inspect(url)} as #{inspect(prefetch_url)}")
if Pleroma.Config.get(:env) == :test do
fetch(prefetch_url)
else
ConcurrentLimiter.limit(__MODULE__, fn ->
Task.start(fn -> fetch(prefetch_url) end)
end)
end
fetch(prefetch_url)
end
end
defp fetch(url), do: HTTP.get(url, [], @adapter_options)
defp fetch(url) do
http_client_opts = Pleroma.Config.get([:media_proxy, :proxy_opts, :http], pool: :media)
HTTP.get(url, [], http_client_opts)
end
defp preload(%{"object" => %{"attachment" => attachments}} = _message) do
defp preload(%{"object" => %{"attachment" => attachments}} = _activity) do
Enum.each(attachments, fn
%{"url" => url} when is_list(url) ->
url
@ -57,15 +49,15 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do
end
@impl true
def filter(%{"type" => type, "object" => %{"attachment" => attachments} = _object} = message)
def filter(%{"type" => type, "object" => %{"attachment" => attachments} = _object} = activity)
when type in ["Create", "Update"] and is_list(attachments) and length(attachments) > 0 do
preload(message)
preload(activity)
{:ok, message}
{:ok, activity}
end
@impl true
def filter(message), do: {:ok, message}
def filter(activity), do: {:ok, activity}
@impl true
def describe, do: {:ok, %{}}

12
lib/pleroma/web/activity_pub/mrf/mention_policy.ex
View file

@ -3,25 +3,25 @@
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.ActivityPub.MRF.MentionPolicy do
@moduledoc "Block messages which mention a user"
@moduledoc "Block activities which mention a user"
@behaviour Pleroma.Web.ActivityPub.MRF.Policy
@impl true
def filter(%{"type" => "Create"} = message) do
def filter(%{"type" => "Create"} = activity) do
reject_actors = Pleroma.Config.get([:mrf_mention, :actors], [])
recipients = (message["to"] || []) ++ (message["cc"] || [])
recipients = (activity["to"] || []) ++ (activity["cc"] || [])
if rejected_mention =
Enum.find(recipients, fn recipient -> Enum.member?(reject_actors, recipient) end) do
{:reject, "[MentionPolicy] Rejected for mention of #{rejected_mention}"}
else
{:ok, message}
{:ok, activity}
end
end
@impl true
def filter(message), do: {:ok, message}
def filter(activity), do: {:ok, activity}
@impl true
def describe, do: {:ok, %{}}
@ -32,7 +32,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.MentionPolicy do
key: :mrf_mention,
related_policy: "Pleroma.Web.ActivityPub.MRF.MentionPolicy",
label: "MRF Mention",
description: "Block messages which mention a specific user",
description: "Block activities which mention a specific user",
children: [
%{
key: :actors,

14
lib/pleroma/web/activity_pub/mrf/no_empty_policy.ex
View file

@ -9,20 +9,20 @@ defmodule Pleroma.Web.ActivityPub.MRF.NoEmptyPolicy do
alias Pleroma.Web.Endpoint
@impl true
def filter(%{"actor" => actor} = object) do
def filter(%{"actor" => actor} = activity) do
with true <- local?(actor),
true <- eligible_type?(object),
true <- note?(object),
false <- has_attachment?(object),
true <- only_mentions?(object) do
true <- eligible_type?(activity),
true <- note?(activity),
false <- has_attachment?(activity),
true <- only_mentions?(activity) do
{:reject, "[NoEmptyPolicy]"}
else
_ ->
{:ok, object}
{:ok, activity}
end
end
def filter(object), do: {:ok, object}
def filter(activity), do: {:ok, activity}
defp local?(actor) do
if actor |> String.starts_with?("#{Endpoint.url()}") do

4
lib/pleroma/web/activity_pub/mrf/no_op_policy.ex
View file

@ -7,8 +7,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.NoOpPolicy do
@behaviour Pleroma.Web.ActivityPub.MRF.Policy
@impl true
def filter(object) do
{:ok, object}
def filter(activity) do
{:ok, activity}
end
@impl true

8
lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex
View file

@ -13,15 +13,15 @@ defmodule Pleroma.Web.ActivityPub.MRF.NoPlaceholderTextPolicy do
def filter(
%{
"type" => type,
"object" => %{"content" => content, "attachment" => _} = _child_object
} = object
"object" => %{"content" => content, "attachment" => _} = _object
} = activity
)
when type in ["Create", "Update"] and content in [".", "<p>.</p>"] do
{:ok, put_in(object, ["object", "content"], "")}
{:ok, put_in(activity, ["object", "content"], "")}
end
@impl true
def filter(object), do: {:ok, object}
def filter(activity), do: {:ok, activity}
@impl true
def describe, do: {:ok, %{}}

10
lib/pleroma/web/activity_pub/mrf/normalize_markup.ex
View file

@ -12,20 +12,20 @@ defmodule Pleroma.Web.ActivityPub.MRF.NormalizeMarkup do
def history_awareness, do: :auto
@impl true
def filter(%{"type" => type, "object" => child_object} = object)
def filter(%{"type" => type, "object" => object} = activity)
when type in ["Create", "Update"] do
scrub_policy = Pleroma.Config.get([:mrf_normalize_markup, :scrub_policy])
content =
child_object["content"]
object["content"]
|> HTML.filter_tags(scrub_policy)
object = put_in(object, ["object", "content"], content)
activity = put_in(activity, ["object", "content"], content)
{:ok, object}
{:ok, activity}
end
def filter(object), do: {:ok, object}
def filter(activity), do: {:ok, activity}
@impl true
def describe, do: {:ok, %{}}

264
lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex Normal file
View file

@ -0,0 +1,264 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.ActivityPub.MRF.NsfwApiPolicy do
@moduledoc """
Hide, delete, or mark sensitive NSFW content with artificial intelligence.
Requires a NSFW API server, configured like so:
config :pleroma, Pleroma.Web.ActivityPub.MRF.NsfwMRF,
url: "http://127.0.0.1:5000/",
threshold: 0.7,
mark_sensitive: true,
unlist: false,
reject: false
The NSFW API server must implement an HTTP endpoint like this:
curl http://localhost:5000/?url=https://fedi.com/images/001.jpg
Returning a response like this:
{"score", 0.314}
Where a score is 0-1, with `1` being definitely NSFW.
A good API server is here: https://github.com/EugenCepoi/nsfw_api
You can run it with Docker with a one-liner:
docker run -it -p 127.0.0.1:5000:5000/tcp --env PORT=5000 eugencepoi/nsfw_api:latest
Options:
- `url`: Base URL of the API server. Default: "http://127.0.0.1:5000/"
- `threshold`: Lowest score to take action on. Default: `0.7`
- `mark_sensitive`: Mark sensitive all detected NSFW content? Default: `true`
- `unlist`: Unlist all detected NSFW content? Default: `false`
- `reject`: Reject all detected NSFW content (takes precedence)? Default: `false`
"""
alias Pleroma.Config
alias Pleroma.Constants
alias Pleroma.HTTP
alias Pleroma.User
require Logger
require Pleroma.Constants
@behaviour Pleroma.Web.ActivityPub.MRF.Policy
@policy :mrf_nsfw_api
def build_request_url(url) do
Config.get([@policy, :url])
|> URI.parse()
|> fix_path()
|> Map.put(:query, "url=#{url}")
|> URI.to_string()
end
def parse_url(url) do
request = build_request_url(url)
with {:ok, %Tesla.Env{body: body}} <- HTTP.get(request) do
Jason.decode(body)
else
error ->
Logger.warning("""
[NsfwApiPolicy]: The API server failed. Skipping.
#{inspect(error)}
""")
error
end
end
def check_url_nsfw(url) when is_binary(url) do
threshold = Config.get([@policy, :threshold])
case parse_url(url) do
{:ok, %{"score" => score}} when score >= threshold ->
{:nsfw, %{url: url, score: score, threshold: threshold}}
{:ok, %{"score" => score}} ->
{:sfw, %{url: url, score: score, threshold: threshold}}
_ ->
{:sfw, %{url: url, score: nil, threshold: threshold}}
end
end
def check_url_nsfw(%{"href" => url}) when is_binary(url) do
check_url_nsfw(url)
end
def check_url_nsfw(url) do
threshold = Config.get([@policy, :threshold])
{:sfw, %{url: url, score: nil, threshold: threshold}}
end
def check_attachment_nsfw(%{"url" => urls} = attachment) when is_list(urls) do
if Enum.all?(urls, &match?({:sfw, _}, check_url_nsfw(&1))) do
{:sfw, attachment}
else
{:nsfw, attachment}
end
end
def check_attachment_nsfw(%{"url" => url} = attachment) when is_binary(url) do
case check_url_nsfw(url) do
{:sfw, _} -> {:sfw, attachment}
{:nsfw, _} -> {:nsfw, attachment}
end
end
def check_attachment_nsfw(attachment), do: {:sfw, attachment}
def check_object_nsfw(%{"attachment" => attachments} = object) when is_list(attachments) do
if Enum.all?(attachments, &match?({:sfw, _}, check_attachment_nsfw(&1))) do
{:sfw, object}
else
{:nsfw, object}
end
end
def check_object_nsfw(%{"object" => %{} = object} = activity) do
case check_object_nsfw(object) do
{:sfw, _} -> {:sfw, activity}
{:nsfw, _} -> {:nsfw, activity}
end
end
def check_object_nsfw(object), do: {:sfw, object}
@impl true
def filter(activity) do
with {:sfw, activity} <- check_object_nsfw(activity) do
{:ok, activity}
else
{:nsfw, _data} -> handle_nsfw(activity)
end
end
defp handle_nsfw(activity) do
if Config.get([@policy, :reject]) do
{:reject, activity}
else
{:ok,
activity
|> maybe_unlist()
|> maybe_mark_sensitive()}
end
end
defp maybe_unlist(activity) do
if Config.get([@policy, :unlist]) do
unlist(activity)
else
activity
end
end
defp maybe_mark_sensitive(activity) do
if Config.get([@policy, :mark_sensitive]) do
mark_sensitive(activity)
else
activity
end
end
def unlist(%{"to" => to, "cc" => cc, "actor" => actor} = activity) do
with %User{} = user <- User.get_cached_by_ap_id(actor) do
to =
[user.follower_address | to]
|> List.delete(Constants.as_public())
|> Enum.uniq()
cc =
[Constants.as_public() | cc]
|> List.delete(user.follower_address)
|> Enum.uniq()
activity
|> Map.put("to", to)
|> Map.put("cc", cc)
else
_ -> raise "[NsfwApiPolicy]: Could not find user #{actor}"
end
end
def mark_sensitive(%{"object" => object} = activity) when is_map(object) do
Map.put(activity, "object", mark_sensitive(object))
end
def mark_sensitive(activity) when is_map(activity) do
tags = (activity["tag"] || []) ++ ["nsfw"]
activity
|> Map.put("tag", tags)
|> Map.put("sensitive", true)
end
# Hackney needs a trailing slash
defp fix_path(%URI{path: path} = uri) when is_binary(path) do
path = String.trim_trailing(path, "/") <> "/"
Map.put(uri, :path, path)
end
defp fix_path(%URI{path: nil} = uri), do: Map.put(uri, :path, "/")
@impl true
def describe do
options = %{
threshold: Config.get([@policy, :threshold]),
mark_sensitive: Config.get([@policy, :mark_sensitive]),
unlist: Config.get([@policy, :unlist]),
reject: Config.get([@policy, :reject])
}
{:ok, %{@policy => options}}
end
@impl true
def config_description do
%{
key: @policy,
related_policy: to_string(__MODULE__),
label: "NSFW API Policy",
description:
"Hide, delete, or mark sensitive NSFW content with artificial intelligence. Requires running an external API server.",
children: [
%{
key: :url,
type: :string,
description: "Base URL of the API server.",
suggestions: ["http://127.0.0.1:5000/"]
},
%{
key: :threshold,
type: :float,
description: "Lowest score to take action on. Between 0 and 1.",
suggestions: [0.7]
},
%{
key: :mark_sensitive,
type: :boolean,
description: "Mark sensitive all detected NSFW content?",
suggestions: [true]
},
%{
key: :unlist,
type: :boolean,
description: "Unlist sensitive all detected NSFW content?",
suggestions: [false]
},
%{
key: :reject,
type: :boolean,
description: "Reject sensitive all detected NSFW content (takes precedence)?",
suggestions: [false]
}
]
}
end
end

58
lib/pleroma/web/activity_pub/mrf/object_age_policy.ex
View file

@ -11,12 +11,12 @@ defmodule Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy do
@moduledoc "Filter activities depending on their age"
@behaviour Pleroma.Web.ActivityPub.MRF.Policy
defp check_date(%{"object" => %{"published" => published}} = message) do
defp check_date(%{"object" => %{"published" => published}} = activity) do
with %DateTime{} = now <- DateTime.utc_now(),
{:ok, %DateTime{} = then, _} <- DateTime.from_iso8601(published),
max_ttl <- Config.get([:mrf_object_age, :threshold]),
{:ttl, false} <- {:ttl, DateTime.diff(now, then) > max_ttl} do
{:ok, message}
{:ok, activity}
else
{:ttl, true} ->
{:reject, nil}
@ -26,73 +26,73 @@ defmodule Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy do
end
end
defp check_reject(message, actions) do
defp check_reject(activity, actions) do
if :reject in actions do
{:reject, "[ObjectAgePolicy]"}
else
{:ok, message}
{:ok, activity}
end
end
defp check_delist(message, actions) do
defp check_delist(activity, actions) do
if :delist in actions do
with %User{} = user <- User.get_cached_by_ap_id(message["actor"]) do
with %User{} = user <- User.get_cached_by_ap_id(activity["actor"]) do
to =
List.delete(message["to"] || [], Pleroma.Constants.as_public()) ++
List.delete(activity["to"] || [], Pleroma.Constants.as_public()) ++
[user.follower_address]
cc =
List.delete(message["cc"] || [], user.follower_address) ++
List.delete(activity["cc"] || [], user.follower_address) ++
[Pleroma.Constants.as_public()]
message =
message
activity =
activity
|> Map.put("to", to)
|> Map.put("cc", cc)
|> Kernel.put_in(["object", "to"], to)
|> Kernel.put_in(["object", "cc"], cc)
{:ok, message}
{:ok, activity}
else
_e ->
{:reject, "[ObjectAgePolicy] Unhandled error"}
end
else
{:ok, message}
{:ok, activity}
end
end
defp check_strip_followers(message, actions) do
defp check_strip_followers(activity, actions) do
if :strip_followers in actions do
with %User{} = user <- User.get_cached_by_ap_id(message["actor"]) do
to = List.delete(message["to"] || [], user.follower_address)
cc = List.delete(message["cc"] || [], user.follower_address)
with %User{} = user <- User.get_cached_by_ap_id(activity["actor"]) do
to = List.delete(activity["to"] || [], user.follower_address)
cc = List.delete(activity["cc"] || [], user.follower_address)
message =
message
activity =
activity
|> Map.put("to", to)
|> Map.put("cc", cc)
|> Kernel.put_in(["object", "to"], to)
|> Kernel.put_in(["object", "cc"], cc)
{:ok, message}
{:ok, activity}
else
_e ->
{:reject, "[ObjectAgePolicy] Unhandled error"}
end
else
{:ok, message}
{:ok, activity}
end
end
@impl true
def filter(%{"type" => "Create", "object" => %{"published" => _}} = message) do
def filter(%{"type" => "Create", "object" => %{"published" => _}} = activity) do
with actions <- Config.get([:mrf_object_age, :actions]),
{:reject, _} <- check_date(message),
{:ok, message} <- check_reject(message, actions),
{:ok, message} <- check_delist(message, actions),
{:ok, message} <- check_strip_followers(message, actions) do
{:ok, message}
{:reject, _} <- check_date(activity),
{:ok, activity} <- check_reject(activity, actions),
{:ok, activity} <- check_delist(activity, actions),
{:ok, activity} <- check_strip_followers(activity, actions) do
{:ok, activity}
else
# check_date() is allowed to short-circuit the pipeline
e -> e
@ -100,7 +100,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy do
end
@impl true
def filter(message), do: {:ok, message}
def filter(activity), do: {:ok, activity}
@impl true
def describe do
@ -131,8 +131,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy do
type: {:list, :atom},
description:
"A list of actions to apply to the post. `:delist` removes the post from public timelines; " <>
"`:strip_followers` removes followers from the ActivityPub recipient list ensuring they won't be delivered to home timelines, additionally for followers-only it degrades to a direct message; " <>
"`:reject` rejects the message entirely",
"`:strip_followers` removes followers from the ActivityPub recipient list ensuring they won't be delivered to home timelines, additionally for followers-only it degrades to a direct activity; " <>
"`:reject` rejects the activity entirely",
suggestions: [:delist, :strip_followers, :reject]
}
]

5
lib/pleroma/web/activity_pub/mrf/policy.ex
View file

@ -3,7 +3,8 @@
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.ActivityPub.MRF.Policy do
@callback filter(map()) :: {:ok | :reject, map()}
@callback filter(Pleroma.Activity.t()) :: {:ok | :reject, Pleroma.Activity.t()}
@callback id_filter(String.t()) :: boolean()
@callback describe() :: {:ok | :error, map()}
@callback config_description() :: %{
optional(:children) => [map()],
@ -13,5 +14,5 @@ defmodule Pleroma.Web.ActivityPub.MRF.Policy do
description: String.t()
}
@callback history_awareness() :: :auto | :manual
@optional_callbacks config_description: 0, history_awareness: 0
@optional_callbacks config_description: 0, history_awareness: 0, id_filter: 1
end

60
lib/pleroma/web/activity_pub/mrf/quiet_reply.ex Normal file
View file

@ -0,0 +1,60 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2023 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.ActivityPub.MRF.QuietReply do
@moduledoc """
QuietReply alters the scope of activities from local users when replying by enforcing them to be "Unlisted" or "Quiet Public". This delivers the activity to all the expected recipients and instances, but it will not be published in the Federated / The Whole Known Network timelines. It will still be published to the Home timelines of the user's followers and visible to anyone who opens the thread.
"""
require Pleroma.Constants
alias Pleroma.User
@behaviour Pleroma.Web.ActivityPub.MRF.Policy
@impl true
def history_awareness, do: :auto
@impl true
def filter(
%{
"type" => "Create",
"to" => to,
"cc" => cc,
"object" => %{
"actor" => actor,
"type" => "Note",
"inReplyTo" => in_reply_to
}
} = activity
) do
with true <- is_binary(in_reply_to),
false <- match?([], cc),
%User{follower_address: followers_collection, local: true} <-
User.get_by_ap_id(actor) do
updated_to =
to
|> Kernel.++([followers_collection])
|> Kernel.--([Pleroma.Constants.as_public()])
updated_cc = [Pleroma.Constants.as_public()]
updated_activity =
activity
|> Map.put("to", updated_to)
|> Map.put("cc", updated_cc)
|> put_in(["object", "to"], updated_to)
|> put_in(["object", "cc"], updated_cc)
{:ok, updated_activity}
else
_ -> {:ok, activity}
end
end
@impl true
def filter(activity), do: {:ok, activity}
@impl true
def describe, do: {:ok, %{}}
end

10
lib/pleroma/web/activity_pub/mrf/quote_to_link_tag_policy.ex
View file

@ -10,18 +10,18 @@ defmodule Pleroma.Web.ActivityPub.MRF.QuoteToLinkTagPolicy do
require Pleroma.Constants
@impl Pleroma.Web.ActivityPub.MRF.Policy
@impl true
def filter(%{"object" => %{"quoteUrl" => _} = object} = activity) do
{:ok, Map.put(activity, "object", filter_object(object))}
end
@impl Pleroma.Web.ActivityPub.MRF.Policy
def filter(object), do: {:ok, object}
@impl true
def filter(activity), do: {:ok, activity}
@impl Pleroma.Web.ActivityPub.MRF.Policy
@impl true
def describe, do: {:ok, %{}}
@impl Pleroma.Web.ActivityPub.MRF.Policy
@impl true
def history_awareness, do: :auto
defp filter_object(%{"quoteUrl" => quote_url} = object) do

118
lib/pleroma/web/activity_pub/mrf/remote_report_policy.ex Normal file
View file

@ -0,0 +1,118 @@
defmodule Pleroma.Web.ActivityPub.MRF.RemoteReportPolicy do
@moduledoc "Drop remote reports if they don't contain enough information."
@behaviour Pleroma.Web.ActivityPub.MRF.Policy
alias Pleroma.Config
@impl true
def filter(%{"type" => "Flag"} = object) do
with {_, false} <- {:local, local?(object)},
{:ok, _} <- maybe_reject_all(object),
{:ok, _} <- maybe_reject_anonymous(object),
{:ok, _} <- maybe_reject_third_party(object),
{:ok, _} <- maybe_reject_empty_message(object) do
{:ok, object}
else
{:local, true} -> {:ok, object}
{:reject, message} -> {:reject, message}
error -> {:reject, error}
end
end
def filter(object), do: {:ok, object}
defp maybe_reject_all(object) do
if Config.get([:mrf_remote_report, :reject_all]) do
{:reject, "[RemoteReportPolicy] Remote report"}
else
{:ok, object}
end
end
defp maybe_reject_anonymous(%{"actor" => actor} = object) do
with true <- Config.get([:mrf_remote_report, :reject_anonymous]),
%URI{path: "/actor"} <- URI.parse(actor) do
{:reject, "[RemoteReportPolicy] Anonymous: #{actor}"}
else
_ -> {:ok, object}
end
end
defp maybe_reject_third_party(%{"object" => objects} = object) do
{_, to} =
case objects do
[head | tail] when is_binary(head) -> {tail, head}
s when is_binary(s) -> {[], s}
_ -> {[], ""}
end
with true <- Config.get([:mrf_remote_report, :reject_third_party]),
false <- String.starts_with?(to, Pleroma.Web.Endpoint.url()) do
{:reject, "[RemoteReportPolicy] Third-party: #{to}"}
else
_ -> {:ok, object}
end
end
defp maybe_reject_empty_message(%{"content" => content} = object)
when is_binary(content) and content != "" do
{:ok, object}
end
defp maybe_reject_empty_message(object) do
if Config.get([:mrf_remote_report, :reject_empty_message]) do
{:reject, ["RemoteReportPolicy] No content"]}
else
{:ok, object}
end
end
defp local?(%{"actor" => actor}) do
String.starts_with?(actor, Pleroma.Web.Endpoint.url())
end
@impl true
def describe do
mrf_remote_report =
Config.get(:mrf_remote_report)
|> Enum.into(%{})
{:ok, %{mrf_remote_report: mrf_remote_report}}
end
@impl true
def config_description do
%{
key: :mrf_remote_report,
related_policy: "Pleroma.Web.ActivityPub.MRF.RemoteReportPolicy",
label: "MRF Remote Report",
description: "Drop remote reports if they don't contain enough information.",
children: [
%{
key: :reject_all,
type: :boolean,
description: "Reject all remote reports? (this option takes precedence)",
suggestions: [false]
},
%{
key: :reject_anonymous,
type: :boolean,
description: "Reject anonymous remote reports?",
suggestions: [true]
},
%{
key: :reject_third_party,
type: :boolean,
description: "Reject reports on users from third-party instances?",
suggestions: [true]
},
%{
key: :reject_empty_message,
type: :boolean,
description: "Reject remote reports with no message?",
suggestions: [true]
}
]
}
end
end

168
lib/pleroma/web/activity_pub/mrf/simple_policy.ex
View file

@ -13,20 +13,20 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do
require Pleroma.Constants
defp check_accept(%{host: actor_host} = _actor_info, object) do
defp check_accept(%{host: actor_host} = _actor_info, activity) do
accepts =
instance_list(:accept)
|> MRF.subdomains_regex()
cond do
accepts == [] -> {:ok, object}
actor_host == Config.get([Pleroma.Web.Endpoint, :url, :host]) -> {:ok, object}
MRF.subdomain_match?(accepts, actor_host) -> {:ok, object}
accepts == [] -> {:ok, activity}
actor_host == Config.get([Pleroma.Web.Endpoint, :url, :host]) -> {:ok, activity}
MRF.subdomain_match?(accepts, actor_host) -> {:ok, activity}
true -> {:reject, "[SimplePolicy] host not in accept list"}
end
end
defp check_reject(%{host: actor_host} = _actor_info, object) do
defp check_reject(%{host: actor_host} = _actor_info, activity) do
rejects =
instance_list(:reject)
|> MRF.subdomains_regex()
@ -34,109 +34,109 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do
if MRF.subdomain_match?(rejects, actor_host) do
{:reject, "[SimplePolicy] host in reject list"}
else
{:ok, object}
{:ok, activity}
end
end
defp check_media_removal(
%{host: actor_host} = _actor_info,
%{"type" => type, "object" => %{"attachment" => child_attachment}} = object
%{"type" => type, "object" => %{"attachment" => object_attachment}} = activity
)
when length(child_attachment) > 0 and type in ["Create", "Update"] do
when length(object_attachment) > 0 and type in ["Create", "Update"] do
media_removal =
instance_list(:media_removal)
|> MRF.subdomains_regex()
object =
activity =
if MRF.subdomain_match?(media_removal, actor_host) do
child_object = Map.delete(object["object"], "attachment")
Map.put(object, "object", child_object)
object = Map.delete(activity["object"], "attachment")
Map.put(activity, "object", object)
else
object
activity
end
{:ok, object}
{:ok, activity}
end
defp check_media_removal(_actor_info, object), do: {:ok, object}
defp check_media_removal(_actor_info, activity), do: {:ok, activity}
defp check_media_nsfw(
%{host: actor_host} = _actor_info,
%{
"type" => type,
"object" => %{} = _child_object
} = object
"object" => %{} = _object
} = activity
)
when type in ["Create", "Update"] do
media_nsfw =
instance_list(:media_nsfw)
|> MRF.subdomains_regex()
object =
activity =
if MRF.subdomain_match?(media_nsfw, actor_host) do
Kernel.put_in(object, ["object", "sensitive"], true)
Kernel.put_in(activity, ["object", "sensitive"], true)
else
object
activity
end
{:ok, object}
{:ok, activity}
end
defp check_media_nsfw(_actor_info, object), do: {:ok, object}
defp check_media_nsfw(_actor_info, activity), do: {:ok, activity}
defp check_ftl_removal(%{host: actor_host} = _actor_info, object) do
defp check_ftl_removal(%{host: actor_host} = _actor_info, activity) do
timeline_removal =
instance_list(:federated_timeline_removal)
|> MRF.subdomains_regex()
object =
activity =
with true <- MRF.subdomain_match?(timeline_removal, actor_host),
user <- User.get_cached_by_ap_id(object["actor"]),
true <- Pleroma.Constants.as_public() in object["to"] do
to = List.delete(object["to"], Pleroma.Constants.as_public()) ++ [user.follower_address]
user <- User.get_cached_by_ap_id(activity["actor"]),
true <- Pleroma.Constants.as_public() in activity["to"] do
to = List.delete(activity["to"], Pleroma.Constants.as_public()) ++ [user.follower_address]
cc = List.delete(object["cc"], user.follower_address) ++ [Pleroma.Constants.as_public()]
cc = List.delete(activity["cc"], user.follower_address) ++ [Pleroma.Constants.as_public()]
object
activity
|> Map.put("to", to)
|> Map.put("cc", cc)
else
_ -> object
_ -> activity
end
{:ok, object}
{:ok, activity}
end
defp intersection(list1, list2) do
list1 -- list1 -- list2
end
defp check_followers_only(%{host: actor_host} = _actor_info, object) do
defp check_followers_only(%{host: actor_host} = _actor_info, activity) do
followers_only =
instance_list(:followers_only)
|> MRF.subdomains_regex()
object =
activity =
with true <- MRF.subdomain_match?(followers_only, actor_host),
user <- User.get_cached_by_ap_id(object["actor"]) do
user <- User.get_cached_by_ap_id(activity["actor"]) do
# Don't use Map.get/3 intentionally, these must not be nil
fixed_to = object["to"] || []
fixed_cc = object["cc"] || []
fixed_to = activity["to"] || []
fixed_cc = activity["cc"] || []
to = FollowingRelationship.followers_ap_ids(user, fixed_to)
cc = FollowingRelationship.followers_ap_ids(user, fixed_cc)
object
activity
|> Map.put("to", intersection([user.follower_address | to], fixed_to))
|> Map.put("cc", intersection([user.follower_address | cc], fixed_cc))
else
_ -> object
_ -> activity
end
{:ok, object}
{:ok, activity}
end
defp check_report_removal(%{host: actor_host} = _actor_info, %{"type" => "Flag"} = object) do
defp check_report_removal(%{host: actor_host} = _actor_info, %{"type" => "Flag"} = activity) do
report_removal =
instance_list(:report_removal)
|> MRF.subdomains_regex()
@ -144,39 +144,39 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do
if MRF.subdomain_match?(report_removal, actor_host) do
{:reject, "[SimplePolicy] host in report_removal list"}
else
{:ok, object}
{:ok, activity}
end
end
defp check_report_removal(_actor_info, object), do: {:ok, object}
defp check_report_removal(_actor_info, activity), do: {:ok, activity}
defp check_avatar_removal(%{host: actor_host} = _actor_info, %{"icon" => _icon} = object) do
defp check_avatar_removal(%{host: actor_host} = _actor_info, %{"icon" => _icon} = activity) do
avatar_removal =
instance_list(:avatar_removal)
|> MRF.subdomains_regex()
if MRF.subdomain_match?(avatar_removal, actor_host) do
{:ok, Map.delete(object, "icon")}
{:ok, Map.delete(activity, "icon")}
else
{:ok, object}
{:ok, activity}
end
end
defp check_avatar_removal(_actor_info, object), do: {:ok, object}
defp check_avatar_removal(_actor_info, activity), do: {:ok, activity}
defp check_banner_removal(%{host: actor_host} = _actor_info, %{"image" => _image} = object) do
defp check_banner_removal(%{host: actor_host} = _actor_info, %{"image" => _image} = activity) do
banner_removal =
instance_list(:banner_removal)
|> MRF.subdomains_regex()
if MRF.subdomain_match?(banner_removal, actor_host) do
{:ok, Map.delete(object, "image")}
{:ok, Map.delete(activity, "image")}
else
{:ok, object}
{:ok, activity}
end
end
defp check_banner_removal(_actor_info, object), do: {:ok, object}
defp check_banner_removal(_actor_info, activity), do: {:ok, activity}
defp check_object(%{"object" => object} = activity) do
with {:ok, _object} <- filter(object) do
@ -184,7 +184,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do
end
end
defp check_object(object), do: {:ok, object}
defp check_object(activity), do: {:ok, activity}
defp instance_list(config_key) do
Config.get([:mrf_simple, config_key])
@ -192,7 +192,19 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do
end
@impl true
def filter(%{"type" => "Delete", "actor" => actor} = object) do
def id_filter(id) do
host_info = URI.parse(id)
with {:ok, _} <- check_accept(host_info, %{}),
{:ok, _} <- check_reject(host_info, %{}) do
true
else
_ -> false
end
end
@impl true
def filter(%{"type" => "Delete", "actor" => actor} = activity) do
%{host: actor_host} = URI.parse(actor)
reject_deletes =
@ -202,60 +214,54 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do
if MRF.subdomain_match?(reject_deletes, actor_host) do
{:reject, "[SimplePolicy] host in reject_deletes list"}
else
{:ok, object}
{:ok, activity}
end
end
@impl true
def filter(%{"actor" => actor} = object) do
def filter(%{"actor" => actor} = activity) do
actor_info = URI.parse(actor)
with {:ok, object} <- check_accept(actor_info, object),
{:ok, object} <- check_reject(actor_info, object),
{:ok, object} <- check_media_removal(actor_info, object),
{:ok, object} <- check_media_nsfw(actor_info, object),
{:ok, object} <- check_ftl_removal(actor_info, object),
{:ok, object} <- check_followers_only(actor_info, object),
{:ok, object} <- check_report_removal(actor_info, object),
{:ok, object} <- check_object(object) do
{:ok, object}
with {:ok, activity} <- check_accept(actor_info, activity),
{:ok, activity} <- check_reject(actor_info, activity),
{:ok, activity} <- check_media_removal(actor_info, activity),
{:ok, activity} <- check_media_nsfw(actor_info, activity),
{:ok, activity} <- check_ftl_removal(actor_info, activity),
{:ok, activity} <- check_followers_only(actor_info, activity),
{:ok, activity} <- check_report_removal(actor_info, activity),
{:ok, activity} <- check_object(activity) do
{:ok, activity}
else
{:reject, nil} -> {:reject, "[SimplePolicy]"}
{:reject, _} = e -> e
_ -> {:reject, "[SimplePolicy]"}
end
end
def filter(%{"id" => actor, "type" => obj_type} = object)
when obj_type in ["Application", "Group", "Organization", "Person", "Service"] do
def filter(%{"id" => actor, "type" => actor_type} = activity)
when actor_type in ["Application", "Group", "Organization", "Person", "Service"] do
actor_info = URI.parse(actor)
with {:ok, object} <- check_accept(actor_info, object),
{:ok, object} <- check_reject(actor_info, object),
{:ok, object} <- check_avatar_removal(actor_info, object),
{:ok, object} <- check_banner_removal(actor_info, object) do
{:ok, object}
with {:ok, activity} <- check_accept(actor_info, activity),
{:ok, activity} <- check_reject(actor_info, activity),
{:ok, activity} <- check_avatar_removal(actor_info, activity),
{:ok, activity} <- check_banner_removal(actor_info, activity) do
{:ok, activity}
else
{:reject, nil} -> {:reject, "[SimplePolicy]"}
{:reject, _} = e -> e
_ -> {:reject, "[SimplePolicy]"}
end
end
def filter(object) when is_binary(object) do
uri = URI.parse(object)
def filter(activity) when is_binary(activity) do
uri = URI.parse(activity)
with {:ok, object} <- check_accept(uri, object),
{:ok, object} <- check_reject(uri, object) do
{:ok, object}
with {:ok, activity} <- check_accept(uri, activity),
{:ok, activity} <- check_reject(uri, activity) do
{:ok, activity}
else
{:reject, nil} -> {:reject, "[SimplePolicy]"}
{:reject, _} = e -> e
_ -> {:reject, "[SimplePolicy]"}
end
end
def filter(object), do: {:ok, object}
def filter(activity), do: {:ok, activity}
@impl true
def describe do

6
lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex
View file

@ -62,7 +62,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.StealEmojiPolicy do
end
@impl true
def filter(%{"object" => %{"emoji" => foreign_emojis, "actor" => actor}} = message) do
def filter(%{"object" => %{"emoji" => foreign_emojis, "actor" => actor}} = activity) do
host = URI.parse(actor).host
if host != Pleroma.Web.Endpoint.host() and accept_host?(host) do
@ -97,10 +97,10 @@ defmodule Pleroma.Web.ActivityPub.MRF.StealEmojiPolicy do
end
end
{:ok, message}
{:ok, activity}
end
def filter(message), do: {:ok, message}
def filter(activity), do: {:ok, activity}
@impl true
@spec config_description :: %{

10
lib/pleroma/web/activity_pub/mrf/subchain_policy.ex
View file

@ -20,20 +20,20 @@ defmodule Pleroma.Web.ActivityPub.MRF.SubchainPolicy do
end
@impl true
def filter(%{"actor" => actor} = message) do
def filter(%{"actor" => actor} = activity) do
with {:ok, match, subchain} <- lookup_subchain(actor) do
Logger.debug(
"[SubchainPolicy] Matched #{actor} against #{inspect(match)} with subchain #{inspect(subchain)}"
)
MRF.filter(subchain, message)
MRF.filter(subchain, activity)
else
_e -> {:ok, message}
_e -> {:ok, activity}
end
end
@impl true
def filter(message), do: {:ok, message}
def filter(activity), do: {:ok, activity}
@impl true
def describe, do: {:ok, %{}}
@ -45,7 +45,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.SubchainPolicy do
related_policy: "Pleroma.Web.ActivityPub.MRF.SubchainPolicy",
label: "MRF Subchain",
description:
"This policy processes messages through an alternate pipeline when a given message matches certain criteria." <>
"This policy processes activities through an alternate pipeline when a given activity matches certain criteria." <>
" All criteria are configured as a map of regular expressions to lists of policy modules.",
children: [
%{

62
lib/pleroma/web/activity_pub/mrf/tag_policy.ex
View file

@ -28,25 +28,25 @@ defmodule Pleroma.Web.ActivityPub.MRF.TagPolicy do
"mrf_tag:media-force-nsfw",
%{
"type" => type,
"object" => %{"attachment" => child_attachment}
} = message
"object" => %{"attachment" => object_attachment}
} = activity
)
when length(child_attachment) > 0 and type in ["Create", "Update"] do
{:ok, Kernel.put_in(message, ["object", "sensitive"], true)}
when length(object_attachment) > 0 and type in ["Create", "Update"] do
{:ok, Kernel.put_in(activity, ["object", "sensitive"], true)}
end
defp process_tag(
"mrf_tag:media-strip",
%{
"type" => type,
"object" => %{"attachment" => child_attachment} = object
} = message
"object" => %{"attachment" => object_attachment} = object
} = activity
)
when length(child_attachment) > 0 and type in ["Create", "Update"] do
when length(object_attachment) > 0 and type in ["Create", "Update"] do
object = Map.delete(object, "attachment")
message = Map.put(message, "object", object)
activity = Map.put(activity, "object", object)
{:ok, message}
{:ok, activity}
end
defp process_tag(
@ -57,7 +57,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.TagPolicy do
"cc" => cc,
"actor" => actor,
"object" => object
} = message
} = activity
) do
user = User.get_cached_by_ap_id(actor)
@ -70,15 +70,15 @@ defmodule Pleroma.Web.ActivityPub.MRF.TagPolicy do
|> Map.put("to", to)
|> Map.put("cc", cc)
message =
message
activity =
activity
|> Map.put("to", to)
|> Map.put("cc", cc)
|> Map.put("object", object)
{:ok, message}
{:ok, activity}
else
{:ok, message}
{:ok, activity}
end
end
@ -90,7 +90,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.TagPolicy do
"cc" => cc,
"actor" => actor,
"object" => object
} = message
} = activity
) do
user = User.get_cached_by_ap_id(actor)
@ -104,26 +104,26 @@ defmodule Pleroma.Web.ActivityPub.MRF.TagPolicy do
|> Map.put("to", to)
|> Map.put("cc", cc)
message =
message
activity =
activity
|> Map.put("to", to)
|> Map.put("cc", cc)
|> Map.put("object", object)
{:ok, message}
{:ok, activity}
else
{:ok, message}
{:ok, activity}
end
end
defp process_tag(
"mrf_tag:disable-remote-subscription",
%{"type" => "Follow", "actor" => actor} = message
%{"type" => "Follow", "actor" => actor} = activity
) do
user = User.get_cached_by_ap_id(actor)
if user.local == true do
{:ok, message}
{:ok, activity}
else
{:reject,
"[TagPolicy] Follow from #{actor} tagged with mrf_tag:disable-remote-subscription"}
@ -133,14 +133,14 @@ defmodule Pleroma.Web.ActivityPub.MRF.TagPolicy do
defp process_tag("mrf_tag:disable-any-subscription", %{"type" => "Follow", "actor" => actor}),
do: {:reject, "[TagPolicy] Follow from #{actor} tagged with mrf_tag:disable-any-subscription"}
defp process_tag(_, message), do: {:ok, message}
defp process_tag(_, activity), do: {:ok, activity}
def filter_message(actor, message) do
def filter_activity(actor, activity) do
User.get_cached_by_ap_id(actor)
|> get_tags()
|> Enum.reduce({:ok, message}, fn
tag, {:ok, message} ->
process_tag(tag, message)
|> Enum.reduce({:ok, activity}, fn
tag, {:ok, activity} ->
process_tag(tag, activity)
_, error ->
error
@ -148,15 +148,15 @@ defmodule Pleroma.Web.ActivityPub.MRF.TagPolicy do
end
@impl true
def filter(%{"object" => target_actor, "type" => "Follow"} = message),
do: filter_message(target_actor, message)
def filter(%{"object" => target_actor, "type" => "Follow"} = activity),
do: filter_activity(target_actor, activity)
@impl true
def filter(%{"actor" => actor, "type" => type} = message) when type in ["Create", "Update"],
do: filter_message(actor, message)
def filter(%{"actor" => actor, "type" => type} = activity) when type in ["Create", "Update"],
do: filter_activity(actor, activity)
@impl true
def filter(message), do: {:ok, message}
def filter(activity), do: {:ok, activity}
@impl true
def describe, do: {:ok, %{}}

12
lib/pleroma/web/activity_pub/mrf/user_allow_list_policy.ex
View file

@ -8,18 +8,18 @@ defmodule Pleroma.Web.ActivityPub.MRF.UserAllowListPolicy do
@moduledoc "Accept-list of users from specified instances"
@behaviour Pleroma.Web.ActivityPub.MRF.Policy
defp filter_by_list(object, []), do: {:ok, object}
defp filter_by_list(activity, []), do: {:ok, activity}
defp filter_by_list(%{"actor" => actor} = object, allow_list) do
defp filter_by_list(%{"actor" => actor} = activity, allow_list) do
if actor in allow_list do
{:ok, object}
{:ok, activity}
else
{:reject, "[UserAllowListPolicy] #{actor} not in the list"}
end
end
@impl true
def filter(%{"actor" => actor} = object) do
def filter(%{"actor" => actor} = activity) do
actor_info = URI.parse(actor)
allow_list =
@ -28,10 +28,10 @@ defmodule Pleroma.Web.ActivityPub.MRF.UserAllowListPolicy do
[]
)
filter_by_list(object, allow_list)
filter_by_list(activity, allow_list)
end
def filter(object), do: {:ok, object}
def filter(activity), do: {:ok, activity}
@impl true
def describe do

31
lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex
View file

@ -3,39 +3,38 @@
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.ActivityPub.MRF.VocabularyPolicy do
@moduledoc "Filter messages which belong to certain activity vocabularies"
@moduledoc "Filter activities which belong to certain activity vocabularies"
@behaviour Pleroma.Web.ActivityPub.MRF.Policy
@impl true
def filter(%{"type" => "Undo", "object" => child_message} = message) do
with {:ok, _} <- filter(child_message) do
{:ok, message}
def filter(%{"type" => "Undo", "object" => object} = activity) do
with {:ok, _} <- filter(object) do
{:ok, activity}
else
{:reject, _} = e -> e
end
end
def filter(%{"type" => message_type} = message) do
def filter(%{"type" => activity_type} = activity) do
with accepted_vocabulary <- Pleroma.Config.get([:mrf_vocabulary, :accept]),
rejected_vocabulary <- Pleroma.Config.get([:mrf_vocabulary, :reject]),
{_, true} <-
{:accepted,
Enum.empty?(accepted_vocabulary) || Enum.member?(accepted_vocabulary, message_type)},
Enum.empty?(accepted_vocabulary) || Enum.member?(accepted_vocabulary, activity_type)},
{_, false} <-
{:rejected,
length(rejected_vocabulary) > 0 && Enum.member?(rejected_vocabulary, message_type)},
{:ok, _} <- filter(message["object"]) do
{:ok, message}
length(rejected_vocabulary) > 0 && Enum.member?(rejected_vocabulary, activity_type)},
{:ok, _} <- filter(activity["object"]) do
{:ok, activity}
else
{:reject, _} = e -> e
{:accepted, _} -> {:reject, "[VocabularyPolicy] #{message_type} not in accept list"}
{:rejected, _} -> {:reject, "[VocabularyPolicy] #{message_type} in reject list"}
_ -> {:reject, "[VocabularyPolicy]"}
{:accepted, _} -> {:reject, "[VocabularyPolicy] #{activity_type} not in accept list"}
{:rejected, _} -> {:reject, "[VocabularyPolicy] #{activity_type} in reject list"}
end
end
def filter(message), do: {:ok, message}
def filter(activity), do: {:ok, activity}
@impl true
def describe,
@ -47,20 +46,20 @@ defmodule Pleroma.Web.ActivityPub.MRF.VocabularyPolicy do
key: :mrf_vocabulary,
related_policy: "Pleroma.Web.ActivityPub.MRF.VocabularyPolicy",
label: "MRF Vocabulary",
description: "Filter messages which belong to certain activity vocabularies",
description: "Filter activities which belong to certain activity vocabularies",
children: [
%{
key: :accept,
type: {:list, :string},
description:
"A list of ActivityStreams terms to accept. If empty, all supported messages are accepted.",
"A list of ActivityStreams terms to accept. If empty, all supported activities are accepted.",
suggestions: ["Create", "Follow", "Mention", "Announce", "Like"]
},
%{
key: :reject,
type: {:list, :string},
description:
"A list of ActivityStreams terms to reject. If empty, no messages are rejected.",
"A list of ActivityStreams terms to reject. If empty, no activities are rejected.",
suggestions: ["Create", "Follow", "Mention", "Announce", "Like"]
}
]

25
lib/pleroma/web/activity_pub/object_validator.ex
View file

@ -11,6 +11,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
@behaviour Pleroma.Web.ActivityPub.ObjectValidator.Validating
import Pleroma.Constants, only: [activity_types: 0, object_types: 0]
alias Pleroma.Activity
alias Pleroma.EctoType.ActivityPub.ObjectValidators
alias Pleroma.Object
@ -39,6 +41,16 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
@impl true
def validate(object, meta)
# This overload works together with the InboxGuardPlug
# and ensures that we are not accepting any activity type
# that cannot pass InboxGuardPlug.
# If we want to support any more activity types, make sure to
# add it in Pleroma.Constants's activity_types or object_types,
# and, if applicable, allowed_activity_types_from_strangers.
def validate(%{"type" => type}, _meta)
when type not in activity_types() and type not in object_types(),
do: {:error, :not_allowed_object_type}
def validate(%{"type" => "Block"} = block_activity, meta) do
with {:ok, block_activity} <-
block_activity
@ -165,7 +177,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
meta = Keyword.put(meta, :object_data, object_data),
{:ok, update_activity} <-
update_activity
|> UpdateValidator.cast_and_validate()
|> UpdateValidator.cast_and_validate(meta)
|> Ecto.Changeset.apply_action(:insert) do
update_activity = stringify_keys(update_activity)
{:ok, update_activity, meta}
@ -173,7 +185,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
{:local, _} ->
with {:ok, object} <-
update_activity
|> UpdateValidator.cast_and_validate()
|> UpdateValidator.cast_and_validate(meta)
|> Ecto.Changeset.apply_action(:insert) do
object = stringify_keys(object)
{:ok, object, meta}
@ -203,9 +215,16 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
"Answer" -> AnswerValidator
end
cast_func =
if type == "Update" do
fn o -> validator.cast_and_validate(o, meta) end
else
fn o -> validator.cast_and_validate(o) end
end
with {:ok, object} <-
object
|> validator.cast_and_validate()
|> cast_func.()
|> Ecto.Changeset.apply_action(:insert) do
object = stringify_keys(object)
{:ok, object, meta}

Some files were not shown because too many files have changed in this diff Show more