hj/pleroma
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge remote-tracking branch 'origin/develop' into pr-7444-finish

Browse source
This commit is contained in:
Lain Soykaf 2026-07-26 13:20:50 +04:00
commit 7a741736ea
No known key found for this signature in database
1544 changed files with 43219 additions and 9351 deletions
Download patch file Download diff file Expand all files Collapse all files

9
lib/mix/pleroma.ex
View file

@ -14,7 +14,8 @@ defmodule Mix.Pleroma do
:swoosh,
:timex,
:fast_html,
:oban
:oban,
:logger_backends
]
@cachex_children ["object", "user", "scrubber", "web_resp"]
@doc "Common functions to be reused in mix tasks"
@ -25,7 +26,11 @@ defmodule Mix.Pleroma do
Application.put_env(:phoenix, :serve_endpoints, false, persistent: true)
unless System.get_env("DEBUG") do
Logger.remove_backend(:console)
try do
Logger.remove_backend(:console)
catch
:exit, _ -> :ok
end
end
adapter = Application.get_env(:tesla, :adapter)

141
lib/mix/tasks/pleroma/config.ex
View file

@ -205,6 +205,93 @@ defmodule Mix.Tasks.Pleroma.Config do
end
end
# Removes any policies that are not a real module
# as they will prevent the server from starting
def run(["fix_mrf_policies"]) do
check_configdb(fn ->
start_pleroma()
group = :pleroma
key = :mrf
%{value: value} =
group
|> ConfigDB.get_by_group_and_key(key)
policies =
Keyword.get(value, :policies, [])
|> Enum.filter(&is_atom(&1))
|> Enum.filter(fn mrf ->
case Code.ensure_compiled(mrf) do
{:module, _} -> true
{:error, _} -> false
end
end)
value = Keyword.put(value, :policies, policies)
ConfigDB.update_or_create(%{group: group, key: key, value: value})
end)
end
# Removes non-whitelisted configuration sections
def run(["filter_whitelisted" | rest]) do
{options, [], []} =
OptionParser.parse(
rest,
strict: [force: :boolean],
aliases: [f: :force]
)
force = Keyword.get(options, :force, false)
start_pleroma()
whitelisted_configs = Pleroma.Config.get(:database_config_whitelist)
if whitelisted_configs in [nil, false] do
shell_error("No unwanted settings in ConfigDB. No changes made.")
else
{whitelisted_groups, whitelisted_keys} = get_filters(whitelisted_configs)
filtered =
from(c in ConfigDB)
|> Repo.all()
|> Enum.filter(&not_whitelisted?(&1, whitelisted_groups, whitelisted_keys))
do_config_filter(filtered, force)
end
end
# Removes blacklisted configuration sections
def run(["filter_blacklisted" | rest]) do
{options, [], []} =
OptionParser.parse(
rest,
strict: [force: :boolean],
aliases: [f: :force]
)
force = Keyword.get(options, :force, false)
start_pleroma()
blacklisted_configs = Pleroma.Config.get(:database_config_blacklist)
if blacklisted_configs in [nil, false] do
shell_error("No unwanted settings in ConfigDB. No changes made.")
else
{blacklisted_groups, blacklisted_keys} = get_filters(blacklisted_configs)
filtered =
from(c in ConfigDB)
|> Repo.all()
|> Enum.filter(&blacklisted?(&1, blacklisted_groups, blacklisted_keys))
do_config_filter(filtered, force)
end
end
@spec migrate_to_db(Path.t() | nil) :: any()
def migrate_to_db(file_path \\ nil) do
with :ok <- Pleroma.Config.DeprecationWarnings.warn() do
@ -301,7 +388,13 @@ defmodule Mix.Tasks.Pleroma.Config do
|> Enum.each(&write_and_delete(&1, file, opts[:delete]))
:ok = File.close(file)
System.cmd("mix", ["format", path])
# Ensure `mix format` runs in the same env as the current task and doesn't
# emit config-time stderr noise (e.g. dev secret warnings) into `mix test`.
System.cmd("mix", ["format", path],
env: [{"MIX_ENV", to_string(Mix.env())}],
stderr_to_stdout: true
)
end
defp config_header, do: "import Config\r\n\r\n"
@ -399,4 +492,50 @@ defmodule Mix.Tasks.Pleroma.Config do
Ecto.Adapters.SQL.query!(Repo, "TRUNCATE config;")
Ecto.Adapters.SQL.query!(Repo, "ALTER SEQUENCE config_id_seq RESTART;")
end
defp get_filters(config) do
groups =
config
|> Enum.filter(fn
{_group} -> true
_ -> false
end)
|> Enum.map(fn {group} -> group end)
keys =
config
|> Enum.filter(fn
{_group, _key} -> true
_ -> false
end)
{groups, keys}
end
defp not_whitelisted?(%{group: group, key: key}, whitelisted_groups, whitelisted_keys) do
not Enum.member?(whitelisted_groups, group) and
not Enum.member?(whitelisted_keys, {group, key})
end
defp blacklisted?(%{group: group, key: key}, blacklisted_groups, blacklisted_keys) do
Enum.member?(blacklisted_groups, group) or
Enum.member?(blacklisted_keys, {group, key})
end
defp do_config_filter(filtered, _) when length(filtered) == 0 do
shell_error("No unwanted settings in ConfigDB. No changes made.")
end
defp do_config_filter(filtered, force) do
shell_info("The following settings will be removed from ConfigDB:\n")
Enum.each(filtered, &dump(&1))
if force or shell_prompt("Are you sure you want to continue?", "n") in ~w(Yn Y y) do
filtered_ids = Enum.map(filtered, fn %{id: id} -> id end)
Repo.delete_all(from(c in ConfigDB, where: c.id in ^filtered_ids))
else
shell_error("No changes made.")
end
end
end

19
lib/mix/tasks/pleroma/database.ex
View file

@ -226,7 +226,12 @@ defmodule Mix.Tasks.Pleroma.Database do
DELETE FROM hashtags AS ht
WHERE NOT EXISTS (
SELECT 1 FROM hashtags_objects hto
WHERE ht.id = hto.hashtag_id)
WHERE ht.id = hto.hashtag_id
)
AND NOT EXISTS (
SELECT 1 FROM user_follows_hashtag ufh
WHERE ht.id = ufh.hashtag_id
)
"""
|> Repo.query()
@ -295,10 +300,12 @@ defmodule Mix.Tasks.Pleroma.Database do
|> DateTime.from_naive!("Etc/UTC")
|> Timex.shift(days: days)
Pleroma.Workers.PurgeExpiredActivity.enqueue(%{
activity_id: activity.id,
expires_at: expires_at
})
Pleroma.Workers.PurgeExpiredActivity.enqueue(
%{
activity_id: activity.id
},
scheduled_at: expires_at
)
end)
end)
|> Stream.run()
@ -351,7 +358,7 @@ defmodule Mix.Tasks.Pleroma.Database do
)
end
shell_info('Done.')
shell_info(~c"Done.")
end
end

15
lib/mix/tasks/pleroma/emoji.ex
View file

@ -93,6 +93,7 @@ defmodule Mix.Tasks.Pleroma.Emoji do
)
files = fetch_and_decode!(files_loc)
files_to_unzip = for({_, f} <- files, do: f)
IO.puts(IO.ANSI.format(["Unpacking ", :bright, pack_name]))
@ -103,17 +104,7 @@ defmodule Mix.Tasks.Pleroma.Emoji do
pack_name
])
files_to_unzip =
Enum.map(
files,
fn {_, f} -> to_charlist(f) end
)
{:ok, _} =
:zip.unzip(binary_archive,
cwd: String.to_charlist(pack_path),
file_list: files_to_unzip
)
{:ok, _} = Pleroma.SafeZip.unzip_data(binary_archive, pack_path, files_to_unzip)
IO.puts(IO.ANSI.format(["Writing pack.json for ", :bright, pack_name]))
@ -201,7 +192,7 @@ defmodule Mix.Tasks.Pleroma.Emoji do
tmp_pack_dir = Path.join(System.tmp_dir!(), "emoji-pack-#{name}")
{:ok, _} = :zip.unzip(binary_archive, cwd: String.to_charlist(tmp_pack_dir))
{:ok, _} = Pleroma.SafeZip.unzip_data(binary_archive, tmp_pack_dir)
emoji_map = Pleroma.Emoji.Loader.make_shortcode_to_file_map(tmp_pack_dir, exts)

2
lib/mix/tasks/pleroma/instance.ex
View file

@ -271,7 +271,7 @@ defmodule Mix.Tasks.Pleroma.Instance do
[config_dir, psql_dir, static_dir, uploads_dir]
|> Enum.reject(&File.exists?/1)
|> Enum.each(fn dir ->
File.mkdir_p!(dir)
Pleroma.Backports.mkdir_p!(dir)
File.chmod!(dir, 0o700)
end)

2
lib/mix/tasks/pleroma/openapi_spec.ex
View file

@ -22,7 +22,7 @@ defmodule Mix.Tasks.Pleroma.OpenapiSpec do
else
{_, errors} ->
IO.puts(IO.ANSI.format([:red, :bright, "Spec check failed, errors:"]))
Enum.map(errors, &IO.puts/1)
Enum.each(errors, &IO.puts/1)
raise "Spec check failed"
end

2
lib/mix/tasks/pleroma/robots_txt.ex
View file

@ -22,7 +22,7 @@ defmodule Mix.Tasks.Pleroma.RobotsTxt do
static_dir = Pleroma.Config.get([:instance, :static_dir], "instance/static/")
if !File.exists?(static_dir) do
File.mkdir_p!(static_dir)
Pleroma.Backports.mkdir_p!(static_dir)
end
robots_txt_path = Path.join(static_dir, "robots.txt")

9
lib/mix/tasks/pleroma/search/indexer.ex
View file

@ -33,15 +33,18 @@ defmodule Mix.Tasks.Pleroma.Search.Indexer do
OptionParser.parse(
options,
strict: [
limit: :integer
chunk: :integer,
limit: :integer,
step: :integer
]
)
start_pleroma()
chunk_size = Keyword.get(options, :chunk, 100)
limit = Keyword.get(options, :limit, 100_000)
per_step = Keyword.get(options, :step, 1000)
per_step = 1000
chunks = max(div(limit, per_step), 1)
1..chunks
@ -65,7 +68,7 @@ defmodule Mix.Tasks.Pleroma.Search.Indexer do
IO.puts("Got #{length(ids)} activities, adding to indexer")
ids
|> Enum.chunk_every(100)
|> Enum.chunk_every(chunk_size)
|> Enum.each(fn chunk ->
IO.puts("Adding #{length(chunk)} activities to indexing queue")

8
lib/mix/tasks/pleroma/search/meilisearch.ex
View file

@ -9,7 +9,7 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do
import Ecto.Query
import Pleroma.Search.Meilisearch,
only: [meili_post: 2, meili_put: 2, meili_get: 1, meili_delete: 1]
only: [meili_put: 2, meili_get: 1, meili_delete: 1]
def run(["index"]) do
start_pleroma()
@ -28,7 +28,7 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do
end
{:ok, _} =
meili_post(
meili_put(
"/indexes/objects/settings/ranking-rules",
[
"published:desc",
@ -42,7 +42,7 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do
)
{:ok, _} =
meili_post(
meili_put(
"/indexes/objects/settings/searchable-attributes",
[
"content"
@ -72,7 +72,7 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do
query,
timeout: :infinity
)
|> Stream.map(&Pleroma.Search.Meilisearch.object_to_search_data/1)
|> Stream.map(&Pleroma.Search.object_to_search_data/1)
|> Stream.filter(fn o -> not is_nil(o) end)
|> Stream.chunk_every(chunk_size)
|> Stream.transform(0, fn objects, acc ->

27
lib/mix/tasks/pleroma/test_runner.ex Normal file
View file

@ -0,0 +1,27 @@
defmodule Mix.Tasks.Pleroma.TestRunner do
@shortdoc "Retries tests once if they fail"
use Mix.Task
def run(args \\ []) do
case System.cmd("mix", ["test", "--warnings-as-errors"] ++ args,
into: IO.stream(:stdio, :line)
) do
{_, 0} ->
:ok
_ ->
retry(args)
end
end
def retry(args) do
case System.cmd("mix", ["test", "--failed"] ++ args, into: IO.stream(:stdio, :line)) do
{_, 0} ->
:ok
_ ->
exit(1)
end
end
end

2
lib/pleroma/activity/html.ex
View file

@ -38,7 +38,7 @@ defmodule Pleroma.Activity.HTML do
def invalidate_cache_for(activity_id) do
keys = get_cache_keys_for(activity_id)
Enum.map(keys, &@cachex.del(:scrubber_cache, &1))
Enum.each(keys, &@cachex.del(:scrubber_cache, &1))
@cachex.del(:scrubber_management_cache, activity_id)
end

18
lib/pleroma/activity/queries.ex
View file

@ -7,7 +7,7 @@ defmodule Pleroma.Activity.Queries do
Contains queries for Activity.
"""
import Ecto.Query, only: [from: 2, where: 3]
import Ecto.Query, only: [from: 2]
@type query :: Ecto.Queryable.t() | Pleroma.Activity.t()
@ -70,22 +70,6 @@ defmodule Pleroma.Activity.Queries do
)
end
@spec by_object_in_reply_to_id(query, String.t(), keyword()) :: query
def by_object_in_reply_to_id(query, in_reply_to_id, opts \\ []) do
query =
if opts[:skip_preloading] do
Activity.with_joined_object(query)
else
Activity.with_preloaded_object(query)
end
where(
query,
[activity, object: o],
fragment("(?)->>'inReplyTo' = ?", o.data, ^to_string(in_reply_to_id))
)
end
@spec by_type(query, String.t()) :: query
def by_type(query \\ Activity, activity_type) do
from(

81
lib/pleroma/application.ex
View file

@ -14,7 +14,6 @@ defmodule Pleroma.Application do
@name Mix.Project.config()[:name]
@version Mix.Project.config()[:version]
@repository Mix.Project.config()[:source_url]
@compile_env Mix.env()
def name, do: @name
def version, do: @version
@ -44,20 +43,21 @@ defmodule Pleroma.Application do
# every time the application is restarted, so we disable module
# conflicts at runtime
Code.compiler_options(ignore_module_conflict: true)
# Disable warnings_as_errors at runtime, it breaks Phoenix live reload
# due to protocol consolidation warnings
Code.compiler_options(warnings_as_errors: false)
configure_logger()
Pleroma.Telemetry.Logger.attach()
Config.Holder.save_default()
Pleroma.HTML.compile_scrubbers()
Pleroma.Config.Oban.warn()
Config.DeprecationWarnings.warn()
if @compile_env != :test do
if Config.get([Pleroma.Web.Plugs.HTTPSecurityPlug, :enable], true) do
Pleroma.Web.Plugs.HTTPSecurityPlug.warn_if_disabled()
end
Pleroma.ApplicationRequirements.verify!()
if Config.get(:env) != :test do
Pleroma.ApplicationRequirements.verify!()
end
load_custom_modules()
Pleroma.Docs.JSON.compile()
limiters_setup()
@ -69,32 +69,18 @@ defmodule Pleroma.Application do
Finch.start_link(name: MyFinch)
end
if adapter == Tesla.Adapter.Gun do
if version = Pleroma.OTPVersion.version() do
[major, minor] =
version
|> String.split(".")
|> Enum.map(&String.to_integer/1)
|> Enum.take(2)
if (major == 22 and minor < 2) or major < 22 do
raise "
!!!OTP VERSION WARNING!!!
You are using gun adapter with OTP version #{version}, which doesn't support correct handling of unordered certificates chains. Please update your Erlang/OTP to at least 22.2.
"
end
else
raise "
!!!OTP VERSION WARNING!!!
To support correct handling of unordered certificates chains - OTP version must be > 22.2.
"
end
# Disable warnings_as_errors at runtime, it breaks Phoenix live reload
# due to protocol consolidation warnings
# :warnings_as_errors is deprecated via Code.compiler_options/2 since 1.18
if Version.compare(System.version(), "1.18.0") == :lt do
Code.compiler_options(warnings_as_errors: false)
end
# Define workers and child supervisors to be supervised
children =
[
Pleroma.PromEx,
Pleroma.LDAP,
Pleroma.Repo,
Config.TransferTask,
Pleroma.Emoji,
@ -128,6 +114,44 @@ defmodule Pleroma.Application do
Supervisor.start_link(children, opts)
end
def configure_logger do
Config.get([:logger, :backends], [])
|> Enum.each(fn backend ->
case backend_to_logger(backend) do
{:ok, logger} ->
add_logger(logger)
{:error, :console_backend} ->
Logger.warning(":console is no longer considered a backend and is enabled by default")
end
end)
end
defp add_logger(backend) do
case LoggerBackends.add(backend) do
{:ok, _} ->
Logger.debug("Successfully added logger backend: #{inspect(backend)}")
{:error, :already_present} ->
:ok
{:error, reason} ->
Logger.error("Failed to add logger backend #{inspect(backend)}: #{inspect(reason)}")
end
end
defp backend_to_logger(:console), do: {:error, :console_backend}
defp backend_to_logger({:ex_syslogger = backend, name}) do
Logger.warning(
"Configuration {:#{backend}, :#{name}} is incorrect. Use {ExSyslogger, :#{name}} instead!"
)
{:ok, {ExSyslogger, name}}
end
defp backend_to_logger(backend), do: {:ok, backend}
def load_custom_modules do
dir = Config.get([:modules, :runtime_dir])
@ -169,7 +193,8 @@ defmodule Pleroma.Application do
limit: 500_000
),
build_cachex("rel_me", limit: 2500),
build_cachex("host_meta", default_ttl: :timer.minutes(120), limit: 5000)
build_cachex("host_meta", default_ttl: :timer.minutes(120), limit: 5_000),
build_cachex("translations", default_ttl: :timer.hours(24), limit: 5_000)
]
end
@ -292,8 +317,6 @@ defmodule Pleroma.Application do
config = Config.get(ConcurrentLimiter, [])
[
Pleroma.Web.RichMedia.Helpers,
Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy,
Pleroma.Search
]
|> Enum.each(fn module ->

42
lib/pleroma/application_requirements.ex
View file

@ -189,7 +189,40 @@ defmodule Pleroma.ApplicationRequirements do
false
end
if Enum.all?([preview_proxy_commands_status | filter_commands_statuses], & &1) do
language_detector_commands_status =
if Pleroma.Language.LanguageDetector.missing_dependencies() == [] do
true
else
Logger.error(
"The following dependencies required by the currently enabled " <>
"language detection provider are not installed: " <>
inspect(Pleroma.Language.LanguageDetector.missing_dependencies())
)
false
end
translation_commands_status =
if Pleroma.Language.Translation.missing_dependencies() == [] do
true
else
Logger.error(
"The following dependencies required by the currently enabled " <>
"translation provider are not installed: " <>
inspect(Pleroma.Language.Translation.missing_dependencies())
)
false
end
if Enum.all?(
[
preview_proxy_commands_status,
language_detector_commands_status,
translation_commands_status | filter_commands_statuses
],
& &1
) do
:ok
else
{:error,
@ -241,10 +274,9 @@ defmodule Pleroma.ApplicationRequirements do
missing_mrfs =
Enum.reduce(mrfs, [], fn x, acc ->
if Code.ensure_compiled(x) do
acc
else
acc ++ [x]
case Code.ensure_compiled(x) do
{:module, _} -> acc
{:error, _} -> acc ++ [x]
end
end)

72
lib/pleroma/backports.ex Normal file
View file

@ -0,0 +1,72 @@
# Copyright 2012 Plataformatec
# Copyright 2021 The Elixir Team
# SPDX-License-Identifier: Apache-2.0
defmodule Pleroma.Backports do
import File, only: [dir?: 1]
# <https://github.com/elixir-lang/elixir/pull/14242>
# To be removed when we require Elixir 1.19
@doc """
Tries to create the directory `path`.
Missing parent directories are created. Returns `:ok` if successful, or
`{:error, reason}` if an error occurs.
Typical error reasons are:
* `:eacces` - missing search or write permissions for the parent
directories of `path`
* `:enospc` - there is no space left on the device
* `:enotdir` - a component of `path` is not a directory
"""
@spec mkdir_p(Path.t()) :: :ok | {:error, File.posix() | :badarg}
def mkdir_p(path) do
do_mkdir_p(IO.chardata_to_string(path))
end
defp do_mkdir_p("/") do
:ok
end
defp do_mkdir_p(path) do
parent = Path.dirname(path)
if parent == path do
:ok
else
case do_mkdir_p(parent) do
:ok ->
case :file.make_dir(path) do
{:error, :eexist} ->
if dir?(path), do: :ok, else: {:error, :enotdir}
other ->
other
end
e ->
e
end
end
end
@doc """
Same as `mkdir_p/1`, but raises a `File.Error` exception in case of failure.
Otherwise `:ok`.
"""
@spec mkdir_p!(Path.t()) :: :ok
def mkdir_p!(path) do
case mkdir_p(path) do
:ok ->
:ok
{:error, reason} ->
raise File.Error,
reason: reason,
action: "make directory (with -p)",
path: IO.chardata_to_string(path)
end
end
end

11
lib/pleroma/bookmark.ex
View file

@ -19,7 +19,7 @@ defmodule Pleroma.Bookmark do
schema "bookmarks" do
belongs_to(:user, User, type: FlakeId.Ecto.CompatType)
belongs_to(:activity, Activity, type: FlakeId.Ecto.CompatType)
belongs_to(:folder, BookmarkFolder, type: FlakeId.Ecto.CompatType)
belongs_to(:folder, BookmarkFolder, type: FlakeId.Ecto.Type)
timestamps()
end
@ -38,7 +38,7 @@ defmodule Pleroma.Bookmark do
|> validate_required([:user_id, :activity_id])
|> unique_constraint(:activity_id, name: :bookmarks_user_id_activity_id_index)
|> Repo.insert(
on_conflict: [set: [folder_id: folder_id]],
on_conflict: [set: [folder_id: folder_id, updated_at: NaiveDateTime.utc_now()]],
conflict_target: [:user_id, :activity_id]
)
end
@ -76,11 +76,4 @@ defmodule Pleroma.Bookmark do
|> Repo.one()
|> Repo.delete()
end
def set_folder(bookmark, folder_id) do
bookmark
|> cast(%{folder_id: folder_id}, [:folder_id])
|> validate_required([:folder_id])
|> Repo.update()
end
end

2
lib/pleroma/bookmark_folder.ex
View file

@ -14,7 +14,7 @@ defmodule Pleroma.BookmarkFolder do
alias Pleroma.User
@type t :: %__MODULE__{}
@primary_key {:id, FlakeId.Ecto.CompatType, autogenerate: true}
@primary_key {:id, FlakeId.Ecto.Type, autogenerate: true}
schema "bookmark_folders" do
field(:name, :string)

16
lib/pleroma/chat.ex
View file

@ -25,6 +25,8 @@ defmodule Pleroma.Chat do
belongs_to(:user, User, type: FlakeId.Ecto.CompatType)
field(:recipient, :string)
field(:pinned, :boolean)
timestamps()
end
@ -94,4 +96,18 @@ defmodule Pleroma.Chat do
order_by: [desc: c.updated_at]
)
end
def pin(%__MODULE__{} = chat) do
chat
|> cast(%{pinned: true}, [:pinned])
|> Repo.update()
end
def unpin(%__MODULE__{} = chat) do
chat
|> cast(%{pinned: false}, [:pinned])
|> Repo.update()
end
def enabled?, do: Pleroma.Config.get([__MODULE__, :enabled], true)
end

1
lib/pleroma/config.ex
View file

@ -27,6 +27,7 @@ defmodule Pleroma.Config do
Application.get_env(:pleroma, key, default)
end
@impl true
def get!(key) do
value = get(key, nil)

73
lib/pleroma/config/deprecation_warnings.ex
View file

@ -20,6 +20,8 @@ defmodule Pleroma.Config.DeprecationWarnings do
"\n* `config :pleroma, :instance, mrf_transparency_exclusions` is now `config :pleroma, :mrf, transparency_exclusions`"}
]
@logger_formatter_config_knobs [:colors, :format, :metadata, :truncate, :utc_log]
def check_exiftool_filter do
filters = Config.get([Pleroma.Upload]) |> Keyword.get(:filters, [])
@ -218,7 +220,8 @@ defmodule Pleroma.Config.DeprecationWarnings do
check_quarantined_instances_tuples(),
check_transparency_exclusions_tuples(),
check_simple_policy_tuples(),
check_exiftool_filter()
check_exiftool_filter(),
check_deprecated_logger_config()
]
|> Enum.reduce(:ok, fn
:ok, :ok -> :ok
@ -415,4 +418,72 @@ defmodule Pleroma.Config.DeprecationWarnings do
:ok
end
end
defp merge_deprecated_logger_config(config) do
formatter_config = Enum.filter(config, fn {k, _} -> k in @logger_formatter_config_knobs end)
formatter = Logger.default_formatter(formatter_config)
log_level = Keyword.get(config, :level, nil)
Logger.debug("""
Reconfiguring console Logger with deprecated configuration syntax.
Handler configuration:
log_level: #{inspect(log_level)}
Formatter:
#{inspect(formatter)}
""")
if Config.get(:env) != :test do
if log_level, do: :logger.update_handler_config(:default, :level, log_level)
:logger.set_handler_config(:default, :formatter, formatter)
end
end
@spec check_deprecated_logger_config() :: :ok | :error
def check_deprecated_logger_config do
backends_config = Application.get_env(:logger, :backends)
console_config = Application.get_env(:logger, :console)
# NOTE: No need to merge the old backends config since it still works.
# And new configuration will just add new Logger backends.
backend =
if backends_config do
Logger.warning("""
!!!DEPRECATION WARNING!!!
Your configuration is using deprecated syntax for configuring backends of Elixir's logger.
`config :logger, backends: [...]` is deprecated syntax due to changes in Elixir.
Please update your configuration at your earliest convenience to use:
`config :pleroma, :logger, backends: [...]`
Note: `:console` is no longer considered a backend and is used by default, you can disable it using:
`config :logger, :default_handler: false`
""")
true
else
false
end
console =
if console_config do
Logger.warning("""
!!!DEPRECATION WARNING!!!
Your configuration is using deprecated syntax for configuring logging to console.
`config :logger, :console` is deprecated syntax due to changes in Elixir.
Please update your configuration at your earliest convenience to use
`config :logger, default_handler` and `config :logger, :default_formatter`.
Note: `:default_handler` is used only for the `level` setting. All other configurations go under
`:default_formatter`. For more info visit: https://hexdocs.pm/logger/Logger.html#module-backends-and-backwards-compatibility
""")
merge_deprecated_logger_config(console_config)
true
else
false
end
if backend or console, do: :error, else: :ok
end
end

3
lib/pleroma/config/getting.ex
View file

@ -5,10 +5,13 @@
defmodule Pleroma.Config.Getting do
@callback get(any()) :: any()
@callback get(any(), any()) :: any()
@callback get!(any()) :: any()
def get(key), do: get(key, nil)
def get(key, default), do: impl().get(key, default)
def get!(key), do: impl().get!(key)
def impl do
Application.get_env(:pleroma, :config_impl, Pleroma.Config)
end

48
lib/pleroma/config/transfer_task.ex
View file

@ -1,5 +1,5 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# Copyright © 2017-2023 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Config.TransferTask do
@ -22,7 +22,8 @@ defmodule Pleroma.Config.TransferTask do
{:pleroma, :markup},
{:pleroma, :streamer},
{:pleroma, :pools},
{:pleroma, :connections_pool}
{:pleroma, :connections_pool},
{:pleroma, :ldap}
]
defp reboot_time_subkeys,
@ -44,18 +45,13 @@ defmodule Pleroma.Config.TransferTask do
with {_, true} <- {:configurable, Config.get(:configurable_from_database)} do
# We need to restart applications for loaded settings take effect
{logger, other} =
settings =
(Repo.all(ConfigDB) ++ deleted_settings)
|> Enum.map(&merge_with_default/1)
|> Enum.split_with(fn {group, _, _, _} -> group in [:logger] end)
logger
|> Enum.sort()
|> Enum.each(&configure/1)
started_applications = Application.started_applications()
reject = [nil, :postgrex]
reject = [nil, :postgrex, :logger]
reject =
if restart_pleroma? do
@ -64,7 +60,7 @@ defmodule Pleroma.Config.TransferTask do
[:pleroma | reject]
end
other
settings
|> Enum.map(&update/1)
|> Enum.uniq()
|> Enum.reject(&(&1 in reject))
@ -102,38 +98,6 @@ defmodule Pleroma.Config.TransferTask do
{group, key, value, merged}
end
# change logger configuration in runtime, without restart
defp configure({_, :backends, _, merged}) do
# removing current backends
Enum.each(Application.get_env(:logger, :backends), &Logger.remove_backend/1)
Enum.each(merged, &Logger.add_backend/1)
:ok = update_env(:logger, :backends, merged)
end
defp configure({_, key, _, merged}) when key in [:console, :ex_syslogger] do
merged =
if key == :console do
put_in(merged[:format], merged[:format] <> "\n")
else
merged
end
backend =
if key == :ex_syslogger,
do: {ExSyslogger, :ex_syslogger},
else: key
Logger.configure_backend(backend, merged)
:ok = update_env(:logger, key, merged)
end
defp configure({_, key, _, merged}) do
Logger.configure([{key, merged}])
:ok = update_env(:logger, key, merged)
end
defp update({group, key, value, merged}) do
try do
:ok = update_env(group, key, merged)

66
lib/pleroma/config_db.ex
View file

@ -10,6 +10,7 @@ defmodule Pleroma.ConfigDB do
import Pleroma.Web.Gettext
alias __MODULE__
alias Pleroma.EctoType.Config.RateLimit
alias Pleroma.Repo
@type t :: %__MODULE__{}
@ -60,8 +61,59 @@ defmodule Pleroma.ConfigDB do
|> cast(params, [:key, :group, :value])
|> validate_required([:key, :group, :value])
|> unique_constraint(:key, name: :config_group_key_index)
|> validate_rate_limit()
end
defp validate_rate_limit(changeset) do
group = get_field(changeset, :group)
key = get_field(changeset, :key)
if group == :pleroma and key == :rate_limit do
value = get_field(changeset, :value)
case normalize_rate_limit(value) do
{:ok, normalized_value} ->
put_change(changeset, :value, normalized_value)
{:error, {limiter_name, reason}} ->
add_error(
changeset,
:value,
"invalid :rate_limit value for #{inspect(limiter_name)}: #{reason}"
)
end
else
changeset
end
end
defp normalize_rate_limit(nil), do: {:ok, nil}
defp normalize_rate_limit(%{} = value), do: normalize_rate_limit(Map.to_list(value))
defp normalize_rate_limit(value) when is_list(value) do
if Keyword.keyword?(value) do
value
|> Enum.reduce_while({:ok, []}, fn {limiter_name, limiter_value}, {:ok, acc} ->
case RateLimit.cast_with_error(limiter_value) do
{:ok, normalized_limiter_value} ->
{:cont, {:ok, [{limiter_name, normalized_limiter_value} | acc]}}
{:error, reason} ->
{:halt, {:error, {limiter_name, reason}}}
end
end)
|> case do
{:ok, acc} -> {:ok, Enum.reverse(acc)}
{:error, _} = error -> error
end
else
{:error, {:rate_limit, "must be a keyword list"}}
end
end
defp normalize_rate_limit(_), do: {:error, {:rate_limit, "must be a keyword list"}}
defp create(params) do
%ConfigDB{}
|> changeset(params)
@ -165,8 +217,7 @@ defmodule Pleroma.ConfigDB do
{:pleroma, :ecto_repos},
{:mime, :types},
{:cors_plug, [:max_age, :methods, :expose, :headers]},
{:swarm, :node_blacklist},
{:logger, :backends}
{:swarm, :node_blacklist}
]
Enum.any?(full_key_update, fn
@ -303,7 +354,7 @@ defmodule Pleroma.ConfigDB do
end
def to_elixir_types(%{"tuple" => entity}) do
Enum.reduce(entity, {}, &Tuple.append(&2, to_elixir_types(&1)))
Enum.reduce(entity, {}, &Tuple.insert_at(&2, tuple_size(&2), to_elixir_types(&1)))
end
def to_elixir_types(entity) when is_map(entity) do
@ -385,7 +436,12 @@ defmodule Pleroma.ConfigDB do
@spec module_name?(String.t()) :: boolean()
def module_name?(string) do
Regex.match?(~r/^(Pleroma|Phoenix|Tesla|Ueberauth|Swoosh)\./, string) or
string in ["Oban", "Ueberauth", "ExSyslogger", "ConcurrentLimiter"]
if String.contains?(string, ".") do
[name | _] = String.split(string, ".", parts: 2)
name in ~w[Pleroma Phoenix Tesla Ueberauth Swoosh Logger LoggerBackends]
else
string in ~w[Oban Ueberauth ExSyslogger ConcurrentLimiter]
end
end
end

56
lib/pleroma/constants.ex
View file

@ -20,13 +20,16 @@ defmodule Pleroma.Constants do
"deleted_activity_id",
"pleroma_internal",
"generator",
"rules"
"rules",
"language",
"voters",
"assigned_account"
]
)
const(static_only_files,
do:
~w(index.html robots.txt static static-fe finmoji emoji packs sounds images instance sw.js sw-pleroma.js favicon.png schemas doc embed.js embed.css)
~w(index.html robots.txt static static-fe finmoji emoji packs sounds images instance sw.js sw-pleroma.js schemas doc embed.js embed.css)
)
const(status_updatable_fields,
@ -36,10 +39,12 @@ defmodule Pleroma.Constants do
"updated",
"emoji",
"content",
"contentMap",
"summary",
"sensitive",
"attachment",
"generator"
"generator",
"language"
]
)
@ -85,12 +90,57 @@ defmodule Pleroma.Constants do
]
)
const(activity_types,
do: [
"Block",
"Create",
"Update",
"Delete",
"Follow",
"Accept",
"Reject",
"Add",
"Remove",
"Like",
"Dislike",
"Announce",
"Undo",
"Flag",
"EmojiReact",
"Listen"
]
)
const(allowed_activity_types_from_strangers,
do: [
"Block",
"Create",
"Flag",
"Follow",
"Like",
"Dislike",
"EmojiReact",
"Announce"
]
)
const(object_types,
do: ~w[Event Question Answer Audio Video Image Article Note Page ChatMessage]
)
# basic regex, just there to weed out potential mistakes
# https://datatracker.ietf.org/doc/html/rfc2045#section-5.1
const(mime_regex,
do: ~r/^[^[:cntrl:] ()<>@,;:\\"\/\[\]?=]+\/[^[:cntrl:] ()<>@,;:\\"\/\[\]?=]+(; .*)?$/
)
# List of allowed chars in the path segment of a URI
# unreserved, sub-delims, ":", "@" and "/" allowed as the separator in path
# https://datatracker.ietf.org/doc/html/rfc3986
const(uri_path_allowed_reserved_chars,
do: ~c"!$&'()*+,;=/:@"
)
const(upload_object_types, do: ["Document", "Image"])
const(activity_json_canonical_mime_type,

2
lib/pleroma/conversation/participation.ex
View file

@ -12,6 +12,8 @@ defmodule Pleroma.Conversation.Participation do
import Ecto.Changeset
import Ecto.Query
@type t :: %__MODULE__{}
schema "conversation_participations" do
belongs_to(:user, User, type: FlakeId.Ecto.CompatType)
belongs_to(:conversation, Conversation)

3
lib/pleroma/date_time.ex Normal file
View file

@ -0,0 +1,3 @@
defmodule Pleroma.DateTime do
@callback utc_now() :: NaiveDateTime.t()
end

6
lib/pleroma/date_time/impl.ex Normal file
View file

@ -0,0 +1,6 @@
defmodule Pleroma.DateTime.Impl do
@behaviour Pleroma.DateTime
@impl true
def utc_now, do: NaiveDateTime.utc_now()
end

23
lib/pleroma/docs/generator.ex
View file

@ -12,13 +12,27 @@ defmodule Pleroma.Docs.Generator do
@spec list_behaviour_implementations(behaviour :: module()) :: [module()]
def list_behaviour_implementations(behaviour) do
:code.all_loaded()
|> Enum.filter(fn {module, _} ->
# :code.all_loaded() no longer works on Elixir 1.19 and newer since modules are lazily loaded.
# We can get a complete list of modules by interating over Application.loaded_applications/0,
# but that includes every module which would get loaded below, which is very slow.
# Hardcode the Pleroma application instead, which is good enough for now.
application_modules =
case :application.get_key(:pleroma, :modules) do
{:ok, modules} -> modules
:undefined -> []
end
# At compile time or in tests some modules will be outside of the Pleroma application.
loaded_modules = Enum.map(:code.all_loaded(), fn {module, _} -> module end)
(application_modules ++ loaded_modules)
|> Enum.uniq()
|> Enum.filter(fn module ->
Code.ensure_loaded(module)
# This shouldn't be needed as all modules are expected to have module_info/1,
# but in test environments some transient modules `:elixir_compiler_XX`
# are loaded for some reason (where XX is a random integer).
Code.ensure_loaded(module)
if function_exported?(module, :module_info, 1) do
module.module_info(:attributes)
|> Keyword.get_values(:behaviour)
@ -26,7 +40,6 @@ defmodule Pleroma.Docs.Generator do
|> Enum.member?(behaviour)
end
end)
|> Enum.map(fn {module, _} -> module end)
end
@doc """

8
lib/pleroma/ecto_enums.ex
View file

@ -27,11 +27,3 @@ defenum(Pleroma.DataMigration.State,
failed: 4,
manual: 5
)
defenum(Pleroma.User.Backup.State,
pending: 1,
running: 2,
complete: 3,
failed: 4,
invalid: 5
)

49
lib/pleroma/ecto_type/activity_pub/object_validators/content_language_map.ex Normal file
View file

@ -0,0 +1,49 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2023 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.EctoType.ActivityPub.ObjectValidators.ContentLanguageMap do
use Ecto.Type
import Pleroma.EctoType.ActivityPub.ObjectValidators.LanguageCode,
only: [good_locale_code?: 1]
def type, do: :map
def cast(%{} = object) do
with {status, %{} = data} when status in [:modified, :ok] <- validate_map(object) do
{:ok, data}
else
{_, nil} -> {:ok, nil}
{:error, _} -> :error
end
end
def cast(_), do: :error
def dump(data), do: {:ok, data}
def load(data), do: {:ok, data}
defp validate_map(%{} = object) do
{status, data} =
object
|> Enum.reduce({:ok, %{}}, fn
{lang, value}, {status, acc} when is_binary(lang) and is_binary(value) ->
if good_locale_code?(lang) do
{status, Map.put(acc, lang, value)}
else
{:modified, acc}
end
_, {_status, acc} ->
{:modified, acc}
end)
if data == %{} do
{status, nil}
else
{status, data}
end
end
end

27
lib/pleroma/ecto_type/activity_pub/object_validators/language_code.ex Normal file
View file

@ -0,0 +1,27 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2023 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.EctoType.ActivityPub.ObjectValidators.LanguageCode do
use Ecto.Type
def type, do: :string
def cast(language) when is_binary(language) do
if good_locale_code?(language) do
{:ok, language}
else
{:error, :invalid_language}
end
end
def cast(_), do: :error
def dump(data), do: {:ok, data}
def load(data), do: {:ok, data}
def good_locale_code?(code) when is_binary(code), do: code =~ ~r<^[a-zA-Z0-9\-]+\z$>
def good_locale_code?(_code), do: false
end

71
lib/pleroma/ecto_type/config/rate_limit.ex Normal file
View file

@ -0,0 +1,71 @@
# Pleroma: A lightweight social networking server
# Copyright © Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.EctoType.Config.RateLimit do
@moduledoc false
use Ecto.Type
@type t ::
nil
| {non_neg_integer(), non_neg_integer()}
| [{non_neg_integer(), non_neg_integer()}]
@impl true
def type, do: :term
@impl true
def cast(value) do
case cast_with_error(value) do
{:ok, normalized} -> {:ok, normalized}
{:error, _reason} -> :error
end
end
@impl true
def load(value), do: cast(value)
@impl true
def dump(value), do: cast(value)
@spec cast_with_error(term()) :: {:ok, t()} | {:error, String.t()}
def cast_with_error(nil), do: {:ok, nil}
def cast_with_error({scale, limit}) do
with {:ok, scale} <- parse_integer(scale, "scale"),
{:ok, limit} <- parse_integer(limit, "limit"),
true <- scale >= 1 and limit >= 1 do
{:ok, {scale, limit}}
else
false -> {:error, "scale and limit must be >= 1"}
{:error, reason} -> {:error, reason}
end
end
def cast_with_error([{_, _} = unauth, {_, _} = auth]) do
with {:ok, unauth} <- cast_with_error(unauth),
{:ok, auth} <- cast_with_error(auth) do
{:ok, [unauth, auth]}
else
{:error, reason} -> {:error, reason}
end
end
def cast_with_error(_),
do:
{:error, "must be a {scale, limit} tuple, a [{scale, limit}, {scale, limit}] list, or nil"}
defp parse_integer(value, _label) when is_integer(value), do: {:ok, value}
defp parse_integer(value, label) when is_binary(value) do
value = String.trim(value)
case Integer.parse(value) do
{number, ""} -> {:ok, number}
_ -> {:error, "#{label} must be an integer"}
end
end
defp parse_integer(_value, label), do: {:error, "#{label} must be an integer"}
end

3
lib/pleroma/emails/mailer.ex
View file

@ -25,7 +25,8 @@ defmodule Pleroma.Emails.Mailer do
|> :erlang.term_to_binary()
|> Base.encode64()
MailerWorker.enqueue("email", %{"encoded_email" => encoded_email, "config" => config})
MailerWorker.new(%{"op" => "email", "encoded_email" => encoded_email, "config" => config})
|> Oban.insert()
end
@doc "callback to perform send email from queue"

35
lib/pleroma/emails/user_email.ex
View file

@ -345,37 +345,22 @@ defmodule Pleroma.Emails.UserEmail do
Router.Helpers.subscription_url(Endpoint, :unsubscribe, token)
end
def backup_is_ready_email(backup, admin_user_id \\ nil) do
def backup_is_ready_email(backup) do
%{user: user} = Pleroma.Repo.preload(backup, :user)
Gettext.with_locale_or_default user.language do
download_url = Pleroma.Web.PleromaAPI.BackupView.download_url(backup)
html_body =
if is_nil(admin_user_id) do
Gettext.dpgettext(
"static_pages",
"account archive email body - self-requested",
"""
<p>You requested a full backup of your Pleroma account. It's ready for download:</p>
<p><a href="%{download_url}">%{download_url}</a></p>
""",
download_url: download_url
)
else
admin = Pleroma.Repo.get(User, admin_user_id)
Gettext.dpgettext(
"static_pages",
"account archive email body - admin requested",
"""
<p>Admin @%{admin_nickname} requested a full backup of your Pleroma account. It's ready for download:</p>
<p><a href="%{download_url}">%{download_url}</a></p>
""",
admin_nickname: admin.nickname,
download_url: download_url
)
end
Gettext.dpgettext(
"static_pages",
"account archive email body",
"""
<p>A full backup of your Pleroma account was requested. It's ready for download:</p>
<p><a href="%{download_url}">%{download_url}</a></p>
""",
download_url: download_url
)
new()
|> to(recipient(user))

30
lib/pleroma/emoji.ex
View file

@ -11,6 +11,8 @@ defmodule Pleroma.Emoji do
alias Pleroma.Emoji.Combinations
alias Pleroma.Emoji.Loader
alias Pleroma.Utils.URIEncoding
alias Pleroma.Web.Endpoint
require Logger
@ -189,6 +191,34 @@ defmodule Pleroma.Emoji do
def emoji_url(_), do: nil
@spec local_url(String.t() | nil) :: String.t() | nil
def local_url(nil), do: nil
def local_url("http" <> _ = url) do
URIEncoding.encode_url(url)
end
def local_url("/" <> _ = path) do
path = URIEncoding.encode_url(path, bypass_parse: true, bypass_decode: true)
Endpoint.url() <> path
end
def local_url(path) when is_binary(path) do
local_url("/" <> path)
end
def build_emoji_tag({name, url}) do
url = URIEncoding.encode_url(url)
%{
"icon" => %{"url" => "#{url}", "type" => "Image"},
"name" => ":" <> name <> ":",
"type" => "Emoji",
"updated" => "1970-01-01T00:00:00Z",
"id" => url
}
end
def emoji_name_with_instance(name, url) do
url = url |> URI.parse() |> Map.get(:host)
"#{name}@#{url}"

3
lib/pleroma/emoji/formatter.ex
View file

@ -5,7 +5,6 @@
defmodule Pleroma.Emoji.Formatter do
alias Pleroma.Emoji
alias Pleroma.HTML
alias Pleroma.Web.Endpoint
alias Pleroma.Web.MediaProxy
def emojify(text) do
@ -44,7 +43,7 @@ defmodule Pleroma.Emoji.Formatter do
Emoji.get_all()
|> Enum.filter(fn {emoji, %Emoji{}} -> String.contains?(text, ":#{emoji}:") end)
|> Enum.reduce(%{}, fn {name, %Emoji{file: file}}, acc ->
Map.put(acc, name, to_string(URI.merge(Endpoint.url(), file)))
Map.put(acc, name, Emoji.local_url(file))
end)
end

198
lib/pleroma/emoji/pack.ex
View file

@ -24,12 +24,13 @@ defmodule Pleroma.Emoji.Pack do
alias Pleroma.Emoji
alias Pleroma.Emoji.Pack
alias Pleroma.SafeZip
alias Pleroma.Utils
@spec create(String.t()) :: {:ok, t()} | {:error, File.posix()} | {:error, :empty_values}
def create(name) do
with :ok <- validate_not_empty([name]),
dir <- Path.join(emoji_path(), name),
dir <- path_join_name_safe(emoji_path(), name),
:ok <- File.mkdir(dir) do
save_pack(%__MODULE__{pack_file: Path.join(dir, "pack.json")})
end
@ -65,43 +66,21 @@ defmodule Pleroma.Emoji.Pack do
{:ok, [binary()]} | {:error, File.posix(), binary()} | {:error, :empty_values}
def delete(name) do
with :ok <- validate_not_empty([name]),
pack_path <- Path.join(emoji_path(), name) do
pack_path <- path_join_name_safe(emoji_path(), name) do
File.rm_rf(pack_path)
end
end
@spec unpack_zip_emojies(list(tuple())) :: list(map())
defp unpack_zip_emojies(zip_files) do
Enum.reduce(zip_files, [], fn
{_, path, s, _, _, _}, acc when elem(s, 2) == :regular ->
with(
filename <- Path.basename(path),
shortcode <- Path.basename(filename, Path.extname(filename)),
false <- Emoji.exist?(shortcode)
) do
[%{path: path, filename: path, shortcode: shortcode} | acc]
else
_ -> acc
end
_, acc ->
acc
end)
end
@spec add_file(t(), String.t(), Path.t(), Plug.Upload.t()) ::
{:ok, t()}
| {:error, File.posix() | atom()}
def add_file(%Pack{} = pack, _, _, %Plug.Upload{content_type: "application/zip"} = file) do
with {:ok, zip_files} <- :zip.table(to_charlist(file.path)),
[_ | _] = emojies <- unpack_zip_emojies(zip_files),
with {:ok, zip_files} <- SafeZip.list_dir_file(file.path),
[_ | _] = emojies <- map_zip_emojies(zip_files),
{:ok, tmp_dir} <- Utils.tmp_dir("emoji") do
try do
{:ok, _emoji_files} =
:zip.unzip(
to_charlist(file.path),
[{:file_list, Enum.map(emojies, & &1[:path])}, {:cwd, String.to_charlist(tmp_dir)}]
)
SafeZip.unzip_file(file.path, tmp_dir, Enum.map(emojies, & &1[:path]))
{_, updated_pack} =
Enum.map_reduce(emojies, pack, fn item, emoji_pack ->
@ -246,6 +225,97 @@ defmodule Pleroma.Emoji.Pack do
end
end
def download_zip(name, opts \\ %{}) do
with :ok <- validate_not_empty([name]),
:ok <- validate_new_pack(name),
{:ok, archive_data} <- fetch_archive_data(opts),
pack_path <- path_join_name_safe(emoji_path(), name),
:ok <- create_pack_dir(pack_path),
:ok <- safe_unzip(archive_data, pack_path) do
ensure_pack_json(pack_path, archive_data, opts)
else
{:error, :empty_values} -> {:error, "Pack name cannot be empty"}
{:error, reason} when is_binary(reason) -> {:error, reason}
_ -> {:error, "Could not process pack"}
end
end
defp create_pack_dir(pack_path) do
case File.mkdir_p(pack_path) do
:ok -> :ok
{:error, _} -> {:error, "Could not create the pack directory"}
end
end
defp safe_unzip(archive_data, pack_path) do
case SafeZip.unzip_data(archive_data, pack_path) do
{:ok, _} -> :ok
{:error, reason} when is_binary(reason) -> {:error, reason}
_ -> {:error, "Could not unzip pack"}
end
end
defp validate_new_pack(name) do
pack_path = path_join_name_safe(emoji_path(), name)
if File.exists?(pack_path) do
{:error, "Pack already exists, refusing to import #{name}"}
else
:ok
end
end
defp fetch_archive_data(%{url: url}) do
case Pleroma.HTTP.get(url) do
{:ok, %{status: 200, body: data}} -> {:ok, data}
_ -> {:error, "Could not download pack"}
end
end
defp fetch_archive_data(%{file: %Plug.Upload{path: path}}) do
case File.read(path) do
{:ok, data} -> {:ok, data}
_ -> {:error, "Could not read the uploaded pack file"}
end
end
defp fetch_archive_data(_) do
{:error, "Neither file nor URL was present in the request"}
end
defp ensure_pack_json(pack_path, archive_data, opts) do
pack_json_path = Path.join(pack_path, "pack.json")
if not File.exists?(pack_json_path) do
create_pack_json(pack_path, pack_json_path, archive_data, opts)
end
:ok
end
defp create_pack_json(pack_path, pack_json_path, archive_data, opts) do
emoji_map =
Pleroma.Emoji.Loader.make_shortcode_to_file_map(
pack_path,
Map.get(opts, :exts, [".png", ".gif", ".jpg"])
)
archive_sha = :crypto.hash(:sha256, archive_data) |> Base.encode16()
pack_json = %{
pack: %{
license: Map.get(opts, :license, ""),
homepage: Map.get(opts, :homepage, ""),
description: Map.get(opts, :description, ""),
src: Map.get(opts, :url),
src_sha256: archive_sha
},
files: emoji_map
}
File.write!(pack_json_path, Jason.encode!(pack_json, pretty: true))
end
@spec download(String.t(), String.t(), String.t()) :: {:ok, t()} | {:error, atom()}
def download(name, url, as) do
uri = url |> String.trim() |> URI.parse()
@ -292,7 +362,7 @@ defmodule Pleroma.Emoji.Pack do
@spec load_pack(String.t()) :: {:ok, t()} | {:error, :file.posix()}
def load_pack(name) do
name = Path.basename(name)
pack_file = Path.join([emoji_path(), name, "pack.json"])
pack_file = path_join_name_safe(emoji_path(), name) |> Path.join("pack.json")
with {:ok, _} <- File.stat(pack_file),
{:ok, pack_data} <- File.read(pack_file) do
@ -416,10 +486,9 @@ defmodule Pleroma.Emoji.Pack do
end
defp create_archive_and_cache(pack, hash) do
files = ['pack.json' | Enum.map(pack.files, fn {_, file} -> to_charlist(file) end)]
{:ok, {_, result}} =
:zip.zip('#{pack.name}.zip', files, [:memory, cwd: to_charlist(pack.path)])
pack_file_list = Enum.into(pack.files, [], fn {_, f} -> f end)
files = ["pack.json" | pack_file_list]
{:ok, {_, result}} = SafeZip.zip("#{pack.name}.zip", files, pack.path, true)
ttl_per_file = Pleroma.Config.get!([:emoji, :shared_pack_cache_seconds_per_file])
overall_ttl = :timer.seconds(ttl_per_file * Enum.count(files))
@ -478,7 +547,7 @@ defmodule Pleroma.Emoji.Pack do
end
defp save_file(%Plug.Upload{path: upload_path}, pack, filename) do
file_path = Path.join(pack.path, filename)
file_path = path_join_safe(pack.path, filename)
create_subdirs(file_path)
with {:ok, _} <- File.copy(upload_path, file_path) do
@ -497,8 +566,8 @@ defmodule Pleroma.Emoji.Pack do
end
defp rename_file(pack, filename, new_filename) do
old_path = Path.join(pack.path, filename)
new_path = Path.join(pack.path, new_filename)
old_path = path_join_safe(pack.path, filename)
new_path = path_join_safe(pack.path, new_filename)
create_subdirs(new_path)
with :ok <- File.rename(old_path, new_path) do
@ -510,13 +579,13 @@ defmodule Pleroma.Emoji.Pack do
with true <- String.contains?(file_path, "/"),
path <- Path.dirname(file_path),
false <- File.exists?(path) do
File.mkdir_p!(path)
Pleroma.Backports.mkdir_p!(path)
end
end
defp remove_file(pack, shortcode) do
with {:ok, filename} <- get_filename(pack, shortcode),
emoji <- Path.join(pack.path, filename),
emoji <- path_join_safe(pack.path, filename),
:ok <- File.rm(emoji) do
remove_dir_if_empty(emoji, filename)
end
@ -534,7 +603,7 @@ defmodule Pleroma.Emoji.Pack do
defp get_filename(pack, shortcode) do
with %{^shortcode => filename} when is_binary(filename) <- pack.files,
file_path <- Path.join(pack.path, filename),
file_path <- path_join_safe(pack.path, filename),
{:ok, _} <- File.stat(file_path) do
{:ok, filename}
else
@ -558,7 +627,7 @@ defmodule Pleroma.Emoji.Pack do
emoji_path = emoji_path()
# Create the directory first if it does not exist. This is probably the first request made
# with the API so it should be sufficient
with {:create_dir, :ok} <- {:create_dir, File.mkdir_p(emoji_path)},
with {:create_dir, :ok} <- {:create_dir, Pleroma.Backports.mkdir_p(emoji_path)},
{:ls, {:ok, results}} <- {:ls, File.ls(emoji_path)} do
{:ok, Enum.sort(results)}
else
@ -583,12 +652,11 @@ defmodule Pleroma.Emoji.Pack do
end
defp unzip(archive, pack_info, remote_pack, local_pack) do
with :ok <- File.mkdir_p!(local_pack.path) do
files = Enum.map(remote_pack["files"], fn {_, path} -> to_charlist(path) end)
with :ok <- Pleroma.Backports.mkdir_p!(local_pack.path) do
files = Enum.map(remote_pack["files"], fn {_, path} -> path end)
# Fallback cannot contain a pack.json file
files = if pack_info[:fallback], do: files, else: ['pack.json' | files]
:zip.unzip(archive, cwd: to_charlist(local_pack.path), file_list: files)
files = if pack_info[:fallback], do: files, else: ["pack.json" | files]
SafeZip.unzip_data(archive, local_pack.path, files)
end
end
@ -649,13 +717,43 @@ defmodule Pleroma.Emoji.Pack do
end
defp validate_has_all_files(pack, zip) do
with {:ok, f_list} <- :zip.unzip(zip, [:memory]) do
# Check if all files from the pack.json are in the archive
pack.files
|> Enum.all?(fn {_, from_manifest} ->
List.keyfind(f_list, to_charlist(from_manifest), 0)
# Check if all files from the pack.json are in the archive
eset =
Enum.reduce(pack.files, MapSet.new(), fn
{_, file}, s -> MapSet.put(s, to_charlist(file))
end)
|> if(do: :ok, else: {:error, :incomplete})
if SafeZip.contains_all_data?(zip, eset),
do: :ok,
else: {:error, :incomplete}
end
defp path_join_name_safe(dir, name) do
if to_string(name) != Path.basename(name) or name in ["..", ".", ""] do
raise "Invalid or malicious pack name: #{name}"
else
Path.join(dir, name)
end
end
defp path_join_safe(dir, path) do
{:ok, safe_path} = Path.safe_relative(path)
Path.join(dir, safe_path)
end
defp map_zip_emojies(zip_files) do
Enum.reduce(zip_files, [], fn path, acc ->
with(
filename <- Path.basename(path),
shortcode <- Path.basename(filename, Path.extname(filename)),
# note: this only checks the shortcode, if an emoji already exists on the same path, but
# with a different shortcode, the existing one will be degraded to an alias of the new
false <- Emoji.exist?(shortcode)
) do
[%{path: path, filename: path, shortcode: shortcode} | acc]
else
_ -> acc
end
end)
end
end

11
lib/pleroma/filter.ex
View file

@ -133,10 +133,13 @@ defmodule Pleroma.Filter do
defp maybe_add_expires_at(changeset, _), do: changeset
defp maybe_add_expiration_job(%{expires_at: %NaiveDateTime{} = expires_at} = filter) do
Pleroma.Workers.PurgeExpiredFilter.enqueue(%{
filter_id: filter.id,
expires_at: DateTime.from_naive!(expires_at, "Etc/UTC")
})
Pleroma.Workers.PurgeExpiredFilter.new(
%{
filter_id: filter.id
},
scheduled_at: DateTime.from_naive!(expires_at, "Etc/UTC")
)
|> Oban.insert()
end
defp maybe_add_expiration_job(_), do: {:ok, nil}

22
lib/pleroma/following_relationship.ex
View file

@ -147,14 +147,22 @@ defmodule Pleroma.FollowingRelationship do
|> Repo.aggregate(:count, :id)
end
def get_follow_requests(%User{id: id}) do
def get_follow_requests_query(%User{id: id}) do
__MODULE__
|> join(:inner, [r], f in assoc(r, :follower))
|> join(:inner, [r], f in assoc(r, :follower), as: :follower)
|> where([r], r.state == ^:follow_pending)
|> where([r], r.following_id == ^id)
|> where([r, f], f.is_active == true)
|> select([r, f], f)
|> Repo.all()
|> where([r, follower: f], f.is_active == true)
|> select([r, follower: f], f)
end
def get_outgoing_follow_requests_query(%User{id: id}) do
__MODULE__
|> join(:inner, [r], f in assoc(r, :following), as: :following)
|> where([r], r.state == ^:follow_pending)
|> where([r], r.follower_id == ^id)
|> where([r, following: f], f.is_active == true)
|> select([r, following: f], f)
end
def following?(%User{id: follower_id}, %User{id: followed_id}) do
@ -199,8 +207,8 @@ defmodule Pleroma.FollowingRelationship do
|> preload([:follower])
|> Repo.all()
|> Enum.map(fn following_relationship ->
Pleroma.Web.CommonAPI.follow(following_relationship.follower, target)
Pleroma.Web.CommonAPI.unfollow(following_relationship.follower, origin)
Pleroma.Web.CommonAPI.follow(target, following_relationship.follower)
Pleroma.Web.CommonAPI.unfollow(origin, following_relationship.follower)
end)
|> case do
[] ->

11
lib/pleroma/formatter.ex
View file

@ -127,6 +127,13 @@ defmodule Pleroma.Formatter do
Earmark.as_html!(text, %Earmark.Options{compact_output: true, smartypants: false})
end
def markdown_to_html(text, opts) do
Earmark.as_html!(
text,
%Earmark.Options{compact_output: true, smartypants: false} |> Map.merge(opts)
)
end
def html_escape({text, mentions, hashtags}, type) do
{html_escape(text, type), mentions, hashtags}
end
@ -135,6 +142,10 @@ defmodule Pleroma.Formatter do
HTML.filter_tags(text)
end
def html_escape(text, "text/x.misskeymarkdown") do
HTML.filter_tags(text)
end
def html_escape(text, "text/plain") do
Regex.split(@link_regex, text, include_captures: true)
|> Enum.map_every(2, fn chunk ->

27
lib/pleroma/frontend.ex
View file

@ -43,10 +43,6 @@ defmodule Pleroma.Frontend do
{:download_or_unzip, _} ->
Logger.info("Could not download or unzip the frontend")
{:error, "Could not download or unzip the frontend"}
_e ->
Logger.info("Could not install the frontend")
{:error, "Could not install the frontend"}
end
end
@ -69,27 +65,18 @@ defmodule Pleroma.Frontend do
end
def unzip(zip, dest) do
with {:ok, unzipped} <- :zip.unzip(zip, [:memory]) do
File.rm_rf!(dest)
File.mkdir_p!(dest)
File.rm_rf!(dest)
Pleroma.Backports.mkdir_p!(dest)
Enum.each(unzipped, fn {filename, data} ->
path = filename
new_file_path = Path.join(dest, path)
new_file_path
|> Path.dirname()
|> File.mkdir_p!()
File.write!(new_file_path, data)
end)
case Pleroma.SafeZip.unzip_data(zip, dest) do
{:ok, _} -> :ok
error -> error
end
end
defp download_build(frontend_info, dest) do
Logger.info("Downloading pre-built bundle for #{frontend_info["name"]}")
url = String.replace(frontend_info["build_url"], "${ref}", frontend_info["ref"])
Logger.info("Downloading pre-built bundle for #{frontend_info["name"]} from #{url}")
with {:ok, %{status: 200, body: zip_body}} <-
Pleroma.HTTP.get(url, [], pool: :media, recv_timeout: 120_000) do
@ -103,7 +90,7 @@ defmodule Pleroma.Frontend do
defp install_frontend(frontend_info, source, dest) do
from = frontend_info["build_dir"] || "dist"
File.rm_rf!(dest)
File.mkdir_p!(dest)
Pleroma.Backports.mkdir_p!(dest)
File.cp_r!(Path.join([source, from]), dest)
:ok
end

37
lib/pleroma/gopher/server.ex
View file

@ -21,17 +21,28 @@ defmodule Pleroma.Gopher.Server do
def init([ip, port]) do
Logger.info("Starting gopher server on #{port}")
Process.flag(:trap_exit, true)
:ranch.start_listener(
:gopher,
100,
:ranch_tcp,
[ip: ip, port: port],
__MODULE__.ProtocolHandler,
[]
)
listener = :gopher
{:ok, %{ip: ip, port: port}}
{:ok, _pid} =
:ranch.start_listener(
listener,
:ranch_tcp,
%{
num_acceptors: 100,
max_connections: 100,
socket_opts: [ip: ip, port: port]
},
__MODULE__.ProtocolHandler,
[]
)
{:ok, %{ip: ip, port: port, listener: listener}}
end
def terminate(_reason, state) do
:ranch.stop_listener(state.listener)
end
end
@ -43,13 +54,13 @@ defmodule Pleroma.Gopher.Server.ProtocolHandler do
alias Pleroma.Web.ActivityPub.ActivityPub
alias Pleroma.Web.ActivityPub.Visibility
def start_link(ref, socket, transport, opts) do
pid = spawn_link(__MODULE__, :init, [ref, socket, transport, opts])
def start_link(ref, transport, opts) do
pid = spawn_link(__MODULE__, :init, [ref, transport, opts])
{:ok, pid}
end
def init(ref, socket, transport, [] = _Opts) do
:ok = :ranch.accept_ack(ref)
def init(ref, transport, opts \\ []) do
{:ok, socket} = :ranch.handshake(ref, opts)
loop(socket, transport)
end

9
lib/pleroma/gun.ex
View file

@ -6,9 +6,12 @@ defmodule Pleroma.Gun do
@callback open(charlist(), pos_integer(), map()) :: {:ok, pid()}
@callback info(pid()) :: map()
@callback close(pid()) :: :ok
@callback cancel(pid(), reference() | [reference()]) :: :ok
@callback await_up(pid, pos_integer()) :: {:ok, atom()} | {:error, atom()}
@callback connect(pid(), map()) :: reference()
@callback await(pid(), reference()) :: {:response, :fin, 200, []}
@callback await(pid(), reference()) :: tuple()
@callback await_tunnel_up(pid(), reference() | :undefined, timeout()) ::
{:ok, atom()} | {:error, term()}
@callback set_owner(pid(), pid()) :: :ok
defp api, do: Pleroma.Config.get([Pleroma.Gun], Pleroma.Gun.API)
@ -19,11 +22,15 @@ defmodule Pleroma.Gun do
def close(pid), do: api().close(pid)
def cancel(pid, stream), do: api().cancel(pid, stream)
def await_up(pid, timeout \\ 5_000), do: api().await_up(pid, timeout)
def connect(pid, opts), do: api().connect(pid, opts)
def await(pid, ref), do: api().await(pid, ref)
def await_tunnel_up(pid, ref, timeout), do: api().await_tunnel_up(pid, ref, timeout)
def set_owner(pid, owner), do: api().set_owner(pid, owner)
end

25
lib/pleroma/gun/api.ex
View file

@ -32,6 +32,9 @@ defmodule Pleroma.Gun.API do
@impl Gun
defdelegate close(pid), to: :gun
@impl Gun
defdelegate cancel(pid, stream), to: :gun
@impl Gun
defdelegate await_up(pid, timeout \\ 5_000), to: :gun
@ -41,6 +44,28 @@ defmodule Pleroma.Gun.API do
@impl Gun
defdelegate await(pid, ref), to: :gun
@impl Gun
def await_tunnel_up(pid, stream_ref, timeout) do
monitor_ref = Process.monitor(pid)
result =
receive do
{:gun_tunnel_up, ^pid, ^stream_ref, protocol} ->
{:ok, protocol}
{:gun_down, ^pid, _protocol, reason, _killed_streams} ->
{:error, {:down, reason}}
{:DOWN, ^monitor_ref, :process, ^pid, reason} ->
{:error, {:down, reason}}
after
timeout -> {:error, :timeout}
end
Process.demonitor(monitor_ref, [:flush])
result
end
@impl Gun
defdelegate set_owner(pid, owner), to: :gun
end

139
lib/pleroma/gun/conn.ex
View file

@ -42,18 +42,34 @@ defmodule Pleroma.Gun.Conn do
Map.put(opts, :tls_opts, tls_opts)
end
defp do_open(%URI{scheme: "http"} = uri, %{proxy: {proxy_host, proxy_port}} = opts) do
with open_opts <- opts |> proxy_open_opts() |> Map.put(:protocols, [:http]),
{:ok, conn} <- Gun.open(proxy_host, proxy_port, open_opts),
{:ok, protocol} <- await_up(conn, opts[:connect_timeout]) do
{:ok, conn, protocol, :forward_proxy}
else
error ->
Logger.warning(
"Opening forward proxy connection to #{compose_uri_log(uri)} failed with error #{inspect(error)}"
)
error
end
end
defp do_open(uri, %{proxy: {proxy_host, proxy_port}} = opts) do
connect_opts =
uri
|> destination_opts()
|> add_http2_opts(uri.scheme, Map.get(opts, :tls_opts, []))
|> add_proxy_auth(opts)
with open_opts <- Map.delete(opts, :tls_opts),
with open_opts <- proxy_open_opts(opts),
{:ok, conn} <- Gun.open(proxy_host, proxy_port, open_opts),
{:ok, protocol} <- Gun.await_up(conn, opts[:connect_timeout]),
stream <- Gun.connect(conn, connect_opts),
{:response, :fin, 200, _} <- Gun.await(conn, stream) do
{:ok, conn, protocol}
{:ok, _proxy_protocol} <- await_up(conn, opts[:connect_timeout]) do
conn
|> Gun.connect(connect_opts)
|> await_proxy_connect(conn, opts[:connect_timeout])
else
error ->
Logger.warning(
@ -64,30 +80,29 @@ defmodule Pleroma.Gun.Conn do
end
end
defp do_open(uri, %{proxy: {proxy_type, proxy_host, proxy_port}} = opts) do
version =
proxy_type
|> to_string()
|> String.last()
|> case do
"4" -> 4
_ -> 5
end
defp do_open(_uri, %{proxy: {:socks4, _proxy_host, _proxy_port}}) do
{:error, :socks4_unsupported}
end
defp do_open(uri, %{proxy: {proxy_type, proxy_host, proxy_port}} = opts)
when proxy_type in [:socks, :socks5] do
socks_opts =
uri
|> destination_opts()
|> add_http2_opts(uri.scheme, Map.get(opts, :tls_opts, []))
|> Map.put(:version, version)
|> Map.put(:version, 5)
|> add_socks_proxy_auth(opts)
opts =
open_opts =
opts
|> proxy_open_opts()
|> Map.put(:protocols, [:socks])
|> Map.put(:socks_opts, socks_opts)
with {:ok, conn} <- Gun.open(proxy_host, proxy_port, opts),
{:ok, protocol} <- Gun.await_up(conn, opts[:connect_timeout]) do
{:ok, conn, protocol}
with {:ok, conn} <- Gun.open(proxy_host, proxy_port, open_opts),
{:ok, :socks} <- await_up(conn, opts[:connect_timeout]),
{:ok, protocol} <- await_tunnel_up(conn, :undefined, opts[:connect_timeout]) do
{:ok, conn, protocol, nil}
else
error ->
Logger.warning(
@ -100,10 +115,11 @@ defmodule Pleroma.Gun.Conn do
defp do_open(%URI{host: host, port: port} = uri, opts) do
host = Pleroma.HTTP.AdapterHelper.parse_host(host)
opts = Map.put(opts, :transport, transport(uri.scheme))
with {:ok, conn} <- Gun.open(host, port, opts),
{:ok, protocol} <- Gun.await_up(conn, opts[:connect_timeout]) do
{:ok, conn, protocol}
{:ok, protocol} <- await_up(conn, opts[:connect_timeout]) do
{:ok, conn, protocol, nil}
else
error ->
Logger.warning(
@ -120,11 +136,90 @@ defmodule Pleroma.Gun.Conn do
end
defp add_http2_opts(opts, "https", tls_opts) do
Map.merge(opts, %{protocols: [:http2], transport: :tls, tls_opts: tls_opts})
Map.merge(opts, %{protocols: [:http2, :http], transport: :tls, tls_opts: tls_opts})
end
defp add_http2_opts(opts, _, _), do: opts
defp add_proxy_auth(connect_opts, %{proxy_auth: {username, password}})
when is_binary(username) and is_binary(password) do
Map.merge(connect_opts, %{username: username, password: password})
end
defp add_proxy_auth(connect_opts, _), do: connect_opts
defp add_socks_proxy_auth(socks_opts, %{proxy_auth: {username, password}})
when is_binary(username) and is_binary(password) do
Map.put(socks_opts, :auth, [{:username_password, username, password}])
end
defp add_socks_proxy_auth(socks_opts, _), do: socks_opts
defp proxy_open_opts(opts) do
proxy_tls_opts = Map.get(opts, :proxy_tls_opts, [])
opts =
opts
|> Map.drop([:proxy, :proxy_auth, :proxy_tls_opts, :tls_opts])
|> Map.put_new(:transport, :tcp)
if opts.transport == :tls do
Map.put(opts, :tls_opts, proxy_tls_opts)
else
opts
end
end
defp await_up(conn, timeout) do
case Gun.await_up(conn, timeout) do
{:ok, protocol} ->
{:ok, protocol}
error ->
Gun.close(conn)
error
end
end
defp await_proxy_connect(stream, conn, timeout) do
case Gun.await(conn, stream) do
{:response, :fin, 200, _headers} ->
case await_tunnel_up(conn, stream, timeout) do
{:ok, protocol} -> {:ok, conn, protocol, stream}
error -> error
end
{:response, _fin, 407, _headers} ->
close_with_error(conn, :proxy_auth_failed)
{:response, _fin, status, _headers} when is_integer(status) ->
close_with_error(conn, {:proxy_connect_failed, status})
error ->
Gun.close(conn)
error
end
end
defp close_with_error(conn, reason) do
Gun.close(conn)
{:error, reason}
end
defp transport("https"), do: :tls
defp transport(_), do: :tcp
defp await_tunnel_up(conn, stream_ref, timeout) do
case Gun.await_tunnel_up(conn, stream_ref, timeout) do
{:ok, protocol} ->
{:ok, protocol}
error ->
Gun.close(conn)
error
end
end
def compose_uri_log(%URI{scheme: scheme, host: host, path: path}) do
"#{scheme}://#{host}#{path}"
end

225
lib/pleroma/gun/connection_pool.ex
View file

@ -4,53 +4,132 @@
defmodule Pleroma.Gun.ConnectionPool do
@registry __MODULE__
@locks Module.concat(__MODULE__, Locks)
alias Pleroma.Gun.ConnectionPool.WorkerSupervisor
@connection_option_keys [
:http_opts,
:http2_opts,
:protocols,
:proxy,
:proxy_auth,
:proxy_tls_opts,
:retry,
:retry_timeout,
:socks_opts,
:tcp_opts,
:tls_opts,
:transport,
:ws_opts
]
def children do
[
{Registry, keys: :unique, name: @registry},
{Registry, keys: :duplicate, name: @registry},
{Registry, keys: :unique, name: @locks},
Pleroma.Gun.ConnectionPool.WorkerSupervisor
]
end
@spec get_conn(URI.t(), keyword()) :: {:ok, pid()} | {:error, term()}
def get_conn(uri, opts) do
key = "#{uri.scheme}:#{uri.host}:#{uri.port}"
key = connection_key(uri, opts)
case Registry.lookup(@registry, key) do
# The key has already been registered, but connection is not up yet
[{worker_pid, nil}] ->
get_gun_pid_from_worker(worker_pid, true)
[{worker_pid, {gun_pid, _used_by, _crf, _last_reference}}] ->
GenServer.call(worker_pid, :add_client)
case checkout_existing(Registry.lookup(@registry, key)) do
{:ok, gun_pid} ->
{:ok, gun_pid}
[] ->
# :gun.set_owner fails in :connected state for whatevever reason,
# so we open the connection in the process directly and send it's pid back
# We trust gun to handle timeouts by itself
case WorkerSupervisor.start_worker([key, uri, opts, self()]) do
{:ok, worker_pid} ->
get_gun_pid_from_worker(worker_pid, false)
:not_available ->
provision_or_wait(key, uri, opts)
end
end
{:error, {:already_started, worker_pid}} ->
get_gun_pid_from_worker(worker_pid, true)
defp checkout_existing([]), do: :not_available
err ->
err
defp checkout_existing([{_worker_pid, nil} | workers]), do: checkout_existing(workers)
defp checkout_existing([{worker_pid, _value} | workers]) do
case checkout_worker(worker_pid) do
{:ok, gun_pid} -> {:ok, gun_pid}
_ -> checkout_existing(workers)
end
end
defp provision_or_wait(key, uri, opts) do
result = with_route_lock(key, fn -> provision(key, uri, opts) end)
case result do
{:ok, gun_pid} ->
{:ok, gun_pid}
{:new, worker_pid} ->
get_gun_pid_from_worker(worker_pid)
{:wait, worker_pid} ->
case checkout_worker(worker_pid, :infinity) do
{:ok, gun_pid} -> {:ok, gun_pid}
:busy -> get_conn(uri, opts)
error -> error
end
error ->
error
end
end
defp provision(key, uri, opts) do
workers = Registry.lookup(@registry, key)
case checkout_existing(workers) do
{:ok, gun_pid} ->
{:ok, gun_pid}
:not_available ->
case Enum.find(workers, fn {_worker_pid, value} -> is_nil(value) end) do
{worker_pid, nil} -> {:wait, worker_pid}
nil -> start_worker(key, uri, opts)
end
end
end
defp get_gun_pid_from_worker(worker_pid, register) do
defp with_route_lock(key, function) do
case Registry.register(@locks, key, nil) do
{:ok, _} ->
try do
function.()
after
Registry.unregister(@locks, key)
end
{:error, {:already_registered, _owner}} ->
Process.sleep(1)
with_route_lock(key, function)
end
end
defp start_worker(key, uri, opts) do
# :gun.set_owner fails in :connected state, so the worker opens the connection itself.
case WorkerSupervisor.start_worker([key, uri, opts, self()]) do
{:ok, worker_pid} -> {:new, worker_pid}
error -> error
end
end
defp checkout_worker(worker_pid, timeout \\ 5_000) do
try do
GenServer.call(worker_pid, :checkout, timeout)
catch
:exit, reason -> {:error, {:connection_worker_exit, reason}}
end
end
defp get_gun_pid_from_worker(worker_pid) do
# GenServer.call will block the process for timeout length if
# the server crashes on startup (which will happen if gun fails to connect)
# so instead we use cast + monitor
ref = Process.monitor(worker_pid)
if register, do: GenServer.cast(worker_pid, {:add_client, self()})
receive do
{:conn_pid, pid} ->
@ -68,19 +147,115 @@ defmodule Pleroma.Gun.ConnectionPool do
@spec release_conn(pid()) :: :ok
def release_conn(conn_pid) do
# :ets.fun2ms(fn {_, {worker_pid, {gun_pid, _, _, _}}} when gun_pid == conn_pid ->
# :ets.fun2ms(fn {_, {worker_pid, {gun_pid, _tunnel}}} when gun_pid == conn_pid ->
# worker_pid end)
query_result =
Registry.select(@registry, [
{{:_, :"$1", {:"$2", :_, :_, :_}}, [{:==, :"$2", conn_pid}], [:"$1"]}
{{:_, :"$1", {:"$2", :_}}, [{:==, :"$2", conn_pid}], [:"$1"]}
])
case query_result do
[worker_pid] ->
GenServer.call(worker_pid, :remove_client)
try do
GenServer.call(worker_pid, :remove_client)
catch
:exit, _ -> :ok
end
[] ->
:ok
end
end
@spec discard_conn(pid()) :: :ok
def discard_conn(conn_pid) do
case worker_for_conn(conn_pid) do
nil ->
:ok
worker_pid ->
DynamicSupervisor.terminate_child(WorkerSupervisor, worker_pid)
end
end
@spec cancel_stream(pid(), reference() | [reference()]) :: :ok
def cancel_stream(conn_pid, stream) do
case worker_for_conn(conn_pid) do
nil ->
Pleroma.Gun.cancel(conn_pid, stream)
worker_pid ->
try do
GenServer.call(worker_pid, {:cancel_stream, stream})
catch
:exit, _ -> :ok
end
end
end
@spec register_stream(pid(), reference() | [reference()]) :: :ok
def register_stream(conn_pid, stream) do
call_worker(conn_pid, {:register_stream, stream})
end
@spec finish_stream(pid(), reference() | [reference()]) :: :ok
def finish_stream(conn_pid, stream) do
call_worker(conn_pid, {:finish_stream, stream})
end
@spec release_stream(pid(), reference() | [reference()]) :: :ok
def release_stream(conn_pid, stream) do
call_worker(conn_pid, {:release_stream, stream})
end
@spec tunnel_ref(pid()) :: reference() | :forward_proxy | nil
def tunnel_ref(conn_pid) do
case Registry.select(@registry, [
{{:_, :_, {:"$1", :"$2"}}, [{:==, :"$1", conn_pid}], [:"$2"]}
]) do
[tunnel] -> tunnel
[] -> nil
end
end
defp worker_for_conn(conn_pid) do
case Registry.select(@registry, [
{{:_, :"$1", {:"$2", :_}}, [{:==, :"$2", conn_pid}], [:"$1"]}
]) do
[worker_pid] -> worker_pid
[] -> nil
end
end
defp call_worker(conn_pid, message) do
case worker_for_conn(conn_pid) do
nil ->
:ok
worker_pid ->
try do
GenServer.call(worker_pid, message)
catch
:exit, _ -> :ok
end
end
end
defp connection_key(uri, opts) do
origin = {
String.downcase(uri.scheme),
String.downcase(uri.host),
uri.port
}
profile =
opts
|> Map.new()
|> Map.take(@connection_option_keys)
|> :erlang.term_to_binary()
|> then(&:crypto.hash(:sha256, &1))
fingerprint = Base.encode16(profile, case: :lower)
Enum.join(Tuple.to_list(origin) ++ [fingerprint], ":")
end
end

51
lib/pleroma/gun/connection_pool/reclaimer.ex
View file

@ -9,11 +9,11 @@ defmodule Pleroma.Gun.ConnectionPool.Reclaimer do
def start_monitor do
pid =
case GenServer.start_link(__MODULE__, [], name: {:via, Registry, {registry(), "reclaimer"}}) do
case GenServer.start(__MODULE__, [], name: __MODULE__) do
{:ok, pid} ->
pid
{:error, {:already_registered, pid}} ->
{:error, {:already_started, pid}} ->
pid
end
@ -41,17 +41,16 @@ defmodule Pleroma.Gun.ConnectionPool.Reclaimer do
reclaim_max: reclaim_max
})
# :ets.fun2ms(
# fn {_, {worker_pid, {_, used_by, crf, last_reference}}} when used_by == [] ->
# {worker_pid, crf, last_reference} end)
unused_conns =
workers =
Registry.select(
registry(),
[
{{:_, :"$1", {:_, :"$2", :"$3", :"$4"}}, [{:==, :"$2", []}], [{{:"$1", :"$3", :"$4"}}]}
{{:_, :"$1", {:_, :_}}, [], [:"$1"]}
]
)
unused_conns = Enum.flat_map(workers, &reclaim_info/1)
case unused_conns do
[] ->
:telemetry.execute(
@ -65,25 +64,43 @@ defmodule Pleroma.Gun.ConnectionPool.Reclaimer do
{:stop, :no_unused_conns, nil}
unused_conns ->
reclaimed =
reclaimed_count =
unused_conns
|> Enum.sort(fn {_pid1, crf1, last_reference1}, {_pid2, crf2, last_reference2} ->
crf1 <= crf2 and last_reference1 <= last_reference2
end)
|> Enum.take(reclaim_max)
|> Enum.sort_by(fn {_pid, crf, last_reference} -> {crf, last_reference} end)
|> Enum.reduce_while(0, fn
_worker, count when count == reclaim_max ->
{:halt, count}
reclaimed
|> Enum.each(fn {pid, _, _} ->
DynamicSupervisor.terminate_child(Pleroma.Gun.ConnectionPool.WorkerSupervisor, pid)
end)
{worker_pid, _crf, _last_reference}, count ->
if reclaim(worker_pid), do: {:cont, count + 1}, else: {:cont, count}
end)
:telemetry.execute(
[:pleroma, :connection_pool, :reclaim, :stop],
%{reclaimed_count: Enum.count(reclaimed)},
%{reclaimed_count: reclaimed_count},
%{max_connections: max_connections}
)
{:stop, :normal, nil}
end
end
defp reclaim_info(worker_pid) do
try do
case GenServer.call(worker_pid, :reclaim_info) do
{:idle, crf, last_reference} -> [{worker_pid, crf, last_reference}]
:in_use -> []
end
catch
:exit, _ -> []
end
end
defp reclaim(worker_pid) do
try do
GenServer.call(worker_pid, :reclaim) == :reclaimed
catch
:exit, _ -> false
end
end
end

268
lib/pleroma/gun/connection_pool/worker.ex
View file

@ -8,25 +8,22 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do
defp registry, do: Pleroma.Gun.ConnectionPool
def start_link([key | _] = opts) do
GenServer.start_link(__MODULE__, opts, name: {:via, Registry, {registry(), key}})
end
def start_link(opts), do: GenServer.start_link(__MODULE__, opts)
@impl true
def init([_key, _uri, _opts, _client_pid] = opts) do
def init([key, _uri, _opts, _client_pid] = opts) do
{:ok, _} = Registry.register(registry(), key, nil)
{:ok, nil, {:continue, {:connect, opts}}}
end
@impl true
def handle_continue({:connect, [key, uri, opts, client_pid]}, _) do
with {:ok, conn_pid, protocol} <- Gun.Conn.open(uri, opts),
with {:ok, conn_pid, protocol, tunnel} <- Gun.Conn.open(uri, opts),
Process.link(conn_pid) do
time = :erlang.monotonic_time(:millisecond)
{_, _} =
Registry.update_value(registry(), key, fn _ ->
{conn_pid, [client_pid], 1, time}
end)
Registry.unregister(registry(), key)
{:ok, _} = Registry.register(registry(), key, {conn_pid, tunnel})
send(client_pid, {:conn_pid, conn_pid})
@ -34,8 +31,14 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do
%{
key: key,
timer: nil,
client_monitors: %{client_pid => Process.monitor(client_pid)},
protocol: protocol
clients: %{
client_pid => %{count: 1, monitor: Process.monitor(client_pid), streams: MapSet.new()}
},
conn_pid: conn_pid,
protocol: protocol,
proxied: Keyword.has_key?(opts, :proxy),
crf: 1,
last_reference: time
}, :hibernate}
else
err ->
@ -44,106 +47,225 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do
end
@impl true
def handle_cast({:add_client, client_pid}, state) do
case handle_call(:add_client, {client_pid, nil}, state) do
{:reply, conn_pid, state, :hibernate} ->
send(client_pid, {:conn_pid, conn_pid})
{:noreply, state, :hibernate}
end
def handle_call(:checkout, _from, %{protocol: protocol, clients: clients} = state)
when protocol != :http2 and map_size(clients) > 0 do
{:reply, :busy, state, :hibernate}
end
@impl true
def handle_cast({:remove_client, client_pid}, state) do
case handle_call(:remove_client, {client_pid, nil}, state) do
{:reply, _, state, :hibernate} ->
{:noreply, state, :hibernate}
end
end
@impl true
def handle_call(:add_client, {client_pid, _}, %{key: key, protocol: protocol} = state) do
def handle_call(:checkout, {client_pid, _}, %{key: key, protocol: protocol} = state) do
time = :erlang.monotonic_time(:millisecond)
{{conn_pid, used_by, _, _}, _} =
Registry.update_value(registry(), key, fn {conn_pid, used_by, crf, last_reference} ->
{conn_pid, [client_pid | used_by], crf(time - last_reference, crf), time}
end)
[{_, {conn_pid, _tunnel}}] =
Registry.lookup(registry(), key) |> Enum.filter(&(elem(&1, 0) == self()))
state =
state
|> cancel_idle_timer()
|> add_client(client_pid)
|> Map.put(:crf, crf(time - state.last_reference, state.crf))
|> Map.put(:last_reference, time)
:telemetry.execute(
[:pleroma, :connection_pool, :client, :add],
%{client_pid: client_pid, clients: used_by},
%{client_pid: client_pid, clients: Map.keys(state.clients)},
%{key: state.key, protocol: protocol}
)
state =
if state.timer != nil do
Process.cancel_timer(state[:timer])
%{state | timer: nil}
else
state
end
ref = Process.monitor(client_pid)
state = put_in(state.client_monitors[client_pid], ref)
{:reply, conn_pid, state, :hibernate}
{:reply, {:ok, conn_pid}, state, :hibernate}
end
@impl true
def handle_call(:remove_client, {client_pid, _}, %{key: key} = state) do
{{_conn_pid, used_by, _crf, _last_reference}, _} =
Registry.update_value(registry(), key, fn {conn_pid, used_by, crf, last_reference} ->
{conn_pid, List.delete(used_by, client_pid), crf, last_reference}
end)
def handle_call(:remove_client, {client_pid, _}, state) do
case active_streams(state, client_pid) do
[] ->
{:reply, :ok, remove_client(state, client_pid, true), :hibernate}
{ref, state} = pop_in(state.client_monitors[client_pid])
streams when state.protocol == :http2 ->
Enum.each(streams, &(:ok = Gun.cancel(state.conn_pid, &1)))
{:reply, :ok, remove_client(state, client_pid, true), :hibernate}
Process.demonitor(ref, [:flush])
_streams ->
{:stop, :normal, :ok, state}
end
end
timer =
if used_by == [] do
max_idle = Pleroma.Config.get([:connections_pool, :max_idle_time], 30_000)
Process.send_after(self(), :idle_close, max_idle)
else
nil
end
def handle_call({:register_stream, stream}, {client_pid, _}, state) do
{:reply, :ok, update_stream(state, client_pid, &MapSet.put(&1, stream)), :hibernate}
end
{:reply, :ok, %{state | timer: timer}, :hibernate}
def handle_call({:finish_stream, stream}, {client_pid, _}, state) do
{:reply, :ok, update_stream(state, client_pid, &MapSet.delete(&1, stream)), :hibernate}
end
def handle_call({:release_stream, stream}, {client_pid, _}, state) do
state = update_stream(state, client_pid, &MapSet.delete(&1, stream))
{:reply, :ok, remove_client(state, client_pid, true), :hibernate}
end
def handle_call({:cancel_stream, stream}, {client_pid, _}, %{protocol: :http2} = state) do
:ok = Gun.cancel(state.conn_pid, stream)
state = update_stream(state, client_pid, &MapSet.delete(&1, stream))
{:reply, :ok, remove_client(state, client_pid, true), :hibernate}
end
def handle_call({:cancel_stream, _stream}, _from, state) do
{:stop, :normal, :ok, state}
end
def handle_call(:reclaim_info, _from, %{clients: clients} = state)
when map_size(clients) == 0 do
{:reply, {:idle, state.crf, state.last_reference}, state, :hibernate}
end
def handle_call(:reclaim_info, _from, state) do
{:reply, :in_use, state, :hibernate}
end
def handle_call(:reclaim, _from, %{clients: clients} = state) when map_size(clients) == 0 do
{:stop, :normal, :reclaimed, state}
end
def handle_call(:reclaim, _from, state) do
{:reply, :in_use, state, :hibernate}
end
@impl true
def handle_info(:idle_close, state) do
# Gun monitors the owner process, and will close the connection automatically
# when it's terminated
def handle_info({:idle_close, token}, %{timer: {_timer_ref, token}, clients: clients} = state)
when map_size(clients) == 0 do
# Gun monitors the owner process, and will close the connection automatically.
{:stop, :normal, state}
end
def handle_info({:idle_close, _token}, state), do: {:noreply, state, :hibernate}
@impl true
def handle_info({:gun_up, _pid, _protocol}, state) do
{:noreply, state, :hibernate}
def handle_info({:gun_up, _pid, protocol}, state) do
{:noreply, %{state | protocol: protocol}, :hibernate}
end
# Gracefully shutdown if the connection got closed without any streams left
@impl true
def handle_info({:gun_error, conn_pid, stream, _reason}, %{conn_pid: conn_pid} = state) do
{:noreply, delete_stream(state, stream), :hibernate}
end
@impl true
def handle_info({:gun_down, _pid, _protocol, _reason, _streams}, %{proxied: true} = state) do
{:stop, :normal, state}
end
# Gracefully shutdown if the connection got closed without any streams left.
@impl true
def handle_info({:gun_down, _pid, _protocol, _reason, []}, state) do
{:stop, :normal, state}
end
# Otherwise, wait for retry
# Otherwise, wait for retry.
@impl true
def handle_info({:gun_down, _pid, _protocol, _reason, _killed_streams}, state) do
{:noreply, state, :hibernate}
end
@impl true
def handle_info({:DOWN, _ref, :process, pid, reason}, state) do
:telemetry.execute(
[:pleroma, :connection_pool, :client, :dead],
%{client_pid: pid, reason: reason},
%{key: state.key}
)
def handle_info({:DOWN, ref, :process, pid, reason}, state) do
case state.clients[pid] do
%{monitor: ^ref} ->
:telemetry.execute(
[:pleroma, :connection_pool, :client, :dead],
%{client_pid: pid, reason: reason},
%{key: state.key}
)
handle_cast({:remove_client, pid}, state)
handle_dead_client(state, pid)
_ ->
{:noreply, state, :hibernate}
end
end
defp add_client(state, client_pid) do
update_in(state.clients, fn clients ->
case clients[client_pid] do
nil ->
Map.put(clients, client_pid, %{
count: 1,
monitor: Process.monitor(client_pid),
streams: MapSet.new()
})
client ->
Map.put(clients, client_pid, %{client | count: client.count + 1})
end
end)
end
defp handle_dead_client(state, client_pid) do
streams = active_streams(state, client_pid)
cond do
streams == [] ->
{:noreply, remove_client(state, client_pid, false, true), :hibernate}
state.protocol == :http2 ->
Enum.each(streams, &(:ok = Gun.cancel(state.conn_pid, &1)))
{:noreply, remove_client(state, client_pid, false, true), :hibernate}
true ->
{:stop, :normal, state}
end
end
defp active_streams(state, client_pid) do
case state.clients[client_pid] do
%{streams: streams} -> MapSet.to_list(streams)
nil -> []
end
end
defp update_stream(state, client_pid, update) do
update_in(state.clients, fn clients ->
case clients[client_pid] do
nil -> clients
client -> Map.put(clients, client_pid, %{client | streams: update.(client.streams)})
end
end)
end
defp delete_stream(state, stream) do
update_in(state.clients, fn clients ->
Map.new(clients, fn {pid, client} ->
{pid, %{client | streams: MapSet.delete(client.streams, stream)}}
end)
end)
end
defp remove_client(state, client_pid, demonitor, all \\ false) do
case state.clients[client_pid] do
nil ->
state
%{count: count} = client when count > 1 and not all ->
put_in(state.clients[client_pid], %{client | count: count - 1})
client ->
if demonitor, do: Process.demonitor(client.monitor, [:flush])
state = update_in(state.clients, &Map.delete(&1, client_pid))
if map_size(state.clients) == 0, do: schedule_idle_close(state), else: state
end
end
defp cancel_idle_timer(%{timer: nil} = state), do: state
defp cancel_idle_timer(%{timer: {timer_ref, _token}} = state) do
Process.cancel_timer(timer_ref)
%{state | timer: nil}
end
defp schedule_idle_close(state) do
max_idle = Pleroma.Config.get([:connections_pool, :max_idle_time], 30_000)
token = make_ref()
timer_ref = Process.send_after(self(), {:idle_close, token}, max_idle)
%{state | timer: {timer_ref, token}}
end
# LRFU policy: https://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.55.1478

39
lib/pleroma/gun/connection_pool/worker_supervisor.ex
View file

@ -5,6 +5,9 @@
defmodule Pleroma.Gun.ConnectionPool.WorkerSupervisor do
@moduledoc "Supervisor for pool workers. Does not do anything except enforce max connection limit"
alias Pleroma.Config
alias Pleroma.Gun.ConnectionPool.Worker
use DynamicSupervisor
def start_link(opts) do
@ -14,21 +17,35 @@ defmodule Pleroma.Gun.ConnectionPool.WorkerSupervisor do
def init(_opts) do
DynamicSupervisor.init(
strategy: :one_for_one,
max_children: Pleroma.Config.get([:connections_pool, :max_connections])
max_children: Config.get([:connections_pool, :max_connections])
)
end
def start_worker(opts, last_attempt \\ false) do
case DynamicSupervisor.start_child(__MODULE__, {Pleroma.Gun.ConnectionPool.Worker, opts}) do
{:error, :max_children} ->
funs = [fn -> last_attempt end, fn -> match?(:error, free_pool()) end]
def start_worker(opts, last_attempt \\ false)
if Enum.any?(funs, fn fun -> fun.() end) do
:telemetry.execute([:pleroma, :connection_pool, :provision_failure], %{opts: opts})
{:error, :pool_full}
else
start_worker(opts, true)
end
def start_worker(opts, true) do
case DynamicSupervisor.start_child(__MODULE__, {Worker, opts}) do
{:error, :max_children} ->
[key | _] = opts
:telemetry.execute(
[:pleroma, :connection_pool, :provision_failure],
%{},
%{key: key}
)
{:error, :pool_full}
res ->
res
end
end
def start_worker(opts, false) do
case DynamicSupervisor.start_child(__MODULE__, {Worker, opts}) do
{:error, :max_children} ->
free_pool()
start_worker(opts, true)
res ->
res

89
lib/pleroma/hashtag.ex
View file

@ -12,6 +12,7 @@ defmodule Pleroma.Hashtag do
alias Pleroma.Hashtag
alias Pleroma.Object
alias Pleroma.Repo
alias Pleroma.User.HashtagFollow
schema "hashtags" do
field(:name, :string)
@ -27,6 +28,14 @@ defmodule Pleroma.Hashtag do
|> String.trim()
end
def get_by_id(id) do
Repo.get(Hashtag, id)
end
def get_by_name(name) do
Repo.get_by(Hashtag, name: normalize_name(name))
end
def get_or_create_by_name(name) do
changeset = changeset(%Hashtag{}, %{name: name})
@ -103,4 +112,84 @@ defmodule Pleroma.Hashtag do
{:ok, deleted_count}
end
end
def get_followers(%Hashtag{id: hashtag_id}) do
from(hf in HashtagFollow)
|> where([hf], hf.hashtag_id == ^hashtag_id)
|> join(:inner, [hf], u in assoc(hf, :user))
|> select([hf, u], u.id)
|> Repo.all()
end
def get_recipients_for_activity(%Pleroma.Activity{object: %{hashtags: tags}})
when is_list(tags) do
tags
|> Enum.map(&get_followers/1)
|> List.flatten()
|> Enum.uniq()
end
def get_recipients_for_activity(_activity), do: []
def search(query, options \\ []) do
limit = Keyword.get(options, :limit, 20)
offset = Keyword.get(options, :offset, 0)
search_terms =
query
|> String.downcase()
|> String.trim()
|> String.split(~r/\s+/)
|> Enum.filter(&(&1 != ""))
|> Enum.map(&String.trim_leading(&1, "#"))
|> Enum.filter(&(&1 != ""))
if Enum.empty?(search_terms) do
[]
else
# Use PostgreSQL's ANY operator with array for efficient multi-term search
# This is much more efficient than multiple OR clauses
search_patterns = Enum.map(search_terms, &"%#{&1}%")
# Create ranking query that prioritizes exact matches and closer matches
# Use a subquery to properly handle computed columns in ORDER BY
base_query =
from(ht in Hashtag,
where: fragment("LOWER(?) LIKE ANY(?)", ht.name, ^search_patterns),
select: %{
name: ht.name,
# Ranking: exact matches get highest priority (0)
# then prefix matches (1), then contains (2)
match_rank:
fragment(
"""
CASE
WHEN LOWER(?) = ANY(?) THEN 0
WHEN LOWER(?) LIKE ANY(?) THEN 1
ELSE 2
END
""",
ht.name,
^search_terms,
ht.name,
^Enum.map(search_terms, &"#{&1}%")
),
# Secondary sort by name length (shorter names first)
name_length: fragment("LENGTH(?)", ht.name)
}
)
from(result in subquery(base_query),
order_by: [
asc: result.match_rank,
asc: result.name_length,
asc: result.name
],
limit: ^limit,
offset: ^offset
)
|> Repo.all()
|> Enum.map(& &1.name)
end
end
end

41
lib/pleroma/http.ex
View file

@ -37,7 +37,7 @@ defmodule Pleroma.HTTP do
See `Pleroma.HTTP.request/5`
"""
@spec post(Request.url(), String.t(), Request.headers(), keyword()) ::
@spec post(Request.url(), Tesla.Env.body(), Request.headers(), keyword()) ::
{:ok, Env.t()} | {:error, any()}
def post(url, body, headers \\ [], options \\ []),
do: request(:post, url, body, headers, options)
@ -56,7 +56,7 @@ defmodule Pleroma.HTTP do
`{:ok, %Tesla.Env{}}` or `{:error, error}`
"""
@spec request(method(), Request.url(), String.t(), Request.headers(), keyword()) ::
@spec request(method(), Request.url(), Tesla.Env.body(), Request.headers(), keyword()) ::
{:ok, Env.t()} | {:error, any()}
def request(method, url, body, headers, options) when is_binary(url) do
uri = URI.parse(url)
@ -68,7 +68,9 @@ defmodule Pleroma.HTTP do
adapter = Application.get_env(:tesla, :adapter)
client = Tesla.client(adapter_middlewares(adapter), adapter)
extra_middleware = options[:tesla_middleware] || []
client = Tesla.client(adapter_middlewares(adapter, extra_middleware), adapter)
maybe_limit(
fn ->
@ -102,20 +104,31 @@ defmodule Pleroma.HTTP do
fun.()
end
defp adapter_middlewares(Tesla.Adapter.Gun) do
[Tesla.Middleware.FollowRedirects, Pleroma.Tesla.Middleware.ConnectionPool]
defp adapter_middlewares(Tesla.Adapter.Gun, extra_middleware) do
default_middleware() ++
[Pleroma.Tesla.Middleware.ConnectionPool] ++
extra_middleware
end
defp adapter_middlewares({Tesla.Adapter.Finch, _}) do
[Tesla.Middleware.FollowRedirects]
end
defp adapter_middlewares(_, extra_middleware) do
# A lot of tests are written expecting unencoded URLs
# and the burden of fixing that is high. Also it makes
# them hard to read. Tests will opt-in when we want to validate
# the encoding is being done correctly.
cond do
Pleroma.Config.get(:env) == :test and Pleroma.Config.get(:test_url_encoding) ->
default_middleware()
defp adapter_middlewares(_) do
if Pleroma.Config.get(:env) == :test do
# Emulate redirects in test env, which are handled by adapters in other environments
[Tesla.Middleware.FollowRedirects]
else
[]
Pleroma.Config.get(:env) == :test ->
# Emulate redirects in test env, which are handled by adapters in other environments
[Tesla.Middleware.FollowRedirects]
# Hackney and Finch
true ->
default_middleware() ++ extra_middleware
end
end
defp default_middleware,
do: [Tesla.Middleware.FollowRedirects, Pleroma.Tesla.Middleware.EncodeUrl]
end

253
lib/pleroma/http/adapter/gun.ex Normal file
View file

@ -0,0 +1,253 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.HTTP.Adapter.Gun do
@behaviour Tesla.Adapter
alias Pleroma.Gun.ConnectionPool
alias Tesla.Multipart
@default_timeout 1_000
@impl Tesla.Adapter
def call(env, opts) do
adapter_opts =
Tesla.Adapter.opts(
[close_conn: true, body_as: :plain, send_body: :at_once, receive: true],
env,
opts
)
|> Map.new()
with {:ok, status, headers, body} <- request(env, adapter_opts) do
env = %{env | status: status, headers: format_headers(headers), body: body}
{:ok, put_stream_ref(env, body)}
end
end
defp put_stream_ref(env, %{stream: stream}) do
put_in(env.opts[:adapter][:stream], stream)
end
defp put_stream_ref(env, _body), do: env
defp request(env, %{conn: conn, tunnel: tunnel} = opts) do
uri = URI.parse(Tesla.build_url(env))
path = request_path(uri, tunnel)
method = Tesla.Adapter.Shared.format_method(env.method)
{headers, body, send_body} = prepare_body(env.headers, env.body, opts[:send_body])
headers = maybe_add_forward_proxy_headers(headers, uri, opts, tunnel)
request_opts = request_opts(opts[:reply_to], tunnel)
stream =
try do
open_stream(conn, method, path, headers, body, request_opts, send_body)
catch
kind, reason ->
ConnectionPool.release_conn(conn)
:erlang.raise(kind, reason, __STACKTRACE__)
end
:ok = ConnectionPool.register_stream(conn, stream)
:ok = send_stream_body(conn, stream, body, send_body)
response = read_response(conn, stream, opts)
case response do
{:error, reason} ->
ConnectionPool.cancel_stream(conn, stream)
{:error, {:gun_stream_error, reason}}
response ->
unless opts[:body_as] in [:chunks, :stream] and stream_body?(response) do
:ok = ConnectionPool.finish_stream(conn, stream)
end
if opts[:close_conn] and opts[:body_as] not in [:stream, :chunks] do
:gun.close(conn)
end
response
end
end
defp stream_body?({:ok, _status, _headers, %{stream: _stream}}), do: true
defp stream_body?({:ok, _status, _headers, %Stream{}}), do: true
defp stream_body?(_response), do: false
defp request_opts(reply_to, nil), do: %{reply_to: reply_to || self()}
defp request_opts(reply_to, :forward_proxy), do: %{reply_to: reply_to || self()}
defp request_opts(reply_to, tunnel) do
%{reply_to: reply_to || self(), tunnel: tunnel}
end
defp request_path(uri, :forward_proxy) do
path = Tesla.Adapter.Shared.prepare_path(uri.path, uri.query)
"#{uri.scheme}://#{authority(uri)}#{path}"
end
defp request_path(uri, _tunnel) do
Tesla.Adapter.Shared.prepare_path(uri.path, uri.query)
end
defp maybe_add_forward_proxy_headers(headers, uri, opts, :forward_proxy) do
headers
|> put_header_new("host", authority(uri))
|> maybe_put_proxy_authorization(opts)
end
defp maybe_add_forward_proxy_headers(headers, _uri, _opts, _tunnel), do: headers
defp maybe_put_proxy_authorization(headers, %{proxy_auth: {username, password}})
when is_binary(username) and is_binary(password) do
value = "Basic " <> Base.encode64(username <> ":" <> password)
List.keystore(headers, "proxy-authorization", 0, {"proxy-authorization", value})
end
defp maybe_put_proxy_authorization(headers, _opts), do: headers
defp put_header_new(headers, name, value) do
if List.keymember?(headers, name, 0), do: headers, else: [{name, value} | headers]
end
defp authority(%URI{scheme: scheme, host: host, port: port}) do
host = if String.contains?(host, ":"), do: "[#{host}]", else: host
if port == URI.default_port(scheme), do: host, else: "#{host}:#{port}"
end
defp prepare_body(headers, %Multipart{} = multipart, _send_body) do
{format_headers(headers ++ Multipart.headers(multipart)), Multipart.body(multipart), :stream}
end
defp prepare_body(headers, body, _send_body)
when is_struct(body, Stream) or is_function(body) do
{format_headers(headers), body, :stream}
end
defp prepare_body(headers, body, send_body) do
{format_headers(headers), body || "", send_body}
end
defp open_stream(conn, method, path, headers, _body, request_opts, :stream) do
:gun.headers(conn, method, path, headers, request_opts)
end
defp open_stream(conn, method, path, headers, body, request_opts, :at_once) do
:gun.request(conn, method, path, headers, body, request_opts)
end
defp send_stream_body(conn, stream, body, :stream) do
try do
for data <- body, do: :ok = :gun.data(conn, stream, :nofin, data)
:gun.data(conn, stream, :fin, "")
catch
kind, reason ->
ConnectionPool.cancel_stream(conn, stream)
:erlang.raise(kind, reason, __STACKTRACE__)
end
end
defp send_stream_body(_conn, _stream, _body, :at_once), do: :ok
defp read_response(conn, stream, opts) do
receive? = opts[:receive]
receive do
{:gun_response, ^conn, ^stream, :fin, status, headers} ->
{:ok, status, headers, ""}
{:gun_response, ^conn, ^stream, :nofin, status, headers} ->
format_response(conn, stream, opts, status, headers, opts[:body_as])
{:gun_up, ^conn, _protocol} when receive? ->
read_response(conn, stream, opts)
{:gun_error, ^conn, reason} ->
{:error, reason}
{:gun_error, ^conn, ^stream, reason} ->
{:error, reason}
{:gun_down, ^conn, _protocol, _reason, _killed_streams} when receive? ->
read_response(conn, stream, opts)
{:DOWN, _ref, :process, ^conn, reason} ->
{:error, reason}
after
opts[:timeout] || @default_timeout -> {:error, :recv_response_timeout}
end
end
defp format_response(conn, stream, opts, status, headers, :plain) do
case read_body(conn, stream, opts) do
{:ok, body} ->
{:ok, status, headers, body}
{:error, error} ->
:ok = :gun.flush(stream)
{:error, error}
end
end
defp format_response(conn, stream, opts, status, headers, :stream) do
body =
Stream.resource(
fn -> :reading end,
fn
:reading ->
case Tesla.Adapter.Gun.read_chunk(conn, stream, opts) do
{:nofin, part} -> {[part], :reading}
{:fin, part} -> {[part], :done}
{:error, reason} -> raise "Gun stream failed: #{inspect(reason)}"
end
:done ->
{:halt, :done}
end,
fn _state ->
if opts[:close_conn], do: :gun.close(conn)
end
)
{:ok, status, headers, body}
end
defp format_response(conn, stream, opts, status, headers, :chunks) do
{:ok, status, headers, %{pid: conn, stream: stream, opts: Map.to_list(opts)}}
end
defp read_body(conn, stream, opts, acc \\ "") do
receive do
{:gun_data, ^conn, ^stream, :fin, body} ->
append_body(acc, body, opts[:max_body])
{:gun_data, ^conn, ^stream, :nofin, body} ->
with {:ok, acc} <- append_body(acc, body, opts[:max_body]) do
read_body(conn, stream, opts, acc)
end
{:gun_error, ^conn, ^stream, reason} ->
{:error, reason}
{:DOWN, _ref, :process, ^conn, reason} ->
{:error, reason}
after
opts[:timeout] || @default_timeout -> {:error, :recv_body_timeout}
end
end
defp append_body(acc, part, nil), do: {:ok, acc <> part}
defp append_body(acc, part, limit) do
body = acc <> part
if byte_size(body) <= limit, do: {:ok, body}, else: {:error, :body_too_large}
end
defp format_headers(headers) do
Enum.map(headers, fn {key, value} ->
{String.downcase(to_string(key)), to_string(value)}
end)
end
end

10
lib/pleroma/http/adapter_helper.ex
View file

@ -52,6 +52,7 @@ defmodule Pleroma.HTTP.AdapterHelper do
case adapter() do
Tesla.Adapter.Gun -> AdapterHelper.Gun
Tesla.Adapter.Hackney -> AdapterHelper.Hackney
{Tesla.Adapter.Finch, _} -> AdapterHelper.Finch
_ -> AdapterHelper.Default
end
end
@ -118,4 +119,13 @@ defmodule Pleroma.HTTP.AdapterHelper do
host_charlist
end
end
@spec can_stream? :: bool()
def can_stream? do
case Application.get_env(:tesla, :adapter) do
Tesla.Adapter.Gun -> true
{Tesla.Adapter.Finch, _} -> true
_ -> false
end
end
end

33
lib/pleroma/http/adapter_helper/finch.ex Normal file
View file

@ -0,0 +1,33 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.HTTP.AdapterHelper.Finch do
@behaviour Pleroma.HTTP.AdapterHelper
alias Pleroma.Config
alias Pleroma.HTTP.AdapterHelper
@spec options(keyword(), URI.t()) :: keyword()
def options(incoming_opts \\ [], %URI{} = _uri) do
proxy =
[:http, :proxy_url]
|> Config.get()
|> AdapterHelper.format_proxy()
config_opts = Config.get([:http, :adapter], [])
config_opts
|> Keyword.merge(incoming_opts)
|> AdapterHelper.maybe_add_proxy(proxy)
|> maybe_stream()
end
# Finch uses [response: :stream]
defp maybe_stream(opts) do
case Keyword.pop(opts, :stream, nil) do
{true, opts} -> Keyword.put(opts, :response, :stream)
{_, opts} -> opts
end
end
end

11
lib/pleroma/http/adapter_helper/gun.ex
View file

@ -15,7 +15,7 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do
retry_timeout: 1_000
]
@type pool() :: :federation | :upload | :media | :default
@type pool() :: :federation | :upload | :media | :rich_media | :default
@spec options(keyword(), URI.t()) :: keyword()
def options(incoming_opts \\ [], %URI{} = uri) do
@ -32,6 +32,7 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do
|> AdapterHelper.maybe_add_proxy(proxy)
|> Keyword.merge(incoming_opts)
|> put_timeout()
|> maybe_stream()
end
defp add_scheme_opts(opts, %{scheme: "http"}), do: opts
@ -47,6 +48,14 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do
Keyword.put(opts, :timeout, recv_timeout)
end
# Keep the pool lease until the caller consumes or cancels the body.
defp maybe_stream(opts) do
case Keyword.pop(opts, :stream, nil) do
{true, opts} -> Keyword.put(opts, :body_as, :chunks)
{_, opts} -> opts
end
end
@spec pool_timeout(pool()) :: non_neg_integer()
def pool_timeout(pool) do
default = Config.get([:pools, :default, :recv_timeout], 5_000)

10
lib/pleroma/http/adapter_helper/hackney.ex
View file

@ -6,8 +6,9 @@ defmodule Pleroma.HTTP.AdapterHelper.Hackney do
@behaviour Pleroma.HTTP.AdapterHelper
@defaults [
follow_redirect: true,
force_redirect: true
follow_redirect: false,
force_redirect: false,
with_body: true
]
@spec options(keyword(), URI.t()) :: keyword()
@ -16,7 +17,12 @@ defmodule Pleroma.HTTP.AdapterHelper.Hackney do
config_opts = Pleroma.Config.get([:http, :adapter], [])
url_encoding =
Keyword.new()
|> Keyword.put(:path_encode_fun, fn path -> path end)
@defaults
|> Keyword.merge(url_encoding)
|> Keyword.merge(config_opts)
|> Keyword.merge(connection_opts)
|> add_scheme_opts(uri)

37
lib/pleroma/instances.ex
View file

@ -15,25 +15,7 @@ defmodule Pleroma.Instances do
defdelegate set_unreachable(url_or_host, unreachable_since \\ nil), to: Instance
defdelegate get_consistently_unreachable, to: Instance
def set_consistently_unreachable(url_or_host),
do: set_unreachable(url_or_host, reachability_datetime_threshold())
def reachability_datetime_threshold do
federation_reachability_timeout_days =
Pleroma.Config.get([:instance, :federation_reachability_timeout_days], 0)
if federation_reachability_timeout_days > 0 do
NaiveDateTime.add(
NaiveDateTime.utc_now(),
-federation_reachability_timeout_days * 24 * 3600,
:second
)
else
~N[0000-01-01 00:00:00]
end
end
defdelegate get_unreachable, to: Instance
def host(url_or_host) when is_binary(url_or_host) do
if url_or_host =~ ~r/^http/i do
@ -42,4 +24,21 @@ defmodule Pleroma.Instances do
url_or_host
end
end
@doc "Schedules reachability checks for all unreachable instances"
def check_all_unreachable do
get_unreachable()
|> Enum.each(fn {domain, _} ->
Pleroma.Workers.ReachabilityWorker.new(%{"domain" => domain})
|> Oban.insert()
end)
end
@doc "Deletes all users and activities for unreachable instances"
def delete_all_unreachable do
get_unreachable()
|> Enum.each(fn {domain, _} ->
Instance.delete(domain)
end)
end
end

53
lib/pleroma/instances/instance.ex
View file

@ -9,8 +9,7 @@ defmodule Pleroma.Instances.Instance do
alias Pleroma.Instances.Instance
alias Pleroma.Maps
alias Pleroma.Repo
alias Pleroma.User
alias Pleroma.Workers.BackgroundWorker
alias Pleroma.Workers.DeleteWorker
use Ecto.Schema
@ -51,7 +50,7 @@ defmodule Pleroma.Instances.Instance do
|> cast(params, [:software_name, :software_version, :software_repository])
end
def filter_reachable([]), do: %{}
def filter_reachable([]), do: []
def filter_reachable(urls_or_hosts) when is_list(urls_or_hosts) do
hosts =
@ -68,19 +67,15 @@ defmodule Pleroma.Instances.Instance do
)
|> Map.new(& &1)
reachability_datetime_threshold = Instances.reachability_datetime_threshold()
for entry <- Enum.filter(urls_or_hosts, &is_binary/1) do
host = host(entry)
unreachable_since = unreachable_since_by_host[host]
if !unreachable_since ||
NaiveDateTime.compare(unreachable_since, reachability_datetime_threshold) == :gt do
{entry, unreachable_since}
if is_nil(unreachable_since) do
entry
end
end
|> Enum.filter(& &1)
|> Map.new(& &1)
end
def reachable?(url_or_host) when is_binary(url_or_host) do
@ -88,7 +83,7 @@ defmodule Pleroma.Instances.Instance do
from(i in Instance,
where:
i.host == ^host(url_or_host) and
i.unreachable_since <= ^Instances.reachability_datetime_threshold(),
not is_nil(i.unreachable_since),
select: true
)
)
@ -97,9 +92,16 @@ defmodule Pleroma.Instances.Instance do
def reachable?(url_or_host) when is_binary(url_or_host), do: true
def set_reachable(url_or_host) when is_binary(url_or_host) do
%Instance{host: host(url_or_host)}
|> changeset(%{unreachable_since: nil})
|> Repo.insert(on_conflict: {:replace, [:unreachable_since]}, conflict_target: :host)
host = host(url_or_host)
result =
%Instance{host: host}
|> changeset(%{unreachable_since: nil})
|> Repo.insert(on_conflict: {:replace, [:unreachable_since]}, conflict_target: :host)
Pleroma.Workers.ReachabilityWorker.delete_jobs_for_host(host)
result
end
def set_reachable(_), do: {:error, nil}
@ -132,11 +134,9 @@ defmodule Pleroma.Instances.Instance do
def set_unreachable(_, _), do: {:error, nil}
def get_consistently_unreachable do
reachability_datetime_threshold = Instances.reachability_datetime_threshold()
def get_unreachable do
from(i in Instance,
where: ^reachability_datetime_threshold > i.unreachable_since,
where: not is_nil(i.unreachable_since),
order_by: i.unreachable_since,
select: {i.host, i.unreachable_since}
)
@ -296,19 +296,14 @@ defmodule Pleroma.Instances.Instance do
Deletes all users from an instance in a background task, thus also deleting
all of those users' activities and notifications.
"""
def delete_users_and_activities(host) when is_binary(host) do
BackgroundWorker.enqueue("delete_instance", %{"host" => host})
def delete(host) when is_binary(host) do
DeleteWorker.new(%{"op" => "delete_instance", "host" => host})
|> Oban.insert()
end
def perform(:delete_instance, host) when is_binary(host) do
User.Query.build(%{nickname: "@#{host}"})
|> Repo.chunk_stream(100, :batches)
|> Stream.each(fn users ->
users
|> Enum.each(fn user ->
User.perform(:delete, user)
end)
end)
|> Stream.run()
@doc "Schedules reachability check for instance"
def check_unreachable(domain) when is_binary(domain) do
Pleroma.Workers.ReachabilityWorker.new(%{"domain" => domain})
|> Oban.insert()
end
end

59
lib/pleroma/language/language_detector.ex Normal file
View file

@ -0,0 +1,59 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Language.LanguageDetector do
import Pleroma.EctoType.ActivityPub.ObjectValidators.LanguageCode,
only: [good_locale_code?: 1]
@words_threshold 4
@config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config)
def configured? do
provider = get_provider()
!!provider and provider.configured?()
end
def missing_dependencies do
provider = get_provider()
if provider do
provider.missing_dependencies()
else
[]
end
end
# Strip tags from text, etc.
defp prepare_text(text) do
text
|> Floki.parse_fragment!()
|> Floki.filter_out(
".h-card, .mention, .hashtag, .u-url, .quote-inline, .recipients-inline, code, pre"
)
|> Floki.text()
end
def detect(text) do
provider = get_provider()
text = prepare_text(text)
word_count = text |> String.split(~r/\s+/) |> Enum.count()
if word_count < @words_threshold or !provider or !provider.configured?() do
nil
else
with language <- provider.detect(text),
true <- good_locale_code?(language) do
language
else
_ -> nil
end
end
end
defp get_provider do
@config_impl.get([__MODULE__, :provider])
end
end

47
lib/pleroma/language/language_detector/fasttext.ex Normal file
View file

@ -0,0 +1,47 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Language.LanguageDetector.Fasttext do
import Pleroma.Web.Utils.Guards, only: [not_empty_string: 1]
alias Pleroma.Language.LanguageDetector.Provider
@behaviour Provider
@impl Provider
def missing_dependencies do
if Pleroma.Utils.command_available?("fasttext") do
[]
else
["fasttext"]
end
end
@impl Provider
def configured?, do: not_empty_string(get_model())
@impl Provider
def detect(text) do
text_path = Path.join(System.tmp_dir!(), "fasttext-#{Ecto.UUID.generate()}")
File.write(text_path, text |> String.replace(~r/\s+/, " "))
detected_language =
case System.cmd("fasttext", ["predict", get_model(), text_path]) do
{"__label__" <> language, _} ->
language |> String.trim()
_ ->
nil
end
File.rm(text_path)
detected_language
end
defp get_model do
Pleroma.Config.get([__MODULE__, :model])
end
end

11
lib/pleroma/language/language_detector/provider.ex Normal file
View file

@ -0,0 +1,11 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Language.LanguageDetector.Provider do
@callback missing_dependencies() :: [String.t()]
@callback configured?() :: boolean()
@callback detect(text :: String.t()) :: String.t() | nil
end

127
lib/pleroma/language/translation.ex Normal file
View file

@ -0,0 +1,127 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Language.Translation do
@cachex Pleroma.Config.get([:cachex, :provider], Cachex)
def configured? do
provider = get_provider()
!!provider and provider.configured?()
end
def missing_dependencies do
provider = get_provider()
if provider do
provider.missing_dependencies()
else
[]
end
end
def translate(text, source_language, target_language) do
cache_key = get_cache_key(text, source_language, target_language)
case @cachex.get(:translations_cache, cache_key) do
{:ok, nil} ->
provider = get_provider()
result =
if !configured?() do
{:error, :not_found}
else
provider.translate(text, source_language, target_language)
|> scrub_html()
end
store_result(result, cache_key)
result
{:ok, result} ->
{:ok, result}
{:error, error} ->
{:error, error}
end
end
def supported_languages(type) when type in [:source, :target] do
provider = get_provider()
cache_key = "#{type}_languages/#{provider.name()}"
case @cachex.get(:translations_cache, cache_key) do
{:ok, nil} ->
result =
if !configured?() do
{:error, :not_found}
else
provider.supported_languages(type)
end
store_result(result, cache_key)
result
{:ok, result} ->
{:ok, result}
{:error, error} ->
{:error, error}
end
end
def languages_matrix do
provider = get_provider()
cache_key = "languages_matrix/#{provider.name()}"
case @cachex.get(:translations_cache, cache_key) do
{:ok, nil} ->
result =
if !configured?() do
{:error, :not_found}
else
provider.languages_matrix()
end
store_result(result, cache_key)
result
{:ok, result} ->
{:ok, result}
{:error, error} ->
{:error, error}
end
end
defp get_provider, do: Pleroma.Config.get([__MODULE__, :provider])
defp get_cache_key(text, source_language, target_language) do
"#{source_language}/#{target_language}/#{content_hash(text)}"
end
defp store_result({:ok, result}, cache_key) do
@cachex.put(:translations_cache, cache_key, result)
end
defp store_result(_, _), do: nil
defp content_hash(text), do: :crypto.hash(:sha256, text) |> Base.encode64()
defp scrub_html({:ok, %{content: content} = result}) when is_binary(content) do
scrubbers = Pleroma.Config.get([:markup, :scrub_policy])
content
|> Pleroma.HTML.filter_tags(scrubbers)
{:ok, %{result | content: content}}
end
defp scrub_html(result), do: result
end

121
lib/pleroma/language/translation/deepl.ex Normal file
View file

@ -0,0 +1,121 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Language.Translation.Deepl do
import Pleroma.Web.Utils.Guards, only: [not_empty_string: 1]
alias Pleroma.Language.Translation.Provider
use Provider
@behaviour Provider
@name "DeepL"
@impl Provider
def configured?, do: not_empty_string(base_url()) and not_empty_string(api_key())
@impl Provider
def translate(content, source_language, target_language) do
endpoint =
base_url()
|> URI.merge("/v2/translate")
|> URI.to_string()
case Pleroma.HTTP.post(
endpoint,
Jason.encode!(%{
text: [content],
source_lang: source_language |> String.upcase(),
target_lang: target_language,
tag_handling: "html"
}),
[
{"Content-Type", "application/json"},
{"Authorization", "DeepL-Auth-Key #{api_key()}"}
]
) do
{:ok, %{status: 429}} ->
{:error, :too_many_requests}
{:ok, %{status: 456}} ->
{:error, :quota_exceeded}
{:ok, %{status: 200} = res} ->
%{
"translations" => [
%{"text" => content, "detected_source_language" => detected_source_language}
]
} = Jason.decode!(res.body)
{:ok,
%{
content: content,
detected_source_language: detected_source_language,
provider: @name
}}
_ ->
{:error, :internal_server_error}
end
end
@impl Provider
def supported_languages(type) when type in [:source, :target] do
endpoint =
base_url()
|> URI.merge("/v2/languages")
|> URI.to_string()
case Pleroma.HTTP.post(
endpoint <> "?" <> URI.encode_query(%{type: type}),
"",
[
{"Content-Type", "application/x-www-form-urlencoded"},
{"Authorization", "DeepL-Auth-Key #{api_key()}"}
]
) do
{:ok, %{status: 200} = res} ->
languages =
Jason.decode!(res.body)
|> Enum.map(fn %{"language" => language} -> language |> String.downcase() end)
|> Enum.map(fn language ->
if String.contains?(language, "-") do
[language, language |> String.split("-") |> Enum.at(0)]
else
language
end
end)
|> List.flatten()
|> Enum.uniq()
{:ok, languages}
_ ->
{:error, :internal_server_error}
end
end
@impl Provider
def languages_matrix do
with {:ok, source_languages} <- supported_languages(:source),
{:ok, target_languages} <- supported_languages(:target) do
{:ok,
Map.new(source_languages, fn language -> {language, target_languages -- [language]} end)}
else
{:error, error} -> {:error, error}
end
end
@impl Provider
def name, do: @name
defp base_url do
Pleroma.Config.get([__MODULE__, :base_url])
end
defp api_key do
Pleroma.Config.get([__MODULE__, :api_key])
end
end

93
lib/pleroma/language/translation/libretranslate.ex Normal file
View file

@ -0,0 +1,93 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Language.Translation.Libretranslate do
import Pleroma.Web.Utils.Guards, only: [not_empty_string: 1]
alias Pleroma.Language.Translation.Provider
use Provider
@behaviour Provider
@name "LibreTranslate"
@impl Provider
def configured?, do: not_empty_string(base_url()) and not_empty_string(api_key())
@impl Provider
def translate(content, source_language, target_language) do
case Pleroma.HTTP.post(
base_url() <> "/translate",
Jason.encode!(%{
q: content,
source: source_language |> String.upcase(),
target: target_language,
format: "html",
api_key: api_key()
}),
[
{"Content-Type", "application/json"}
]
) do
{:ok, %{status: 429}} ->
{:error, :too_many_requests}
{:ok, %{status: 403}} ->
{:error, :quota_exceeded}
{:ok, %{status: 200} = res} ->
%{
"translatedText" => content
} = Jason.decode!(res.body)
{:ok,
%{
content: content,
detected_source_language: source_language,
provider: @name
}}
_ ->
{:error, :internal_server_error}
end
end
@impl Provider
def supported_languages(_) do
case Pleroma.HTTP.get(base_url() <> "/languages") do
{:ok, %{status: 200} = res} ->
languages =
Jason.decode!(res.body)
|> Enum.map(fn %{"code" => code} -> code end)
{:ok, languages}
_ ->
{:error, :internal_server_error}
end
end
@impl Provider
def languages_matrix do
with {:ok, source_languages} <- supported_languages(:source),
{:ok, target_languages} <- supported_languages(:target) do
{:ok,
Map.new(source_languages, fn language -> {language, target_languages -- [language]} end)}
else
{:error, error} -> {:error, error}
end
end
@impl Provider
def name, do: @name
defp base_url do
Pleroma.Config.get([__MODULE__, :base_url])
end
defp api_key do
Pleroma.Config.get([__MODULE__, :api_key], "")
end
end

109
lib/pleroma/language/translation/mozhi.ex Normal file
View file

@ -0,0 +1,109 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2024 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Language.Translation.Mozhi do
import Pleroma.Web.Utils.Guards, only: [not_empty_string: 1]
alias Pleroma.Language.Translation.Provider
use Provider
@behaviour Provider
@name "Mozhi"
@impl Provider
def configured?, do: not_empty_string(base_url()) and not_empty_string(engine())
@impl Provider
def translate(content, source_language, target_language) do
endpoint =
base_url()
|> URI.merge("/api/translate")
|> URI.to_string()
case Pleroma.HTTP.get(
endpoint <>
"?" <>
URI.encode_query(%{
engine: engine(),
text: content,
from: source_language,
to: target_language
}),
[{"Accept", "application/json"}]
) do
{:ok, %{status: 200} = res} ->
%{
"translated-text" => content,
"source_language" => source_language
} = Jason.decode!(res.body)
{:ok,
%{
content: content,
detected_source_language: source_language,
provider: @name
}}
_ ->
{:error, :internal_server_error}
end
end
@impl Provider
def supported_languages(type) when type in [:source, :target] do
path =
case type do
:source -> "/api/source_languages"
:target -> "/api/target_languages"
end
endpoint =
base_url()
|> URI.merge(path)
|> URI.to_string()
case Pleroma.HTTP.get(
endpoint <>
"?" <>
URI.encode_query(%{
engine: engine()
}),
[{"Accept", "application/json"}]
) do
{:ok, %{status: 200} = res} ->
languages =
Jason.decode!(res.body)
|> Enum.map(fn %{"Id" => language} -> language end)
{:ok, languages}
_ ->
{:error, :internal_server_error}
end
end
@impl Provider
def languages_matrix do
with {:ok, source_languages} <- supported_languages(:source),
{:ok, target_languages} <- supported_languages(:target) do
{:ok,
Map.new(source_languages, fn language -> {language, target_languages -- [language]} end)}
else
{:error, error} -> {:error, error}
end
end
@impl Provider
def name, do: @name
defp base_url do
Pleroma.Config.get([__MODULE__, :base_url])
end
defp engine do
Pleroma.Config.get([__MODULE__, :engine])
end
end

40
lib/pleroma/language/translation/provider.ex Normal file
View file

@ -0,0 +1,40 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Language.Translation.Provider do
alias Pleroma.Language.Translation.Provider
@callback missing_dependencies() :: [String.t()]
@callback configured?() :: boolean()
@callback translate(
content :: String.t(),
source_language :: String.t(),
target_language :: String.t()
) ::
{:ok,
%{
content: String.t(),
detected_source_language: String.t(),
provider: String.t()
}}
| {:error, atom()}
@callback supported_languages(type :: :string | :target) ::
{:ok, [String.t()]} | {:error, atom()}
@callback languages_matrix() :: {:ok, map()} | {:error, atom()}
@callback name() :: String.t()
defmacro __using__(_opts) do
quote do
@impl Provider
def missing_dependencies, do: []
defoverridable missing_dependencies: 0
end
end
end

129
lib/pleroma/language/translation/translate_locally.ex Normal file
View file

@ -0,0 +1,129 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2024 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Language.Translation.TranslateLocally do
alias Pleroma.Language.Translation.Provider
use Provider
@behaviour Provider
@name "translateLocally"
@impl Provider
def missing_dependencies do
if Pleroma.Utils.command_available?("translateLocally") do
[]
else
["translateLocally"]
end
end
@impl Provider
def configured?, do: is_map(models())
@impl Provider
def translate(content, source_language, target_language) do
model =
models()
|> Map.get(source_language, %{})
|> Map.get(target_language)
models =
if model do
[model]
else
[
models()
|> Map.get(source_language, %{})
|> Map.get(intermediary_language()),
models()
|> Map.get(intermediary_language(), %{})
|> Map.get(target_language)
]
end
translated_content =
Enum.reduce(models, content, fn model, content ->
text_path = Path.join(System.tmp_dir!(), "translateLocally-#{Ecto.UUID.generate()}")
File.write(text_path, content)
translated_content =
case System.cmd("translateLocally", ["-m", model, "-i", text_path, "--html"]) do
{content, _} -> content
_ -> nil
end
File.rm(text_path)
translated_content
end)
{:ok,
%{
content: translated_content,
detected_source_language: source_language,
provider: @name
}}
end
@impl Provider
def supported_languages(:source) do
languages =
languages_matrix()
|> elem(1)
|> Map.keys()
{:ok, languages}
end
@impl Provider
def supported_languages(:target) do
languages =
languages_matrix()
|> elem(1)
|> Map.values()
|> List.flatten()
|> Enum.uniq()
{:ok, languages}
end
@impl Provider
def languages_matrix do
languages =
models()
|> Map.to_list()
|> Enum.map(fn {key, value} -> {key, Map.keys(value)} end)
|> Enum.into(%{})
matrix =
if intermediary_language() do
languages
|> Map.to_list()
|> Enum.map(fn {key, value} ->
with_intermediary =
(((value ++ languages[intermediary_language()])
|> Enum.uniq()) --
[key])
|> Enum.sort()
{key, with_intermediary}
end)
|> Enum.into(%{})
else
languages
end
{:ok, matrix}
end
@impl Provider
def name, do: @name
defp models, do: Pleroma.Config.get([__MODULE__, :models])
defp intermediary_language, do: Pleroma.Config.get([__MODULE__, :intermediary_language])
end

271
lib/pleroma/ldap.ex Normal file
View file

@ -0,0 +1,271 @@
defmodule Pleroma.LDAP do
use GenServer
require Logger
alias Pleroma.Config
alias Pleroma.User
import Pleroma.Web.Auth.Helpers, only: [fetch_user: 1]
@connection_timeout 2_000
@search_timeout 2_000
def start_link(_) do
GenServer.start_link(__MODULE__, [], name: __MODULE__)
end
def bind_user(name, password) do
GenServer.call(__MODULE__, {:bind_user, name, password})
end
def change_password(name, password, new_password) do
GenServer.call(__MODULE__, {:change_password, name, password, new_password})
end
@impl true
def init(state) do
case {Config.get(Pleroma.Web.Auth.Authenticator), Config.get([:ldap, :enabled])} do
{Pleroma.Web.Auth.LDAPAuthenticator, true} ->
{:ok, state, {:continue, :connect}}
{Pleroma.Web.Auth.LDAPAuthenticator, false} ->
Logger.error(
"LDAP Authenticator enabled but :pleroma, :ldap is not enabled. Auth will not work."
)
{:ok, state}
{_, true} ->
Logger.warning(
":pleroma, :ldap is enabled but Pleroma.Web.Authenticator is not set to the LDAPAuthenticator. LDAP will not be used."
)
{:ok, state}
_ ->
{:ok, state}
end
end
@impl true
def handle_continue(:connect, _state), do: do_handle_connect()
@impl true
def handle_info(:connect, _state), do: do_handle_connect()
def handle_info({:bind_after_reconnect, name, password, from}, state) do
result = do_bind_user(state[:handle], name, password)
GenServer.reply(from, result)
{:noreply, state}
end
@impl true
def handle_call({:bind_user, name, password}, from, state) do
case do_bind_user(state[:handle], name, password) do
:needs_reconnect ->
Process.send(self(), {:bind_after_reconnect, name, password, from}, [])
{:noreply, state, {:continue, :connect}}
result ->
{:reply, result, state, :hibernate}
end
end
def handle_call({:change_password, name, password, new_password}, _from, state) do
result = change_password(state[:handle], name, password, new_password)
{:reply, result, state, :hibernate}
end
@impl true
def terminate(_, state) do
handle = Keyword.get(state, :handle)
if not is_nil(handle) do
:eldap.close(handle)
end
:ok
end
defp do_handle_connect do
state =
case connect() do
{:ok, handle} ->
:eldap.controlling_process(handle, self())
Process.link(handle)
[handle: handle]
_ ->
Logger.error("Failed to connect to LDAP. Retrying in 5000ms")
Process.send_after(self(), :connect, 5_000)
[]
end
{:noreply, state}
end
defp connect do
ldap = Config.get(:ldap, [])
host = Keyword.get(ldap, :host, "localhost")
port = Keyword.get(ldap, :port, 389)
ssl = Keyword.get(ldap, :ssl, false)
tls = Keyword.get(ldap, :tls, false)
cacertfile = Keyword.get(ldap, :cacertfile) || CAStore.file_path()
if ssl, do: Application.ensure_all_started(:ssl)
default_secure_opts = [
verify: :verify_peer,
cacerts: decode_certfile(cacertfile),
customize_hostname_check: [
fqdn_fun: fn _ -> to_charlist(host) end
]
]
sslopts = Keyword.merge(default_secure_opts, Keyword.get(ldap, :sslopts, []))
tlsopts = Keyword.merge(default_secure_opts, Keyword.get(ldap, :tlsopts, []))
default_options = [{:port, port}, {:ssl, ssl}, {:timeout, @connection_timeout}]
# :sslopts can only be included in :eldap.open/2 when {ssl: true}
# or the connection will fail
options =
if ssl do
default_options ++ [{:sslopts, sslopts}]
else
default_options
end
case :eldap.open([to_charlist(host)], options) do
{:ok, handle} ->
try do
cond do
tls ->
case :eldap.start_tls(
handle,
tlsopts,
@connection_timeout
) do
:ok ->
{:ok, handle}
error ->
Logger.error("Could not start TLS: #{inspect(error)}")
:eldap.close(handle)
end
true ->
{:ok, handle}
end
after
:ok
end
{:error, error} ->
Logger.error("Could not open LDAP connection: #{inspect(error)}")
{:error, {:ldap_connection_error, error}}
end
end
defp do_bind_user(handle, name, password) do
dn = make_dn(name)
case :eldap.simple_bind(handle, dn, password) do
:ok ->
case fetch_user(name) do
%User{} = user ->
user
_ ->
register_user(handle, ldap_base(), ldap_uid(), name)
end
# eldap does not inform us of socket closure
# until it is used
{:error, {:gen_tcp_error, :closed}} ->
:eldap.close(handle)
:needs_reconnect
{:error, error} = e ->
Logger.error("Could not bind LDAP user #{name}: #{inspect(error)}")
e
end
end
defp register_user(handle, base, uid, name) do
case :eldap.search(handle, [
{:base, to_charlist(base)},
{:filter, :eldap.equalityMatch(to_charlist(uid), to_charlist(name))},
{:scope, :eldap.wholeSubtree()},
{:timeout, @search_timeout}
]) do
# The :eldap_search_result record structure changed in OTP 24.3 and added a controls field
# https://github.com/erlang/otp/pull/5538
{:ok, {:eldap_search_result, [{:eldap_entry, _object, attributes}], _referrals}} ->
try_register(name, attributes)
{:ok, {:eldap_search_result, [{:eldap_entry, _object, attributes}], _referrals, _controls}} ->
try_register(name, attributes)
error ->
Logger.error("Couldn't register user because LDAP search failed: #{inspect(error)}")
{:error, {:ldap_search_error, error}}
end
end
defp try_register(name, attributes) do
mail_attribute = Config.get([:ldap, :mail])
params = %{
name: name,
nickname: name,
password: nil
}
params =
case List.keyfind(attributes, to_charlist(mail_attribute), 0) do
{_, [mail]} -> Map.put_new(params, :email, :erlang.list_to_binary(mail))
_ -> params
end
changeset = User.register_changeset_ldap(%User{}, params)
case User.register(changeset) do
{:ok, user} -> user
error -> error
end
end
defp change_password(handle, name, password, new_password) do
dn = make_dn(name)
with :ok <- :eldap.simple_bind(handle, dn, password) do
:eldap.modify_password(handle, dn, to_charlist(new_password), to_charlist(password))
end
end
defp decode_certfile(file) do
with {:ok, data} <- File.read(file) do
data
|> :public_key.pem_decode()
|> Enum.map(fn {_, b, _} -> b end)
else
_ ->
Logger.error("Unable to read certfile: #{file}")
[]
end
end
defp ldap_uid, do: to_charlist(Config.get([:ldap, :uid], "cn"))
defp ldap_base, do: to_charlist(Config.get([:ldap, :base]))
defp make_dn(name) do
uid = ldap_uid()
base = ldap_base()
~c"#{uid}=#{name},#{base}"
end
end

63
lib/pleroma/list.ex
View file

@ -9,6 +9,7 @@ defmodule Pleroma.List do
import Ecto.Changeset
alias Pleroma.Activity
alias Pleroma.Emoji
alias Pleroma.Repo
alias Pleroma.User
@ -17,14 +18,54 @@ defmodule Pleroma.List do
field(:title, :string)
field(:following, {:array, :string}, default: [])
field(:ap_id, :string)
field(:exclusive, :boolean, default: false)
field(:emoji, :string)
timestamps()
end
def title_changeset(list, attrs \\ %{}) do
def update_changeset(list, attrs \\ %{}) do
list
|> cast(attrs, [:title])
|> cast(attrs, [:title, :exclusive, :emoji])
|> validate_required([:title])
|> fix_emoji()
|> validate_emoji()
end
defp fix_emoji(changeset) do
with {:emoji_field, emoji} when is_binary(emoji) <-
{:emoji_field, get_field(changeset, :emoji)},
{:fixed_emoji, emoji} <-
{:fixed_emoji,
emoji
|> Emoji.fully_qualify_emoji()
|> Emoji.maybe_quote()} do
put_change(changeset, :emoji, emoji)
else
{:emoji_field, _} -> changeset
end
end
defp validate_emoji(changeset) do
validate_change(changeset, :emoji, fn
:emoji, nil ->
[]
:emoji, emoji ->
if Emoji.unicode?(emoji) or valid_local_custom_emoji?(emoji) do
[]
else
[emoji: "Invalid emoji"]
end
end)
end
defp valid_local_custom_emoji?(emoji) do
with %{file: _path} <- Emoji.get(emoji) do
true
else
_ -> false
end
end
def follow_changeset(list, attrs \\ %{}) do
@ -91,14 +132,14 @@ defmodule Pleroma.List do
|> Repo.all()
end
def rename(%Pleroma.List{} = list, title) do
def update(%Pleroma.List{} = list, params) do
list
|> title_changeset(%{title: title})
|> update_changeset(params)
|> Repo.update()
end
def create(title, %User{} = creator) do
changeset = title_changeset(%Pleroma.List{user_id: creator.id}, %{title: title})
def create(params, %User{} = creator) do
changeset = update_changeset(%Pleroma.List{user_id: creator.id}, params)
if changeset.valid? do
Repo.transaction(fn ->
@ -149,4 +190,14 @@ defmodule Pleroma.List do
end
def member?(_, _), do: false
def get_exclusive_list_members(%User{id: user_id}) do
Pleroma.List
|> where([l], l.user_id == ^user_id)
|> where([l], l.exclusive == true)
|> select([l], l.following)
|> Repo.all()
|> List.flatten()
|> Enum.uniq()
end
end

4
lib/pleroma/maps.ex
View file

@ -20,15 +20,13 @@ defmodule Pleroma.Maps do
end
def filter_empty_values(data) do
# TODO: Change to Map.filter in Elixir 1.13+
data
|> Enum.filter(fn
|> Map.filter(fn
{_k, nil} -> false
{_k, ""} -> false
{_k, []} -> false
{_k, %{} = v} -> Map.keys(v) != []
{_k, _v} -> true
end)
|> Map.new()
end
end

2
lib/pleroma/marker.ex
View file

@ -78,7 +78,7 @@ defmodule Pleroma.Marker do
defp get_marker(user, timeline) do
case Repo.find_resource(get_query(user, timeline)) do
{:ok, marker} -> %__MODULE__{marker | user: user}
{:ok, %__MODULE__{} = marker} -> %{marker | user: user}
_ -> %__MODULE__{timeline: timeline, user_id: user.id}
end
end

13
lib/pleroma/mfa/changeset.ex
View file

@ -8,7 +8,8 @@ defmodule Pleroma.MFA.Changeset do
alias Pleroma.User
def disable(%Ecto.Changeset{} = changeset, force \\ false) do
settings =
%Settings{} =
settings =
changeset
|> Ecto.Changeset.apply_changes()
|> MFA.fetch_settings()
@ -20,20 +21,20 @@ defmodule Pleroma.MFA.Changeset do
end
end
def disable_totp(%User{multi_factor_authentication_settings: settings} = user) do
def disable_totp(%User{multi_factor_authentication_settings: %Settings{} = settings} = user) do
user
|> put_change(%Settings{settings | totp: %Settings.TOTP{}})
end
def confirm_totp(%User{multi_factor_authentication_settings: settings} = user) do
totp_settings = %Settings.TOTP{settings.totp | confirmed: true}
def confirm_totp(%User{multi_factor_authentication_settings: %Settings{} = settings} = user) do
totp_settings = %Settings.TOTP{(%Settings.TOTP{} = settings.totp) | confirmed: true}
user
|> put_change(%Settings{settings | totp: totp_settings, enabled: true})
end
def setup_totp(%User{} = user, attrs) do
mfa_settings = MFA.fetch_settings(user)
%Settings{} = mfa_settings = MFA.fetch_settings(user)
totp_settings =
%Settings.TOTP{}
@ -46,7 +47,7 @@ defmodule Pleroma.MFA.Changeset do
def cast_backup_codes(%User{} = user, codes) do
user
|> put_change(%Settings{
user.multi_factor_authentication_settings
(%Settings{} = user.multi_factor_authentication_settings)
| backup_codes: codes
})
end

13
lib/pleroma/mfa/token.ex
View file

@ -52,11 +52,14 @@ defmodule Pleroma.MFA.Token do
@spec create(User.t(), Authorization.t() | nil) :: {:ok, t()} | {:error, Ecto.Changeset.t()}
def create(user, authorization \\ nil) do
with {:ok, token} <- do_create(user, authorization) do
Pleroma.Workers.PurgeExpiredToken.enqueue(%{
token_id: token.id,
valid_until: DateTime.from_naive!(token.valid_until, "Etc/UTC"),
mod: __MODULE__
})
Pleroma.Workers.PurgeExpiredToken.new(
%{
token_id: token.id,
mod: __MODULE__
},
scheduled_at: DateTime.from_naive!(token.valid_until, "Etc/UTC")
)
|> Oban.insert()
{:ok, token}
end

44
lib/pleroma/moderation_log.ex
View file

@ -132,11 +132,18 @@ defmodule Pleroma.ModerationLog do
end
def insert_log(%{actor: %User{}, action: action, subject: %Activity{} = subject} = attrs)
when action in ["report_note_delete", "report_update", "report_note"] do
when action in [
"report_note_delete",
"report_update",
"report_note",
"report_unassigned",
"report_assigned"
] do
data =
attrs
|> prepare_log_data
|> Pleroma.Maps.put_if_present("text", attrs[:text])
|> Pleroma.Maps.put_if_present("assigned_account", attrs[:assigned_account])
|> Map.merge(%{"subject" => report_to_map(subject)})
insert_log_entry_with_message(%ModerationLog{data: data})
@ -441,6 +448,35 @@ defmodule Pleroma.ModerationLog do
" with '#{state}' state"
end
def get_log_entry_message(
%ModerationLog{
data: %{
"actor" => %{"nickname" => actor_nickname},
"action" => "report_assigned",
"subject" => %{"id" => subject_id, "type" => "report"},
"assigned_account" => assigned_account
}
} = log
) do
"@#{actor_nickname} assigned report ##{subject_id}" <>
subject_actor_nickname(log, " (on user ", ")") <>
" to user #{assigned_account}"
end
def get_log_entry_message(
%ModerationLog{
data: %{
"actor" => %{"nickname" => actor_nickname},
"action" => "report_unassigned",
"subject" => %{"id" => subject_id, "type" => "report"}
}
} = log
) do
"@#{actor_nickname} unassigned report ##{subject_id}" <>
subject_actor_nickname(log, " (on user ", ")") <>
" from a user"
end
def get_log_entry_message(
%ModerationLog{
data: %{
@ -575,6 +611,12 @@ defmodule Pleroma.ModerationLog do
"@#{actor_nickname} requested account backup for @#{user_nickname}"
end
def get_log_entry_message(%ModerationLog{data: data}) do
actor_name = get_in(data, ["actor", "nickname"]) || "unknown"
action = data["action"] || "unknown"
"@#{actor_name} performed action #{action}"
end
defp nicknames_to_string(nicknames) do
nicknames
|> Enum.map(&"@#{&1}")

15
lib/pleroma/mogrify_behaviour.ex Normal file
View file

@ -0,0 +1,15 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.MogrifyBehaviour do
@moduledoc """
Behaviour for Mogrify operations.
This module defines the interface for Mogrify operations that can be mocked in tests.
"""
@callback open(binary()) :: map()
@callback custom(map(), binary()) :: map()
@callback custom(map(), binary(), binary()) :: map()
@callback save(map(), keyword()) :: map()
end

30
lib/pleroma/mogrify_wrapper.ex Normal file
View file

@ -0,0 +1,30 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.MogrifyWrapper do
@moduledoc """
Default implementation of MogrifyBehaviour that delegates to Mogrify.
"""
@behaviour Pleroma.MogrifyBehaviour
@impl true
def open(file) do
Mogrify.open(file)
end
@impl true
def custom(image, action) do
Mogrify.custom(image, action)
end
@impl true
def custom(image, action, options) do
Mogrify.custom(image, action, options)
end
@impl true
def save(image, opts) do
Mogrify.save(image, opts)
end
end

126
lib/pleroma/notification.ex
View file

@ -35,6 +35,7 @@ defmodule Pleroma.Notification do
# remember to add a migration to add it to the `notifications_type` enum
# as well.
field(:type, :string)
field(:group_key, :string)
belongs_to(:user, User, type: FlakeId.Ecto.CompatType)
belongs_to(:activity, Activity, type: FlakeId.Ecto.CompatType)
@ -49,7 +50,7 @@ defmodule Pleroma.Notification do
|> type_from_activity()
notification
|> changeset(%{type: type})
|> changeset(%{type: type, group_key: grouped_notification_key(type, activity)})
|> Repo.update()
end
end
@ -62,6 +63,99 @@ defmodule Pleroma.Notification do
|> Repo.aggregate(:count, :id)
end
@groupable_notification_types ~w{favourite follow reblog}
@group_bucket_seconds 12 * 60 * 60
def groupable_notification_types, do: @groupable_notification_types
def group_notifications(notifications, grouped_types \\ nil) do
grouped_types = normalize_grouped_types(grouped_types)
{group_keys, grouped_notifications} =
Enum.reduce(notifications, {[], %{}}, fn notification,
{group_keys, grouped_notifications} ->
group_key = group_key(notification, grouped_types)
if Map.has_key?(grouped_notifications, group_key) do
{group_keys, Map.update!(grouped_notifications, group_key, &[notification | &1])}
else
{[group_key | group_keys], Map.put(grouped_notifications, group_key, [notification])}
end
end)
group_keys
|> Enum.reverse()
|> Enum.map(fn group_key ->
grouped_notifications
|> Map.fetch!(group_key)
|> Enum.reverse()
end)
end
def group_key(notification, grouped_types \\ nil)
def group_key(%Notification{} = notification, nil) do
group_key(notification, @groupable_notification_types)
end
def group_key(%Notification{type: type, group_key: group_key} = notification, grouped_types) do
grouped_types = normalize_grouped_types(grouped_types)
if type in @groupable_notification_types and type in grouped_types and is_binary(group_key) do
group_key
else
ungrouped_group_key(notification)
end
end
def normalize_grouped_types(nil), do: @groupable_notification_types
def normalize_grouped_types(types) when is_list(types), do: types
def normalize_grouped_types(type), do: [type]
defp ungrouped_group_key(%Notification{id: id}), do: "ungrouped-#{id}"
defp grouped_notification_key(type, activity) when type in @groupable_notification_types do
with target_id when is_binary(target_id) <- group_target_id(type, activity) do
# Mastodon uses Redis to reuse a recent bucket for rolling 12h windows. Pleroma keeps keys
# deterministic and self-contained; clients must treat group_key as opaque either way.
"#{type}-#{target_id}-#{group_time_bucket(activity)}"
end
end
defp grouped_notification_key(_type, _activity), do: nil
defp group_time_bucket(%Activity{inserted_at: inserted_at}) when not is_nil(inserted_at) do
inserted_at
|> NaiveDateTime.to_erl()
|> :calendar.datetime_to_gregorian_seconds()
|> div(@group_bucket_seconds)
end
defp group_time_bucket(_),
do: group_time_bucket(%Activity{inserted_at: NaiveDateTime.utc_now()})
defp group_target_id(type, activity) when type in ["favourite", "reblog"] do
with object_id when is_binary(object_id) <- object_id_for(activity),
%Activity{id: id} <- Activity.get_create_by_object_ap_id(object_id) do
to_string(id)
else
_ -> nil
end
end
defp group_target_id("follow", %{data: %{"object" => ap_id}}) do
case User.get_cached_by_ap_id(ap_id) do
%User{id: id} -> to_string(id)
_ -> nil
end
end
defp group_target_id(_, _), do: nil
defp object_id_for(%{data: %{"object" => %{"id" => id}}}) when is_binary(id), do: id
defp object_id_for(%{data: %{"object" => id}}) when is_binary(id), do: id
defp object_id_for(_), do: nil
@notification_types ~w{
favourite
follow
@ -74,11 +168,12 @@ defmodule Pleroma.Notification do
reblog
poll
status
update
}
def changeset(%Notification{} = notification, attrs) do
notification
|> cast(attrs, [:seen, :type])
|> cast(attrs, [:seen, :type, :group_key])
|> validate_inclusion(:type, @notification_types)
end
@ -281,10 +376,15 @@ defmodule Pleroma.Notification do
select: n.id
)
Multi.new()
|> Multi.update_all(:ids, query, set: [seen: true, updated_at: NaiveDateTime.utc_now()])
|> Marker.multi_set_last_read_id(user, "notifications")
|> Repo.transaction()
{:ok, %{marker: marker}} =
Multi.new()
|> Multi.update_all(:ids, query, set: [seen: true, updated_at: NaiveDateTime.utc_now()])
|> Marker.multi_set_last_read_id(user, "notifications")
|> Repo.transaction()
Streamer.stream(["user", "user:notification"], marker)
{:ok, %{marker: marker}}
end
@spec read_one(User.t(), String.t()) ::
@ -452,7 +552,8 @@ defmodule Pleroma.Notification do
user_id: user.id,
activity: activity,
seen: mark_as_read?(activity, user),
type: type
type: type,
group_key: grouped_notification_key(type, activity)
})
|> Marker.multi_set_last_read_id(user, "notifications")
|> Repo.transaction()
@ -525,9 +626,7 @@ defmodule Pleroma.Notification do
%Activity{data: %{"type" => "Create"}} = activity,
local_only
) do
notification_enabled_ap_ids =
[]
|> Utils.maybe_notify_subscribers(activity)
notification_enabled_ap_ids = Utils.get_notified_subscribers(activity)
potential_receivers =
User.get_users_from_set(notification_enabled_ap_ids, local_only: local_only)
@ -734,7 +833,7 @@ defmodule Pleroma.Notification do
def mark_as_read?(activity, target_user) do
user = Activity.user_actor(activity)
User.mutes_user?(target_user, user) || CommonAPI.thread_muted?(target_user, activity)
User.mutes_user?(target_user, user) || CommonAPI.thread_muted?(activity, target_user)
end
def for_user_and_activity(user, activity) do
@ -757,8 +856,9 @@ defmodule Pleroma.Notification do
|> Repo.update_all(set: [seen: true])
end
@spec send(list(Notification.t())) :: :ok
def send(notifications) do
@doc "Streams a list of notifications over websockets and web push"
@spec stream(list(Notification.t())) :: :ok
def stream(notifications) do
Enum.each(notifications, fn notification ->
Streamer.stream(["user", "user:notification"], notification)
Push.send(notification)

66
lib/pleroma/object.ex
View file

@ -99,24 +99,6 @@ defmodule Pleroma.Object do
def get_by_id(nil), do: nil
def get_by_id(id), do: Repo.get(Object, id)
def get_by_id_and_maybe_refetch(id, opts \\ []) do
%{updated_at: updated_at} = object = get_by_id(id)
if opts[:interval] &&
NaiveDateTime.diff(NaiveDateTime.utc_now(), updated_at) > opts[:interval] do
case Fetcher.refetch_object(object) do
{:ok, %Object{} = object} ->
object
e ->
Logger.error("Couldn't refresh #{object.data["id"]}:\n#{inspect(e)}")
object
end
else
object
end
end
def get_by_ap_id(nil), do: nil
def get_by_ap_id(ap_id) do
@ -144,7 +126,7 @@ defmodule Pleroma.Object do
Logger.debug("Backtrace: #{inspect(Process.info(:erlang.self(), :current_stacktrace))}")
end
def normalize(_, options \\ [fetch: false, id_only: false])
def normalize(_, options \\ [fetch: false])
# If we pass an Activity to Object.normalize(), we can try to use the preloaded object.
# Use this whenever possible, especially when walking graphs in an O(N) loop!
@ -173,9 +155,6 @@ defmodule Pleroma.Object do
def normalize(ap_id, options) when is_binary(ap_id) do
cond do
Keyword.get(options, :id_only) ->
ap_id
Keyword.get(options, :fetch) ->
case Fetcher.fetch_object_from_id(ap_id, options) do
{:ok, object} -> object
@ -252,7 +231,8 @@ defmodule Pleroma.Object do
@spec cleanup_attachments(boolean(), Object.t()) ::
{:ok, Oban.Job.t() | nil}
def cleanup_attachments(true, %Object{} = object) do
AttachmentsCleanupWorker.enqueue("cleanup_attachments", %{"object" => object})
AttachmentsCleanupWorker.new(%{"op" => "cleanup_attachments", "object" => object})
|> Oban.insert()
end
def cleanup_attachments(_, _), do: {:ok, nil}
@ -392,12 +372,28 @@ defmodule Pleroma.Object do
option
end)
voters = [actor | object.data["voters"] || []] |> Enum.uniq()
existing_voters = object.data["voters"] || []
voters = [actor | existing_voters] |> Enum.uniq()
new_voter? = actor not in existing_voters
existing_voters_count = object.data["votersCount"]
voters_count =
cond do
is_integer(existing_voters_count) and new_voter? ->
existing_voters_count + 1
is_integer(existing_voters_count) ->
existing_voters_count
true ->
length(voters)
end
data =
object.data
|> Map.put(key, options)
|> Map.put("voters", voters)
|> Map.put("votersCount", voters_count)
object
|> Object.change(%{data: data})
@ -418,28 +414,6 @@ defmodule Pleroma.Object do
String.starts_with?(id, Pleroma.Web.Endpoint.url() <> "/")
end
def replies(object, opts \\ []) do
object = Object.normalize(object, fetch: false)
query =
Object
|> where(
[o],
fragment("(?)->>'inReplyTo' = ?", o.data, ^object.data["id"])
)
|> order_by([o], asc: o.id)
if opts[:self_only] do
actor = object.data["actor"]
where(query, [o], fragment("(?)->>'actor' = ?", o.data, ^actor))
else
query
end
end
def self_replies(object, opts \\ []),
do: replies(object, Keyword.put(opts, :self_only, true))
def tags(%Object{data: %{"tag" => tags}}) when is_list(tags), do: tags
def tags(_), do: []

49
lib/pleroma/object/containment.ex
View file

@ -15,12 +15,28 @@ defmodule Pleroma.Object.Containment do
actor
end
def get_actor(%{"actor" => nil, "attributedTo" => actor}) when not is_nil(actor) do
get_actor(%{"actor" => actor})
end
def get_actor(%{"actor" => nil}), do: nil
def get_actor(%{"actor" => []}), do: nil
def get_actor(%{"actor" => actor}) when is_list(actor) do
if is_binary(Enum.at(actor, 0)) do
Enum.at(actor, 0)
else
Enum.find(actor, fn %{"type" => type} -> type in ["Person", "Service", "Application"] end)
|> Map.get("id")
case actor do
[first | _] when is_binary(first) ->
first
_ ->
Enum.find_value(actor, fn
%{"type" => type, "id" => id}
when type in ["Person", "Service", "Application"] and is_binary(id) ->
id
_ ->
nil
end)
end
end
@ -28,10 +44,6 @@ defmodule Pleroma.Object.Containment do
id
end
def get_actor(%{"actor" => nil, "attributedTo" => actor}) when not is_nil(actor) do
get_actor(%{"actor" => actor})
end
def get_object(%{"object" => id}) when is_binary(id) do
id
end
@ -47,6 +59,19 @@ defmodule Pleroma.Object.Containment do
defp compare_uris(%URI{host: host} = _id_uri, %URI{host: host} = _other_uri), do: :ok
defp compare_uris(_id_uri, _other_uri), do: :error
@doc """
Checks whether an URL to fetch from is from the local server.
We never want to fetch from ourselves; if it's not in the database
it can't be authentic and must be a counterfeit.
"""
def contain_local_fetch(id) do
case compare_uris(URI.parse(id), Pleroma.Web.Endpoint.struct_url()) do
:ok -> :error
_ -> :ok
end
end
@doc """
Checks that an imported AP object's actor matches the host it came from.
"""
@ -54,9 +79,11 @@ defmodule Pleroma.Object.Containment do
def contain_origin(id, %{"actor" => _actor} = params) do
id_uri = URI.parse(id)
actor_uri = URI.parse(get_actor(params))
compare_uris(actor_uri, id_uri)
case get_actor(params) do
actor when is_binary(actor) -> compare_uris(URI.parse(actor), id_uri)
_ -> :error
end
end
def contain_origin(id, %{"attributedTo" => actor} = params),

107
lib/pleroma/object/fetcher.ex
View file

@ -4,7 +4,6 @@
defmodule Pleroma.Object.Fetcher do
alias Pleroma.HTTP
alias Pleroma.Instances
alias Pleroma.Maps
alias Pleroma.Object
alias Pleroma.Object.Containment
@ -58,7 +57,12 @@ defmodule Pleroma.Object.Fetcher do
end
end
@typep fetcher_errors ::
:error | :reject | :allowed_depth | :fetch | :containment | :transmogrifier
# Note: will create a Create activity, which we need internally at the moment.
@spec fetch_object_from_id(String.t(), list()) ::
{:ok, Object.t()} | {fetcher_errors(), any()} | Pipeline.errors()
def fetch_object_from_id(id, options \\ []) do
with {_, nil} <- {:fetch_object, Object.get_cached_by_ap_id(id)},
{_, true} <- {:allowed_depth, Federator.allowed_thread_distance?(options[:depth])},
@ -72,50 +76,22 @@ defmodule Pleroma.Object.Fetcher do
{:object, data, Object.normalize(activity, fetch: false)} do
{:ok, object}
else
{:allowed_depth, false} = e ->
log_fetch_error(id, e)
{:error, :allowed_depth}
{:containment, reason} = e ->
log_fetch_error(id, e)
{:error, reason}
{:transmogrifier, {:error, {:reject, reason}}} = e ->
log_fetch_error(id, e)
{:reject, reason}
{:transmogrifier, {:reject, reason}} = e ->
log_fetch_error(id, e)
{:reject, reason}
{:transmogrifier, reason} = e ->
log_fetch_error(id, e)
{:error, reason}
{:object, data, nil} ->
reinject_object(%Object{}, data)
{:normalize, object = %Object{}} ->
{:ok, object}
{:fetch_object, %Object{} = object} ->
{:ok, object}
{:fetch, {:error, reason}} = e ->
log_fetch_error(id, e)
{:error, reason}
{:object, data, nil} ->
reinject_object(%Object{}, data)
e ->
log_fetch_error(id, e)
{:error, e}
Logger.metadata(object: id)
Logger.error("Object rejected while fetching #{id} #{inspect(e)}")
e
end
end
defp log_fetch_error(id, error) do
Logger.metadata(object: id)
Logger.error("Object rejected while fetching #{id} #{inspect(error)}")
end
defp prepare_activity_params(data) do
%{
"type" => "Create",
@ -168,21 +144,25 @@ defmodule Pleroma.Object.Fetcher do
Logger.debug("Fetching object #{id} via AP")
with {:scheme, true} <- {:scheme, String.starts_with?(id, "http")},
{_, true} <- {:mrf, MRF.id_filter(id)},
{_, :ok} <- {:local_fetch, Containment.contain_local_fetch(id)},
{:ok, body} <- get_object(id),
{:ok, data} <- safe_json_decode(body),
:ok <- Containment.contain_origin_from_id(id, data) do
if not Instances.reachable?(id) do
Instances.set_reachable(id)
end
{:ok, data}
else
{:scheme, _} ->
{:error, "Unsupported URI scheme"}
{:local_fetch, _} ->
{:error, "Trying to fetch local resource"}
{:error, e} ->
{:error, e}
{:mrf, false} ->
{:error, {:reject, "Filtered by id"}}
e ->
{:error, e}
end
@ -191,6 +171,14 @@ defmodule Pleroma.Object.Fetcher do
def fetch_and_contain_remote_object_from_id(_id),
do: {:error, "id must be a string"}
defp check_crossdomain_redirect(final_host, _original_url) when is_nil(final_host) do
{:cross_domain_redirect, false}
end
defp check_crossdomain_redirect(final_host, original_url) do
{:cross_domain_redirect, final_host != URI.parse(original_url).host}
end
defp get_object(id) do
date = Pleroma.Signature.signed_date()
@ -200,19 +188,29 @@ defmodule Pleroma.Object.Fetcher do
|> sign_fetch(id, date)
case HTTP.get(id, headers) do
{:ok, %{body: body, status: code, headers: headers, url: final_url}}
when code in 200..299 ->
remote_host = if final_url, do: URI.parse(final_url).host, else: nil
with {:cross_domain_redirect, false} <- check_crossdomain_redirect(remote_host, id),
{_, content_type} <- List.keyfind(headers, "content-type", 0),
{:ok, _media_type} <- verify_content_type(content_type) do
{:ok, body}
else
{:cross_domain_redirect, true} ->
{:error, {:cross_domain_redirect, true}}
error ->
error
end
# Handle the case where URL is not in the response (older HTTP library versions)
{:ok, %{body: body, status: code, headers: headers}} when code in 200..299 ->
case List.keyfind(headers, "content-type", 0) do
{_, content_type} ->
case Plug.Conn.Utils.media_type(content_type) do
{:ok, "application", "activity+json", _} ->
{:ok, body}
{:ok, "application", "ld+json",
%{"profile" => "https://www.w3.org/ns/activitystreams"}} ->
{:ok, body}
_ ->
{:error, {:content_type, content_type}}
case verify_content_type(content_type) do
{:ok, _} -> {:ok, body}
error -> error
end
_ ->
@ -235,4 +233,17 @@ defmodule Pleroma.Object.Fetcher do
defp safe_json_decode(nil), do: {:ok, nil}
defp safe_json_decode(json), do: Jason.decode(json)
defp verify_content_type(content_type) do
case Plug.Conn.Utils.media_type(content_type) do
{:ok, "application", "activity+json", _} ->
{:ok, :activity_json}
{:ok, "application", "ld+json", %{"profile" => "https://www.w3.org/ns/activitystreams"}} ->
{:ok, :ld_json}
_ ->
{:error, {:content_type, content_type}}
end
end
end

7
lib/pleroma/object/updater.ex
View file

@ -5,6 +5,7 @@
defmodule Pleroma.Object.Updater do
require Pleroma.Constants
alias Pleroma.Maps
alias Pleroma.Object
alias Pleroma.Repo
@ -115,6 +116,7 @@ defmodule Pleroma.Object.Updater do
# Choices are the same, but counts are different
to_be_updated
|> Map.put(key, updated_object[key])
|> Maps.put_if_present("votersCount", updated_object["votersCount"])
else
# Choices (or vote type) have changed, do not allow this
_ -> to_be_updated
@ -134,7 +136,10 @@ defmodule Pleroma.Object.Updater do
else
%{updated_object: updated_data} =
updated_data
|> maybe_update_history(original_data, updated: updated, use_history_in_new_object?: false)
|> maybe_update_history(original_data,
updated: updated,
use_history_in_new_object?: false
)
updated_data
|> Map.put("updated", date)

28
lib/pleroma/otp_version.ex
View file

@ -1,28 +0,0 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.OTPVersion do
@spec version() :: String.t() | nil
def version do
# OTP Version https://erlang.org/doc/system_principles/versions.html#otp-version
[
Path.join(:code.root_dir(), "OTP_VERSION"),
Path.join([:code.root_dir(), "releases", :erlang.system_info(:otp_release), "OTP_VERSION"])
]
|> get_version_from_files()
end
@spec get_version_from_files([Path.t()]) :: String.t() | nil
def get_version_from_files([]), do: nil
def get_version_from_files([path | paths]) do
if File.exists?(path) do
path
|> File.read!()
|> String.replace(~r/\r|\n|\s/, "")
else
get_version_from_files(paths)
end
end
end

52
lib/pleroma/pagination.ex
View file

@ -89,19 +89,36 @@ defmodule Pleroma.Pagination do
defp cast_params(params) do
param_types = %{
min_id: :string,
since_id: :string,
max_id: :string,
min_id: params[:id_type] || :string,
since_id: params[:id_type] || :string,
max_id: params[:id_type] || :string,
offset: :integer,
limit: :integer,
skip_extra_order: :boolean,
skip_order: :boolean
skip_order: :boolean,
order_asc: :boolean
}
changeset = cast({%{}, param_types}, params, Map.keys(param_types))
changeset.changes
end
defp order_statement(query, table_binding, :asc) do
order_by(
query,
[{u, table_position(query, table_binding)}],
fragment("? asc nulls last", u.id)
)
end
defp order_statement(query, table_binding, :desc) do
order_by(
query,
[{u, table_position(query, table_binding)}],
fragment("? desc nulls last", u.id)
)
end
defp restrict(query, :min_id, %{min_id: min_id}, table_binding) do
where(query, [{q, table_position(query, table_binding)}], q.id > ^min_id)
end
@ -119,19 +136,16 @@ defmodule Pleroma.Pagination do
defp restrict(%{order_bys: [_ | _]} = query, :order, %{skip_extra_order: true}, _), do: query
defp restrict(query, :order, %{min_id: _}, table_binding) do
order_by(
query,
[{u, table_position(query, table_binding)}],
fragment("? asc nulls last", u.id)
)
order_statement(query, table_binding, :asc)
end
defp restrict(query, :order, _options, table_binding) do
order_by(
query,
[{u, table_position(query, table_binding)}],
fragment("? desc nulls last", u.id)
)
defp restrict(query, :order, %{max_id: _}, table_binding) do
order_statement(query, table_binding, :desc)
end
defp restrict(query, :order, options, table_binding) do
dir = if options[:order_asc], do: :asc, else: :desc
order_statement(query, table_binding, dir)
end
defp restrict(query, :offset, %{offset: offset}, _table_binding) do
@ -151,11 +165,9 @@ defmodule Pleroma.Pagination do
defp restrict(query, _, _, _), do: query
defp enforce_order(result, %{min_id: _}) do
result
|> Enum.reverse()
end
defp enforce_order(result, %{min_id: _, order_asc: true}), do: result
defp enforce_order(result, %{min_id: _}), do: Enum.reverse(result)
defp enforce_order(result, %{max_id: _, order_asc: true}), do: Enum.reverse(result)
defp enforce_order(result, _), do: result
defp table_position(%Ecto.Query{} = query, binding_name) do

42
lib/pleroma/release_tasks.ex
View file

@ -5,7 +5,10 @@
defmodule Pleroma.ReleaseTasks do
@repo Pleroma.Repo
def run(args) do
# TODO: Kept for some backwards compatibility with buggy pleroma_ctl,
# if a mismatch between pleroma_ctl and Pleroma accidentaly happens.
# Remove in the future.
def run(args) when is_binary(args) do
[task | args] = String.split(args)
case task do
@ -16,17 +19,38 @@ defmodule Pleroma.ReleaseTasks do
end
end
# HACK: Script arguments need to be received as a list, otherwise (quoted) arguments with
# whitespace will be broken. Previously the broken string form above was used,
# escaping in the shell does not help.
def run(args) when is_list(args) do
[task | args] = args
case task do
"migrate" -> migrate(args)
"create" -> create()
"rollback" -> rollback(args)
task -> mix_task(task, args)
end
end
def find_module(task) do
module_name =
task
|> String.split(".")
|> Enum.map(&String.capitalize/1)
|> then(fn x -> [Mix, Tasks, Pleroma] ++ x end)
|> Module.concat()
case Code.ensure_loaded(module_name) do
{:module, _} -> module_name
_ -> nil
end
end
defp mix_task(task, args) do
Application.load(:pleroma)
{:ok, modules} = :application.get_key(:pleroma, :modules)
module =
Enum.find(modules, fn module ->
module = Module.split(module)
match?(["Mix", "Tasks", "Pleroma" | _], module) and
String.downcase(List.last(module)) == task
end)
module = find_module(task)
if module do
module.run(args)

252
lib/pleroma/reverse_proxy.ex
View file

@ -3,20 +3,25 @@
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.ReverseProxy do
alias Pleroma.Utils.URIEncoding
@range_headers ~w(range if-range)
@keep_req_headers ~w(accept accept-encoding cache-control if-modified-since) ++
~w(if-unmodified-since if-none-match) ++ @range_headers
@resp_cache_headers ~w(etag date last-modified)
@keep_resp_headers @resp_cache_headers ++
~w(content-type content-disposition content-encoding) ++
~w(content-length content-type content-disposition content-encoding) ++
~w(content-range accept-ranges vary)
@default_cache_control_header "public, max-age=1209600"
@default_cache_control_header "public, max-age=1209600, immutable"
@valid_resp_codes [200, 206, 304]
@max_read_duration :timer.seconds(30)
@max_body_length :infinity
@failed_request_ttl :timer.seconds(60)
@sniff_bytes 8 * 1024
@methods ~w(GET HEAD)
@allowed_mime_types Pleroma.Config.get([Pleroma.Upload, :allowed_mime_types], [])
@cachex Pleroma.Config.get([:cachex, :provider], Cachex)
def max_read_duration_default, do: @max_read_duration
@ -55,7 +60,12 @@ defmodule Pleroma.ReverseProxy do
* `inline_content_types`:
* `true` will not alter `content-disposition` (up to the upstream),
* `false` will add `content-disposition: attachment` to any request,
* a list of whitelisted content types
* a list of whitelisted content types for which `content-disposition: inline`
is always set (overriding any upstream header) so the media can be embedded
in pages; the filename is derived from the content type
* `sniff_content_type` (default `false`): detects an image MIME type from the first
response chunk when the upstream type is missing or `application/octet-stream`.
* `req_headers`, `resp_headers` additional headers.
@ -65,11 +75,15 @@ defmodule Pleroma.ReverseProxy do
@default_options [pool: :media]
@inline_content_types [
"image/apng",
"image/avif",
"image/bmp",
"image/gif",
"image/jpeg",
"image/jpg",
"image/png",
"image/svg+xml",
"image/webp",
"audio/mpeg",
"audio/mp3",
"video/webm",
@ -88,6 +102,7 @@ defmodule Pleroma.ReverseProxy do
| {:req_headers, [{String.t(), String.t()}]}
| {:resp_headers, [{String.t(), String.t()}]}
| {:inline_content_types, boolean() | list(String.t())}
| {:sniff_content_type, boolean()}
| {:redirect_on_failure, boolean()}
@spec call(Plug.Conn.t(), String.t(), list(option())) :: Plug.Conn.t()
@ -106,12 +121,8 @@ defmodule Pleroma.ReverseProxy do
end
with {:ok, nil} <- @cachex.get(:failed_proxy_url_cache, url),
{:ok, code, headers, client} <- request(method, url, req_headers, client_opts),
:ok <-
header_length_constraint(
headers,
Keyword.get(opts, :max_body_length, @max_body_length)
) do
{:ok, code, headers, client} <-
request_with_constraints(method, url, req_headers, client_opts, opts) do
response(conn, client, url, code, headers, opts)
else
{:ok, true} ->
@ -153,9 +164,12 @@ defmodule Pleroma.ReverseProxy do
end
defp request(method, url, headers, opts) do
Logger.debug("#{__MODULE__} #{method} #{url} #{inspect(headers)}")
method = method |> String.downcase() |> String.to_existing_atom()
url = maybe_encode_url(url)
Logger.debug("#{__MODULE__} #{method} #{url} #{inspect(headers)}")
case client().request(method, url, headers, "", opts) do
{:ok, code, headers, client} when code in @valid_resp_codes ->
{:ok, code, downcase_headers(headers), client}
@ -163,7 +177,8 @@ defmodule Pleroma.ReverseProxy do
{:ok, code, headers} when code in @valid_resp_codes ->
{:ok, code, downcase_headers(headers)}
{:ok, code, _, _} ->
{:ok, code, _, client} ->
client().close(client)
{:error, {:invalid_http_response, code}}
{:ok, code, _} ->
@ -174,14 +189,37 @@ defmodule Pleroma.ReverseProxy do
end
end
defp request_with_constraints(method, url, headers, client_opts, opts) do
case request(method, url, headers, client_opts) do
{:ok, _code, headers, client} = response ->
case header_length_constraint(
headers,
Keyword.get(opts, :max_body_length, @max_body_length)
) do
:ok ->
response
error ->
client().close(client)
error
end
response ->
response
end
end
defp response(conn, client, url, status, headers, opts) do
Logger.debug("#{__MODULE__} #{status} #{url} #{inspect(headers)}")
{headers, client, prefetched} = maybe_prefetch_for_content_type(headers, client, opts)
result =
conn
|> put_resp_headers(build_resp_headers(headers, opts))
|> streaming_compat
|> send_chunked(status)
|> chunk_reply(client, opts)
|> chunk_reply(client, opts, prefetched)
case result do
{:ok, conn} ->
@ -201,18 +239,28 @@ defmodule Pleroma.ReverseProxy do
end
end
defp chunk_reply(conn, client, opts) do
defp chunk_reply(conn, client, opts, :none) do
chunk_reply(conn, client, opts, 0, 0)
end
defp chunk_reply(conn, client, opts, sent_so_far, duration) do
with {:ok, duration} <-
check_read_duration(
duration,
Keyword.get(opts, :max_read_duration, @max_read_duration)
defp chunk_reply(conn, _client, _opts, :done), do: {:ok, conn}
defp chunk_reply(conn, _client, _opts, {:error, error}), do: {:error, error, conn}
defp chunk_reply(conn, client, opts, {:ok, data, duration}) do
with :ok <-
body_size_constraint(
byte_size(data),
Keyword.get(opts, :max_body_length, @max_body_length)
),
{:ok, data, client} <- client().stream_body(client),
{:ok, duration} <- increase_read_duration(duration),
{:ok, conn} <- chunk(conn, data) do
chunk_reply(conn, client, opts, byte_size(data), duration)
else
{:error, error} -> {:error, error, conn}
end
end
defp chunk_reply(conn, client, opts, sent_so_far, duration) do
with {:ok, data, client, duration} <- read_chunk(client, duration, opts),
sent_so_far = sent_so_far + byte_size(data),
:ok <-
body_size_constraint(
@ -227,6 +275,66 @@ defmodule Pleroma.ReverseProxy do
end
end
defp read_chunk(client, duration, opts) do
with {:ok, timer} <-
check_read_duration(
duration,
Keyword.get(opts, :max_read_duration, @max_read_duration)
),
{:ok, data, client} <- client().stream_body(client),
{:ok, duration} <- increase_read_duration(timer) do
{:ok, data, client, duration}
end
end
defp maybe_prefetch_for_content_type(headers, client, opts) do
if Keyword.get(opts, :sniff_content_type, false) and generic_content_type?(headers) do
case read_chunk(client, 0, opts) do
{:ok, data, client, duration} ->
{maybe_put_image_content_type(headers, data), client, {:ok, data, duration}}
:done ->
{headers, client, :done}
{:error, error} ->
{headers, client, {:error, error}}
end
else
{headers, client, :none}
end
end
defp generic_content_type?(headers) do
headers
|> get_content_type()
|> String.trim()
|> String.downcase()
|> then(&(&1 in ["", "application/octet-stream"]))
end
defp maybe_put_image_content_type(headers, data) do
with false <- data == "",
{:ok, %{mime_type: "image/" <> _ = content_type}} <-
Majic.perform({:bytes, binary_part(data, 0, min(byte_size(data), @sniff_bytes))},
pool: Pleroma.MajicPool
) do
[
{"content-type", content_type}
| Enum.reject(headers, fn {key, _} -> key == "content-type" end)
]
else
_ -> headers
end
rescue
error ->
Logger.debug("#{__MODULE__}: content-type sniffing failed: #{Exception.message(error)}")
headers
catch
kind, reason ->
Logger.debug("#{__MODULE__}: content-type sniffing failed: #{kind}: #{inspect(reason)}")
headers
end
defp head_response(conn, url, code, headers, opts) do
Logger.debug("#{__MODULE__} #{code} #{url} #{inspect(headers)}")
@ -273,17 +381,34 @@ defmodule Pleroma.ReverseProxy do
|> Enum.filter(fn {k, _} -> k in @keep_req_headers end)
|> build_req_range_or_encoding_header(opts)
|> build_req_user_agent_header(opts)
|> Keyword.merge(Keyword.get(opts, :req_headers, []))
|> merge_headers(Keyword.get(opts, :req_headers, []))
|> maybe_force_identity_encoding(opts)
end
defp merge_headers(headers, extra_headers) do
Enum.reduce(extra_headers, headers, fn {key, value}, headers ->
List.keystore(headers, String.downcase(key), 0, {String.downcase(key), value})
end)
end
defp maybe_force_identity_encoding(headers, opts) do
if Keyword.get(opts, :sniff_content_type, false) do
List.keystore(headers, "accept-encoding", 0, {"accept-encoding", "identity"})
else
headers
end
end
# Disable content-encoding if any @range_headers are requested (see #1823).
defp build_req_range_or_encoding_header(headers, _opts) do
range? = Enum.any?(headers, fn {header, _} -> Enum.member?(@range_headers, header) end)
if range? && List.keymember?(headers, "accept-encoding", 0) do
List.keydelete(headers, "accept-encoding", 0)
else
headers
cond do
range? && List.keymember?(headers, "accept-encoding", 0) ->
List.keydelete(headers, "accept-encoding", 0)
true ->
headers
end
end
@ -300,10 +425,26 @@ defmodule Pleroma.ReverseProxy do
headers
|> Enum.filter(fn {k, _} -> k in @keep_resp_headers end)
|> build_resp_cache_headers(opts)
|> sanitise_content_type()
|> build_resp_content_disposition_header(opts)
|> Keyword.merge(Keyword.get(opts, :resp_headers, []))
end
defp sanitise_content_type(headers) do
original_ct = get_content_type(headers)
safe_ct =
Pleroma.Web.Plugs.Utils.get_safe_mime_type(
%{allowed_mime_types: @allowed_mime_types},
original_ct
)
[
{"content-type", safe_ct}
| Enum.filter(headers, fn {k, _v} -> k != "content-type" end)
]
end
defp build_resp_cache_headers(headers, _opts) do
has_cache? = Enum.any?(headers, fn {k, _} -> k in @resp_cache_headers end)
@ -360,9 +501,33 @@ defmodule Pleroma.ReverseProxy do
disposition = "attachment; filename=\"#{name}\""
List.keystore(headers, "content-disposition", 0, {"content-disposition", disposition})
replace_header(headers, "content-disposition", disposition)
else
headers
if opt == true do
headers
else
name = inline_filename(content_type)
disposition =
if name do
"inline; filename=\"#{name}\""
else
"inline"
end
replace_header(headers, "content-disposition", disposition)
end
end
end
defp replace_header(headers, key, value) do
[{key, value} | Enum.reject(headers, fn {header, _} -> header == key end)]
end
defp inline_filename(content_type) do
case MIME.extensions(content_type) do
[ext | _] when ext != "" -> "inline.#{ext}"
_ -> nil
end
end
@ -382,7 +547,7 @@ defmodule Pleroma.ReverseProxy do
defp header_length_constraint(_, _), do: :ok
defp body_size_constraint(size, limit) when is_integer(limit) and limit > 0 and size >= limit do
defp body_size_constraint(size, limit) when is_integer(limit) and limit > 0 and size > limit do
{:error, :body_too_large}
end
@ -397,7 +562,9 @@ defmodule Pleroma.ReverseProxy do
end
end
defp check_read_duration(_, _), do: {:ok, :no_duration_limit, :no_duration_limit}
defp check_read_duration(_, _), do: {:ok, :no_duration_limit}
defp increase_read_duration(:no_duration_limit), do: {:ok, :no_duration_limit}
defp increase_read_duration({previous_duration, started})
when is_integer(previous_duration) and is_integer(started) do
@ -417,4 +584,31 @@ defmodule Pleroma.ReverseProxy do
@cachex.put(:failed_proxy_url_cache, url, true, ttl: ttl)
end
# When Cowboy handles a chunked response with a content-length header it streams
# over HTTP 1.1 instead of chunking. Bandit cannot stream over HTTP 1.1 so the header
# must be stripped or it breaks RFC compliance for Transfer Encoding: Chunked. RFC9112§6.2
#
# HTTP2 is always streamed for all adapters.
defp streaming_compat(conn) do
with Phoenix.Endpoint.Cowboy2Adapter <- Pleroma.Web.Endpoint.config(:adapter) do
conn
else
_ -> delete_resp_header(conn, "content-length")
end
end
# Only when Tesla adapter is Hackney or Finch does the URL
# need encoding before Reverse Proxying as both end up
# using the raw Hackney client and cannot leverage our
# EncodeUrl Tesla middleware
# Also do it for test environment
defp maybe_encode_url(url) do
case Application.get_env(:tesla, :adapter) do
Tesla.Adapter.Hackney -> URIEncoding.encode_url(url)
{Tesla.Adapter.Finch, _} -> URIEncoding.encode_url(url)
Tesla.Mock -> URIEncoding.encode_url(url)
_ -> url
end
end
end

72
lib/pleroma/reverse_proxy/client/hackney.ex
View file

@ -4,10 +4,52 @@
defmodule Pleroma.ReverseProxy.Client.Hackney do
@behaviour Pleroma.ReverseProxy.Client
@redirect_limit 5
require Logger
# In-app redirect handler to avoid Hackney redirect bugs:
# - https://github.com/benoitc/hackney/issues/527 (relative/protocol-less redirects can crash Hackney)
# - https://github.com/benoitc/hackney/issues/273 (redirects not followed when using HTTP proxy)
#
# Based on a redirect handler from Pleb, slightly modified to work with Hackney:
# https://declin.eu/objects/d4f38e62-5429-4614-86d1-e8fc16e6bf33
@redirect_statuses [301, 302, 303, 307, 308]
defp absolute_redirect_url(original_url, resp_headers) do
location =
Enum.find(resp_headers, fn {header, _location} ->
String.downcase(header) == "location"
end)
URI.merge(original_url, elem(location, 1))
|> URI.to_string()
end
@impl true
def request(method, url, headers, body, opts \\ []) do
:hackney.request(method, url, headers, body, opts)
opts =
Keyword.put_new(opts, :path_encode_fun, fn path ->
path
end)
if opts[:follow_redirect] != false do
{_state, req_opts} = Access.get_and_update(opts, :follow_redirect, fn a -> {a, false} end)
env = %{method: method, headers: headers, body: body, req_opts: req_opts}
res = :hackney.request(method, url, headers, body, req_opts)
case res do
{:ok, code, resp_headers, _client} when code in @redirect_statuses ->
redirect(url, resp_headers, env, @redirect_limit)
{:ok, code, resp_headers} when code in @redirect_statuses ->
redirect(url, resp_headers, env, @redirect_limit)
_ ->
res
end
else
:hackney.request(method, url, headers, body, opts)
end
end
@impl true
@ -21,4 +63,32 @@ defmodule Pleroma.ReverseProxy.Client.Hackney do
@impl true
def close(ref), do: :hackney.close(ref)
defp redirect(url, resp_headers, env, limit) when limit == 0 do
new_url = absolute_redirect_url(url, resp_headers)
Logger.debug(
"#{__MODULE__}: Handling redirect #{url} -> #{new_url}; redirect limit was reached - returning response after final redirect"
)
:hackney.request(env.method, new_url, env.headers, env.body, env.req_opts)
end
defp redirect(url, resp_headers, env, limit) do
new_url = absolute_redirect_url(url, resp_headers)
Logger.debug("#{__MODULE__}: handling redirect #{url} -> #{new_url}; limit = #{limit}")
res = :hackney.request(env.method, new_url, env.headers, env.body, env.req_opts)
case res do
{:ok, code, new_resp_headers, _client} when code in @redirect_statuses ->
redirect(new_url, new_resp_headers, env, limit - 1)
{:ok, code, new_resp_headers} when code in @redirect_statuses ->
redirect(new_url, new_resp_headers, env, limit - 1)
_ ->
res
end
end
end

16
lib/pleroma/reverse_proxy/client/tesla.ex
View file

@ -6,6 +6,7 @@ defmodule Pleroma.ReverseProxy.Client.Tesla do
@behaviour Pleroma.ReverseProxy.Client
alias Pleroma.Gun.ConnectionPool
alias Pleroma.HTTP.AdapterHelper
@type headers() :: [{String.t(), String.t()}]
@type status() :: pos_integer()
@ -20,6 +21,13 @@ defmodule Pleroma.ReverseProxy.Client.Tesla do
def request(method, url, headers, body, opts \\ []) do
check_adapter()
# MediaProxyController inserts MediaProxy proxy configuration into the ReverseProxy call.
# This config can be unformatted and in String representation, while this works for Hackney,
# it does not work for Tesla and the proxy needs to be reformatted.
# Gun AdapterHelper doesn't do it, since it does not overwrite proxy config.
proxy = AdapterHelper.format_proxy(opts[:proxy])
opts = if proxy, do: Keyword.put(opts, :proxy, proxy), else: opts
opts = Keyword.put(opts, :body_as, :chunks)
with {:ok, response} <-
@ -45,8 +53,8 @@ defmodule Pleroma.ReverseProxy.Client.Tesla do
@impl true
@spec stream_body(map()) ::
{:ok, binary(), map()} | {:error, atom() | String.t()} | :done | no_return()
def stream_body(%{pid: pid, fin: true}) do
ConnectionPool.release_conn(pid)
def stream_body(%{pid: pid, stream: stream, fin: true}) do
ConnectionPool.release_stream(pid, stream)
:done
end
@ -70,6 +78,10 @@ defmodule Pleroma.ReverseProxy.Client.Tesla do
@impl true
@spec close(map) :: :ok | no_return()
def close(%{pid: pid, stream: stream}) do
ConnectionPool.cancel_stream(pid, stream)
end
def close(%{pid: pid}) do
ConnectionPool.release_conn(pid)
end

212
lib/pleroma/safe_zip.ex Normal file
View file

@ -0,0 +1,212 @@
# Akkoma: Magically expressive social media
# Copyright © 2024 Akkoma Authors <https://akkoma.dev/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.SafeZip do
@moduledoc """
Wraps the subset of Erlang's zip module we’d like to use
but enforces path-traversal safety everywhere and other checks.
For convenience almost all functions accept both elixir strings and charlists,
but output elixir strings themselves. However, this means the input parameter type
can no longer be used to distinguish archive file paths from archive binary data in memory,
thus where needed both a _data and _file variant are provided.
"""
@type text() :: String.t() | [char()]
defp safe_path?(path) do
# Path accepts elixir’s chardata()
case Path.safe_relative(path) do
{:ok, _} -> true
_ -> false
end
end
defp safe_type?(file_type) do
if file_type in [:regular, :directory] do
true
else
false
end
end
defp maybe_add_file(_type, _path_charlist, nil), do: nil
defp maybe_add_file(:regular, path_charlist, file_list),
do: [to_string(path_charlist) | file_list]
defp maybe_add_file(_type, _path_charlist, file_list), do: file_list
@spec check_safe_archive_and_maybe_list_files(binary() | [char()], [term()], boolean()) ::
{:ok, [String.t()]} | {:error, reason :: term()}
defp check_safe_archive_and_maybe_list_files(archive, opts, list) do
acc = if list, do: [], else: nil
with {:ok, table} <- :zip.table(archive, opts) do
Enum.reduce_while(table, {:ok, acc}, fn
# ZIP comment
{:zip_comment, _}, acc ->
{:cont, acc}
# File entry
{:zip_file, path, info, _comment, _offset, _comp_size}, {:ok, fl} ->
with {_, type} <- {:get_type, elem(info, 2)},
{_, true} <- {:type, safe_type?(type)},
{_, true} <- {:safe_path, safe_path?(path)} do
{:cont, {:ok, maybe_add_file(type, path, fl)}}
else
{:type, _} ->
{:halt, {:error, "Potentially unsafe file type in ZIP at: #{path}"}}
{:safe_path, _} ->
{:halt, {:error, "Unsafe path in ZIP: #{path}"}}
end
# new OTP version?
_, _acc ->
{:halt, {:error, "Unknown ZIP record type"}}
end)
end
end
@spec check_safe_archive_and_list_files(binary() | [char()], [term()]) ::
{:ok, [String.t()]} | {:error, reason :: term()}
defp check_safe_archive_and_list_files(archive, opts \\ []) do
check_safe_archive_and_maybe_list_files(archive, opts, true)
end
@spec check_safe_archive(binary() | [char()], [term()]) :: :ok | {:error, reason :: term()}
defp check_safe_archive(archive, opts \\ []) do
case check_safe_archive_and_maybe_list_files(archive, opts, false) do
{:ok, _} -> :ok
error -> error
end
end
@spec check_safe_file_list([text()], text()) :: :ok | {:error, term()}
defp check_safe_file_list([], _), do: :ok
defp check_safe_file_list([path | tail], cwd) do
with {_, true} <- {:path, safe_path?(path)},
{_, {:ok, fstat}} <- {:stat, File.stat(Path.expand(path, cwd))},
{_, true} <- {:type, safe_type?(fstat.type)} do
check_safe_file_list(tail, cwd)
else
{:path, _} ->
{:error, "Unsafe path escaping cwd: #{path}"}
{:stat, e} ->
{:error, "Unable to check file type of #{path}: #{inspect(e)}"}
{:type, _} ->
{:error, "Unsafe type at #{path}"}
end
end
defp check_safe_file_list(_, _), do: {:error, "Malformed file_list"}
@doc """
Checks whether the archive data contais file entries for all paths from fset
Note this really only accepts entries corresponding to regular _files_,
if a path is contained as for example an directory, this does not count as a match.
"""
@spec contains_all_data?(binary(), MapSet.t()) :: true | false
def contains_all_data?(archive_data, fset) do
with {:ok, table} <- :zip.table(archive_data) do
remaining =
Enum.reduce(table, fset, fn
{:zip_file, path, info, _comment, _offset, _comp_size}, fset ->
if elem(info, 2) == :regular do
MapSet.delete(fset, path)
else
fset
end
_, _ ->
fset
end)
|> MapSet.size()
if remaining == 0, do: true, else: false
else
_ -> false
end
end
@doc """
List all file entries in ZIP, or error if invalid or unsafe.
Note this really only lists regular files, no directories, ZIP comments or other types!
"""
@spec list_dir_file(text()) :: {:ok, [String.t()]} | {:error, reason :: term()}
def list_dir_file(archive) do
path = to_charlist(archive)
check_safe_archive_and_list_files(path)
end
defp stringify_zip({:ok, {fname, data}}), do: {:ok, {to_string(fname), data}}
defp stringify_zip({:ok, fname}), do: {:ok, to_string(fname)}
defp stringify_zip(ret), do: ret
@spec zip(text(), text(), [text()], boolean()) ::
{:ok, file_name :: String.t()}
| {:ok, {file_name :: String.t(), file_data :: binary()}}
| {:error, reason :: term()}
def zip(name, file_list, cwd, memory \\ false) do
opts = [{:cwd, to_charlist(cwd)}]
opts = if memory, do: [:memory | opts], else: opts
with :ok <- check_safe_file_list(file_list, cwd) do
file_list = for f <- file_list, do: to_charlist(f)
name = to_charlist(name)
stringify_zip(:zip.zip(name, file_list, opts))
end
end
@spec unzip_file(text(), text(), [text()] | nil) ::
{:ok, [String.t()]}
| {:error, reason :: term()}
| {:error, {name :: text(), reason :: term()}}
def unzip_file(archive, target_dir, file_list \\ nil) do
do_unzip(to_charlist(archive), to_charlist(target_dir), file_list)
end
@spec unzip_data(binary(), text(), [text()] | nil) ::
{:ok, [String.t()]}
| {:error, reason :: term()}
| {:error, {name :: text(), reason :: term()}}
def unzip_data(archive, target_dir, file_list \\ nil) do
do_unzip(archive, to_charlist(target_dir), file_list)
end
defp stringify_unzip({:ok, [{_fname, _data} | _] = filebinlist}),
do: {:ok, Enum.map(filebinlist, fn {fname, data} -> {to_string(fname), data} end)}
defp stringify_unzip({:ok, [_fname | _] = filelist}),
do: {:ok, Enum.map(filelist, fn fname -> to_string(fname) end)}
defp stringify_unzip({:error, {fname, term}}), do: {:error, {to_string(fname), term}}
defp stringify_unzip(ret), do: ret
@spec do_unzip(binary() | [char()], text(), [text()] | nil) ::
{:ok, [String.t()]}
| {:error, reason :: term()}
| {:error, {name :: text(), reason :: term()}}
defp do_unzip(archive, target_dir, file_list) do
opts =
if file_list != nil do
[
file_list: for(f <- file_list, do: to_charlist(f)),
cwd: target_dir
]
else
[cwd: target_dir]
end
with :ok <- check_safe_archive(archive) do
stringify_unzip(:zip.unzip(archive, opts))
end
end
end

67
lib/pleroma/search.ex
View file

@ -1,12 +1,31 @@
defmodule Pleroma.Search do
alias Pleroma.Activity
alias Pleroma.Object
alias Pleroma.Web.ActivityPub.Visibility
alias Pleroma.Workers.SearchIndexingWorker
def add_to_index(%Pleroma.Activity{id: activity_id}) do
SearchIndexingWorker.enqueue("add_to_index", %{"activity" => activity_id})
@spec add_to_index(Activity.t()) :: {:ok, Oban.Job.t() | :noop} | {:error, Oban.Job.changeset()}
def add_to_index(%Activity{id: activity_id, object: %Object{} = object} = activity) do
with {_, true} <- {:indexable, indexable?(activity)},
{_, "public"} <- {:visibility, Visibility.get_visibility(object)} do
SearchIndexingWorker.new(%{"op" => "add_to_index", "activity" => activity_id})
|> Oban.insert()
else
_ -> {:ok, :noop}
end
end
def add_to_index(%Activity{id: activity_id}) do
case Activity.get_by_id_with_object(activity_id) do
%Activity{} = preloaded -> add_to_index(preloaded)
_ -> {:ok, :noop}
end
end
@spec remove_from_index(Object.t()) :: {:ok, Oban.Job.t()} | {:error, Oban.Job.changeset()}
def remove_from_index(%Pleroma.Object{id: object_id}) do
SearchIndexingWorker.enqueue("remove_from_index", %{"object" => object_id})
SearchIndexingWorker.new(%{"op" => "remove_from_index", "object" => object_id})
|> Oban.insert()
end
def search(query, options) do
@ -16,6 +35,46 @@ defmodule Pleroma.Search do
def healthcheck_endpoints do
search_module = Pleroma.Config.get([Pleroma.Search, :module])
search_module.healthcheck_endpoints
search_module.healthcheck_endpoints()
end
def object_to_search_data(%Object{} = object) do
data = object.data
content_str =
case data["content"] do
[nil | rest] -> to_string(rest)
str -> str
end
content =
with {:ok, scrubbed} <-
FastSanitize.Sanitizer.scrub(content_str, Pleroma.HTML.Scrubber.SearchIndexing),
trimmed <- String.trim(scrubbed) do
trimmed
end
# Make sure we have a non-empty string
if content != "" do
{:ok, published, _} = DateTime.from_iso8601(data["published"])
%{
id: object.id,
content: content,
ap: data["id"],
published: published |> DateTime.to_unix()
}
end
end
defp indexable?(%Activity{
data: %{"type" => "Create"},
object: %Object{
data: %{"content" => content, "published" => published, "type" => "Note"}
}
})
when not is_nil(content) and content not in ["", "."] and not is_nil(published),
do: true
defp indexable?(_), do: false
end

3
lib/pleroma/search/database_search.ex
View file

@ -102,7 +102,8 @@ defmodule Pleroma.Search.DatabaseSearch do
^tsc,
o.data,
^search_query
)
),
order_by: [desc: :inserted_at]
)
end

71
lib/pleroma/search/meilisearch.ex
View file

@ -4,6 +4,8 @@ defmodule Pleroma.Search.Meilisearch do
alias Pleroma.Activity
alias Pleroma.Config.Getting, as: Config
alias Pleroma.Object
alias Pleroma.Search
import Pleroma.Search.DatabaseSearch
import Ecto.Query
@ -118,65 +120,24 @@ defmodule Pleroma.Search.Meilisearch do
end
end
def object_to_search_data(object) do
# Only index public or unlisted Notes
if not is_nil(object) and object.data["type"] == "Note" and
not is_nil(object.data["content"]) and
(Pleroma.Constants.as_public() in object.data["to"] or
Pleroma.Constants.as_public() in object.data["cc"]) and
object.data["content"] not in ["", "."] do
data = object.data
content_str =
case data["content"] do
[nil | rest] -> to_string(rest)
str -> str
end
content =
with {:ok, scrubbed} <-
FastSanitize.Sanitizer.scrub(content_str, Pleroma.HTML.Scrubber.SearchIndexing),
trimmed <- String.trim(scrubbed) do
trimmed
end
# Make sure we have a non-empty string
if content != "" do
{:ok, published, _} = DateTime.from_iso8601(data["published"])
%{
id: object.id,
content: content,
ap: data["id"],
published: published |> DateTime.to_unix()
}
end
end
end
@impl true
def add_to_index(activity) do
maybe_search_data = object_to_search_data(activity.object)
def add_to_index(%Activity{object: %Object{} = object} = activity) do
search_data = Search.object_to_search_data(object)
if activity.data["type"] == "Create" and maybe_search_data do
result =
meili_put(
"/indexes/objects/documents",
[maybe_search_data]
)
result =
meili_put(
"/indexes/objects/documents",
[search_data]
)
with {:ok, %{"status" => "enqueued"}} <- result do
# Added successfully
:ok
else
_ ->
# There was an error, report it
Logger.error("Failed to add activity #{activity.id} to index: #{inspect(result)}")
{:error, result}
end
else
# The post isn't something we can search, that's ok
with {:ok, %{"status" => "enqueued"}} <- result do
# Added successfully
:ok
else
_ ->
# There was an error, report it
Logger.error("Failed to add activity #{activity.id} to index: #{inspect(result)}")
{:error, result}
end
end

83
lib/pleroma/search/qdrant_search.ex
View file

@ -4,11 +4,12 @@ defmodule Pleroma.Search.QdrantSearch do
alias Pleroma.Activity
alias Pleroma.Config.Getting, as: Config
alias Pleroma.Object
alias Pleroma.Search
alias __MODULE__.OpenAIClient
alias __MODULE__.QdrantClient
import Pleroma.Search.Meilisearch, only: [object_to_search_data: 1]
import Pleroma.Search.DatabaseSearch, only: [maybe_fetch: 3]
@impl true
@ -82,23 +83,18 @@ defmodule Pleroma.Search.QdrantSearch do
end
@impl true
def add_to_index(activity) do
# This will only index public or unlisted notes
maybe_search_data = object_to_search_data(activity.object)
def add_to_index(%Activity{object: %Object{} = object} = activity) do
search_data = Search.object_to_search_data(object)
if activity.data["type"] == "Create" and maybe_search_data do
with {:ok, embedding} <- get_embedding(maybe_search_data.content),
{:ok, %{status: 200}} <-
QdrantClient.put(
"/collections/posts/points",
build_index_payload(activity, embedding)
) do
:ok
else
e -> {:error, e}
end
else
with {:ok, embedding} <- get_embedding(search_data.content),
{:ok, %{status: 200}} <-
QdrantClient.put(
"/collections/posts/points",
build_index_payload(activity, embedding)
) do
:ok
else
e -> {:error, e}
end
end
@ -157,26 +153,55 @@ defmodule Pleroma.Search.QdrantSearch do
end
defmodule Pleroma.Search.QdrantSearch.OpenAIClient do
use Tesla
alias Pleroma.Config.Getting, as: Config
plug(Tesla.Middleware.BaseUrl, Config.get([Pleroma.Search.QdrantSearch, :openai_url]))
plug(Tesla.Middleware.JSON)
def post(path, body) do
Tesla.post(client(), path, body)
end
plug(Tesla.Middleware.Headers, [
{"Authorization",
"Bearer #{Pleroma.Config.get([Pleroma.Search.QdrantSearch, :openai_api_key])}"}
])
defp client do
Tesla.client(middleware())
end
defp middleware do
[
{Tesla.Middleware.BaseUrl, Config.get([Pleroma.Search.QdrantSearch, :openai_url])},
Tesla.Middleware.JSON,
{Tesla.Middleware.Headers,
[
{"Authorization", "Bearer #{Config.get([Pleroma.Search.QdrantSearch, :openai_api_key])}"}
]}
]
end
end
defmodule Pleroma.Search.QdrantSearch.QdrantClient do
use Tesla
alias Pleroma.Config.Getting, as: Config
plug(Tesla.Middleware.BaseUrl, Config.get([Pleroma.Search.QdrantSearch, :qdrant_url]))
plug(Tesla.Middleware.JSON)
def delete(path) do
Tesla.delete(client(), path)
end
plug(Tesla.Middleware.Headers, [
{"api-key", Pleroma.Config.get([Pleroma.Search.QdrantSearch, :qdrant_api_key])}
])
def post(path, body) do
Tesla.post(client(), path, body)
end
def put(path, body) do
Tesla.put(client(), path, body)
end
defp client do
Tesla.client(middleware())
end
defp middleware do
[
{Tesla.Middleware.BaseUrl, Config.get([Pleroma.Search.QdrantSearch, :qdrant_url])},
Tesla.Middleware.JSON,
{Tesla.Middleware.Headers,
[
{"api-key", Pleroma.Config.get([Pleroma.Search.QdrantSearch, :qdrant_api_key])}
]}
]
end
end

55
lib/pleroma/signature.ex
View file

@ -3,6 +3,7 @@
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Signature do
@behaviour Pleroma.Signature.API
@behaviour HTTPSignatures.Adapter
alias Pleroma.EctoType.ActivityPub.ObjectValidators
@ -10,6 +11,14 @@ defmodule Pleroma.Signature do
alias Pleroma.User
alias Pleroma.Web.ActivityPub.ActivityPub
import Plug.Conn, only: [put_req_header: 3]
@http_signatures_impl Application.compile_env(
:pleroma,
[__MODULE__, :http_signatures_impl],
HTTPSignatures
)
@known_suffixes ["/publickey", "/main-key"]
def key_id_to_actor_id(key_id) do
@ -45,7 +54,7 @@ defmodule Pleroma.Signature do
def fetch_public_key(conn) do
with {:ok, actor_id} <- get_actor_id(conn),
{:ok, public_key} <- User.get_public_key_for_ap_id(actor_id) do
{:ok, public_key} <- User.get_or_fetch_public_key_for_ap_id(actor_id) do
{:ok, public_key}
else
e ->
@ -85,4 +94,48 @@ defmodule Pleroma.Signature do
def signed_date(%NaiveDateTime{} = date) do
Timex.format!(date, "{WDshort}, {0D} {Mshort} {YYYY} {h24}:{m}:{s} GMT")
end
@spec validate_signature(Plug.Conn.t(), String.t()) :: boolean()
def validate_signature(%Plug.Conn{} = conn, request_target) do
# Newer drafts for HTTP signatures now use @request-target instead of the
# old (request-target). We'll now support both for incoming signatures.
conn =
conn
|> put_req_header("(request-target)", request_target)
|> put_req_header("@request-target", request_target)
@http_signatures_impl.validate_conn(conn) == true
end
@spec validate_signature(Plug.Conn.t()) :: boolean()
def validate_signature(%Plug.Conn{} = conn) do
# This (request-target) is non-standard, but many implementations do it
# this way due to a misinterpretation of
# https://datatracker.ietf.org/doc/html/draft-cavage-http-signatures-06
# "path" was interpreted as not having the query, though later examples
# show that it must be the absolute path + query. This behavior is kept to
# make sure most software (Pleroma itself, Mastodon, and probably others)
# do not break.
request_target = Enum.join([String.downcase(conn.method), conn.request_path], " ")
# This is the proper way to build the @request-target, as expected by
# many HTTP signature libraries, clarified in the following draft:
# https://www.ietf.org/archive/id/draft-ietf-httpbis-message-signatures-11.html#section-2.2.6
# It is the same as before, but containing the query part as well.
proper_target = Enum.join([request_target, "?", conn.query_string], "")
cond do
# Normal, non-standard behavior but expected by Pleroma and more.
validate_signature(conn, request_target) ->
true
# Has query string and the previous one failed: let's try the standard.
conn.query_string != "" ->
validate_signature(conn, proper_target)
# If there's no query string and signature fails, it's rotten.
true ->
false
end
end
end

14
lib/pleroma/signature/api.ex Normal file
View file

@ -0,0 +1,14 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Signature.API do
@moduledoc """
Behaviour for signing requests and producing HTTP Date headers.
This is used to allow tests to replace the signing implementation with Mox.
"""
@callback sign(user :: Pleroma.User.t(), headers :: map()) :: String.t()
@callback signed_date() :: String.t()
end

6
lib/pleroma/telemetry/logger.ex
View file

@ -39,7 +39,7 @@ defmodule Pleroma.Telemetry.Logger do
_,
_
) do
Logger.error(fn ->
Logger.debug(fn ->
"Connection pool failed to reclaim any connections due to all of them being in use. It will have to drop requests for opening connections to new hosts"
end)
end
@ -55,8 +55,8 @@ defmodule Pleroma.Telemetry.Logger do
def handle_event(
[:pleroma, :connection_pool, :provision_failure],
%{opts: [key | _]},
_,
%{key: key},
_
) do
Logger.debug(fn ->
@ -70,7 +70,7 @@ defmodule Pleroma.Telemetry.Logger do
%{key: key},
_
) do
Logger.warning(fn ->
Logger.debug(fn ->
"Pool worker for #{key}: Client #{inspect(client_pid)} died before releasing the connection with #{inspect(reason)}"
end)
end

60
lib/pleroma/tesla/middleware/connection_pool.ex
View file

@ -12,25 +12,38 @@ defmodule Pleroma.Tesla.Middleware.ConnectionPool do
alias Pleroma.Gun.ConnectionPool
@impl Tesla.Middleware
def call(%Tesla.Env{url: url, opts: opts} = env, next, _) do
def call(%Tesla.Env{opts: opts} = env, next, middleware_opts) do
if opts[:adapter][:body_as] == :stream do
{:error, :pooled_streaming_requires_chunks}
else
do_call(env, next, middleware_opts)
end
end
defp do_call(%Tesla.Env{url: url, opts: opts} = env, next, _) do
uri = URI.parse(url)
# Avoid leaking connections when the middleware is called twice
# with body_as: :chunks. We assume only the middleware can set
# opts[:adapter][:conn]
if opts[:adapter][:conn] do
ConnectionPool.release_conn(opts[:adapter][:conn])
end
cleanup_previous_request(env)
case ConnectionPool.get_conn(uri, opts[:adapter]) do
{:ok, conn_pid} ->
adapter_opts = Keyword.merge(opts[:adapter], conn: conn_pid, close_conn: false)
tunnel = ConnectionPool.tunnel_ref(conn_pid)
adapter_opts =
opts[:adapter]
|> Keyword.drop([:conn, :stream, :tunnel])
|> Keyword.merge(conn: conn_pid, close_conn: false, tunnel: tunnel)
opts = Keyword.put(opts, :adapter, adapter_opts)
env = %{env | opts: opts}
next = use_pool_adapter(next)
case Tesla.run(env, next) do
{:ok, env} ->
unless opts[:adapter][:body_as] == :chunks do
unless opts[:adapter][:body_as] == :chunks and chunk_client?(env.body) do
ConnectionPool.release_conn(conn_pid)
{_, res} = pop_in(env.opts[:adapter][:conn])
{:ok, res}
@ -38,8 +51,11 @@ defmodule Pleroma.Tesla.Middleware.ConnectionPool do
{:ok, env}
end
{:error, {:gun_stream_error, reason}} ->
{:error, reason}
err ->
ConnectionPool.release_conn(conn_pid)
ConnectionPool.discard_conn(conn_pid)
err
end
@ -47,4 +63,34 @@ defmodule Pleroma.Tesla.Middleware.ConnectionPool do
err
end
end
defp cleanup_previous_request(%Tesla.Env{
body: %{pid: pid, stream: stream},
opts: opts
}) do
if opts[:adapter][:conn], do: ConnectionPool.cancel_stream(pid, stream)
end
defp cleanup_previous_request(%Tesla.Env{opts: opts}) do
conn = opts[:adapter][:conn]
stream = opts[:adapter][:stream]
cond do
conn && stream -> ConnectionPool.cancel_stream(conn, stream)
conn -> ConnectionPool.release_conn(conn)
true -> :ok
end
end
defp chunk_client?(%{pid: pid, stream: stream}) when is_pid(pid) and not is_nil(stream),
do: true
defp chunk_client?(_body), do: false
defp use_pool_adapter(next) do
List.update_at(next, -1, fn
{Tesla.Adapter.Gun, function, args} -> {Pleroma.HTTP.Adapter.Gun, function, args}
adapter -> adapter
end)
end
end

29
lib/pleroma/tesla/middleware/encode_url.ex Normal file
View file

@ -0,0 +1,29 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2025 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Tesla.Middleware.EncodeUrl do
@moduledoc """
Middleware to encode URLs properly
We must decode and then re-encode to ensure correct encoding.
If we only encode it will re-encode each % as %25 causing a space
already encoded as %20 to be %2520.
Similar problem for query parameters which need spaces to be the + character
"""
@behaviour Tesla.Middleware
@impl Tesla.Middleware
def call(%Tesla.Env{url: url} = env, next, _) do
url = Pleroma.Utils.URIEncoding.encode_url(url)
env = %{env | url: url}
case Tesla.run(env, next) do
{:ok, env} -> {:ok, env}
err -> err
end
end
end

29
lib/pleroma/upload.ex
View file

@ -35,6 +35,7 @@ defmodule Pleroma.Upload do
"""
alias Ecto.UUID
alias Pleroma.Maps
alias Pleroma.Utils.URIEncoding
alias Pleroma.Web.ActivityPub.Utils
require Logger
@ -92,7 +93,7 @@ defmodule Pleroma.Upload do
def store(upload, opts \\ []) do
opts = get_opts(opts)
with {:ok, upload} <- prepare_upload(upload, opts),
with {:ok, %__MODULE__{} = upload} <- prepare_upload(upload, opts),
upload = %__MODULE__{upload | path: upload.path || "#{upload.id}/#{upload.name}"},
{:ok, upload} <- Pleroma.Upload.Filter.filter(opts.filters, upload),
description = get_description(upload),
@ -230,11 +231,18 @@ defmodule Pleroma.Upload do
tmp_path
end
# Encoding the whole path here is fine since the path is in a
# UUID/<file name> form.
# The file at this point isn't %-encoded, so the path shouldn't
# be decoded first like Pleroma.Utils.URIEncoding.encode_url/1 does.
defp url_from_spec(%__MODULE__{name: name}, base_url, {:file, path}) do
encode_opts = [bypass_decode: true, bypass_parse: true]
path =
URI.encode(path, &char_unescaped?/1) <>
URIEncoding.encode_url(path, encode_opts) <>
if Pleroma.Config.get([__MODULE__, :link_name], false) do
"?name=#{URI.encode(name, &char_unescaped?/1)}"
enum = %{name: name}
"?#{URI.encode_query(enum)}"
else
""
end
@ -249,14 +257,16 @@ defmodule Pleroma.Upload do
defp url_from_spec(_upload, _base_url, {:url, url}), do: url
@spec base_url() :: binary
def base_url do
uploader = @config_impl.get([Pleroma.Upload, :uploader])
upload_base_url = @config_impl.get([Pleroma.Upload, :base_url])
upload_fallback_url = Pleroma.Web.Endpoint.url() <> "/media/"
upload_base_url = @config_impl.get([Pleroma.Upload, :base_url]) || upload_fallback_url
public_endpoint = @config_impl.get([uploader, :public_endpoint])
case uploader do
Pleroma.Uploaders.Local ->
upload_base_url || Pleroma.Web.Endpoint.url() <> "/media/"
upload_base_url
Pleroma.Uploaders.S3 ->
bucket = @config_impl.get([Pleroma.Uploaders.S3, :bucket])
@ -268,11 +278,14 @@ defmodule Pleroma.Upload do
!is_nil(truncated_namespace) ->
truncated_namespace
!is_nil(namespace) ->
!is_nil(namespace) and !is_nil(bucket) ->
namespace <> ":" <> bucket
true ->
!is_nil(bucket) ->
bucket
true ->
""
end
if public_endpoint do
@ -285,7 +298,7 @@ defmodule Pleroma.Upload do
@config_impl.get([Pleroma.Uploaders.IPFS, :get_gateway_url])
_ ->
public_endpoint || upload_base_url || Pleroma.Web.Endpoint.url() <> "/media/"
public_endpoint || upload_base_url
end
end
end

10
lib/pleroma/upload/filter/analyze_metadata.ex
View file

@ -90,9 +90,13 @@ defmodule Pleroma.Upload.Filter.AnalyzeMetadata do
{:ok, rgb} =
if Image.has_alpha?(resized_image) do
# remove alpha channel
resized_image
|> Operation.extract_band!(0, n: 3)
|> Image.write_to_binary()
case Operation.extract_band(resized_image, 0, n: 3) do
{:ok, data} ->
Image.write_to_binary(data)
_ ->
Image.write_to_binary(resized_image)
end
else
Image.write_to_binary(resized_image)
end

4
lib/pleroma/upload/filter/anonymize_filename.ex
View file

@ -10,7 +10,7 @@ defmodule Pleroma.Upload.Filter.AnonymizeFilename do
"""
@behaviour Pleroma.Upload.Filter
alias Pleroma.Config
@config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config)
alias Pleroma.Upload
def filter(%Upload{name: name} = upload) do
@ -23,7 +23,7 @@ defmodule Pleroma.Upload.Filter.AnonymizeFilename do
@spec predefined_name(String.t()) :: String.t() | nil
defp predefined_name(extension) do
with name when not is_nil(name) <- Config.get([__MODULE__, :text]),
with name when not is_nil(name) <- @config_impl.get([__MODULE__, :text]),
do: String.replace(name, "{extension}", extension)
end

Some files were not shown because too many files have changed in this diff Show more