hj/pleroma
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge branch 'develop' into gitlab-mr-iid-4315

Browse source
This commit is contained in:
Lain Soykaf 2026-07-27 11:34:31 +04:00
commit b4c624911c
No known key found for this signature in database
1163 changed files with 40459 additions and 6082 deletions
Download patch file Download diff file Expand all files Collapse all files

19
lib/mix/pleroma.ex
View file

@ -26,7 +26,11 @@ defmodule Mix.Pleroma do
Application.put_env(:phoenix, :serve_endpoints, false, persistent: true)
unless System.get_env("DEBUG") do
Logger.remove_backend(:console)
try do
Logger.remove_backend(:console)
catch
:exit, _ -> :ok
end
end
adapter = Application.get_env(:tesla, :adapter)
@ -58,7 +62,8 @@ defmodule Mix.Pleroma do
{Majic.Pool,
[name: Pleroma.MajicPool, pool_size: Pleroma.Config.get([:majic_pool, :size], 2)]}
] ++
http_children(adapter)
http_children(adapter) ++
search_children()
cachex_children = Enum.map(@cachex_children, &Pleroma.Application.build_cachex(&1, []))
@ -137,4 +142,14 @@ defmodule Mix.Pleroma do
end
defp http_children(_), do: []
defp search_children do
case Pleroma.Config.get([Pleroma.Search, :module]) do
Pleroma.Search.ParadeDB ->
[Pleroma.Search.ParadeDB.Repo]
_ ->
[]
end
end
end

121
lib/mix/tasks/pleroma/config.ex
View file

@ -234,6 +234,64 @@ defmodule Mix.Tasks.Pleroma.Config do
end)
end
# Removes non-whitelisted configuration sections
def run(["filter_whitelisted" | rest]) do
{options, [], []} =
OptionParser.parse(
rest,
strict: [force: :boolean],
aliases: [f: :force]
)
force = Keyword.get(options, :force, false)
start_pleroma()
whitelisted_configs = Pleroma.Config.get(:database_config_whitelist)
if whitelisted_configs in [nil, false] do
shell_error("No unwanted settings in ConfigDB. No changes made.")
else
{whitelisted_groups, whitelisted_keys} = get_filters(whitelisted_configs)
filtered =
from(c in ConfigDB)
|> Repo.all()
|> Enum.filter(&not_whitelisted?(&1, whitelisted_groups, whitelisted_keys))
do_config_filter(filtered, force)
end
end
# Removes blacklisted configuration sections
def run(["filter_blacklisted" | rest]) do
{options, [], []} =
OptionParser.parse(
rest,
strict: [force: :boolean],
aliases: [f: :force]
)
force = Keyword.get(options, :force, false)
start_pleroma()
blacklisted_configs = Pleroma.Config.get(:database_config_blacklist)
if blacklisted_configs in [nil, false] do
shell_error("No unwanted settings in ConfigDB. No changes made.")
else
{blacklisted_groups, blacklisted_keys} = get_filters(blacklisted_configs)
filtered =
from(c in ConfigDB)
|> Repo.all()
|> Enum.filter(&blacklisted?(&1, blacklisted_groups, blacklisted_keys))
do_config_filter(filtered, force)
end
end
@spec migrate_to_db(Path.t() | nil) :: any()
def migrate_to_db(file_path \\ nil) do
with :ok <- Pleroma.Config.DeprecationWarnings.warn() do
@ -277,9 +335,12 @@ defmodule Mix.Tasks.Pleroma.Config do
group
|> Pleroma.Config.Loader.filter_group(settings)
|> Enum.each(fn {key, value} ->
{:ok, _} = ConfigDB.update_or_create(%{group: group, key: key, value: value})
shell_info("Settings for key #{key} migrated.")
if ConfigDB.static?(group, key) do
shell_info("Settings for key #{key} must remain in static configuration; skipping.")
else
{:ok, _} = ConfigDB.update_or_create(%{group: group, key: key, value: value})
shell_info("Settings for key #{key} migrated.")
end
end)
shell_info("Settings for group #{inspect(group)} migrated.")
@ -330,7 +391,13 @@ defmodule Mix.Tasks.Pleroma.Config do
|> Enum.each(&write_and_delete(&1, file, opts[:delete]))
:ok = File.close(file)
System.cmd("mix", ["format", path])
# Ensure `mix format` runs in the same env as the current task and doesn't
# emit config-time stderr noise (e.g. dev secret warnings) into `mix test`.
System.cmd("mix", ["format", path],
env: [{"MIX_ENV", to_string(Mix.env())}],
stderr_to_stdout: true
)
end
defp config_header, do: "import Config\r\n\r\n"
@ -428,4 +495,50 @@ defmodule Mix.Tasks.Pleroma.Config do
Ecto.Adapters.SQL.query!(Repo, "TRUNCATE config;")
Ecto.Adapters.SQL.query!(Repo, "ALTER SEQUENCE config_id_seq RESTART;")
end
defp get_filters(config) do
groups =
config
|> Enum.filter(fn
{_group} -> true
_ -> false
end)
|> Enum.map(fn {group} -> group end)
keys =
config
|> Enum.filter(fn
{_group, _key} -> true
_ -> false
end)
{groups, keys}
end
defp not_whitelisted?(%{group: group, key: key}, whitelisted_groups, whitelisted_keys) do
not Enum.member?(whitelisted_groups, group) and
not Enum.member?(whitelisted_keys, {group, key})
end
defp blacklisted?(%{group: group, key: key}, blacklisted_groups, blacklisted_keys) do
Enum.member?(blacklisted_groups, group) or
Enum.member?(blacklisted_keys, {group, key})
end
defp do_config_filter(filtered, _) when length(filtered) == 0 do
shell_error("No unwanted settings in ConfigDB. No changes made.")
end
defp do_config_filter(filtered, force) do
shell_info("The following settings will be removed from ConfigDB:\n")
Enum.each(filtered, &dump(&1))
if force or shell_prompt("Are you sure you want to continue?", "n") in ~w(Yn Y y) do
filtered_ids = Enum.map(filtered, fn %{id: id} -> id end)
Repo.delete_all(from(c in ConfigDB, where: c.id in ^filtered_ids))
else
shell_error("No changes made.")
end
end
end

7
lib/mix/tasks/pleroma/database.ex
View file

@ -226,7 +226,12 @@ defmodule Mix.Tasks.Pleroma.Database do
DELETE FROM hashtags AS ht
WHERE NOT EXISTS (
SELECT 1 FROM hashtags_objects hto
WHERE ht.id = hto.hashtag_id)
WHERE ht.id = hto.hashtag_id
)
AND NOT EXISTS (
SELECT 1 FROM user_follows_hashtag ufh
WHERE ht.id = ufh.hashtag_id
)
"""
|> Repo.query()

15
lib/mix/tasks/pleroma/emoji.ex
View file

@ -93,6 +93,7 @@ defmodule Mix.Tasks.Pleroma.Emoji do
)
files = fetch_and_decode!(files_loc)
files_to_unzip = for({_, f} <- files, do: f)
IO.puts(IO.ANSI.format(["Unpacking ", :bright, pack_name]))
@ -103,17 +104,7 @@ defmodule Mix.Tasks.Pleroma.Emoji do
pack_name
])
files_to_unzip =
Enum.map(
files,
fn {_, f} -> to_charlist(f) end
)
{:ok, _} =
:zip.unzip(binary_archive,
cwd: String.to_charlist(pack_path),
file_list: files_to_unzip
)
{:ok, _} = Pleroma.SafeZip.unzip_data(binary_archive, pack_path, files_to_unzip)
IO.puts(IO.ANSI.format(["Writing pack.json for ", :bright, pack_name]))
@ -201,7 +192,7 @@ defmodule Mix.Tasks.Pleroma.Emoji do
tmp_pack_dir = Path.join(System.tmp_dir!(), "emoji-pack-#{name}")
{:ok, _} = :zip.unzip(binary_archive, cwd: String.to_charlist(tmp_pack_dir))
{:ok, _} = Pleroma.SafeZip.unzip_data(binary_archive, tmp_pack_dir)
emoji_map = Pleroma.Emoji.Loader.make_shortcode_to_file_map(tmp_pack_dir, exts)

2
lib/mix/tasks/pleroma/instance.ex
View file

@ -271,7 +271,7 @@ defmodule Mix.Tasks.Pleroma.Instance do
[config_dir, psql_dir, static_dir, uploads_dir]
|> Enum.reject(&File.exists?/1)
|> Enum.each(fn dir ->
File.mkdir_p!(dir)
Pleroma.Backports.mkdir_p!(dir)
File.chmod!(dir, 0o700)
end)

2
lib/mix/tasks/pleroma/openapi_spec.ex
View file

@ -22,7 +22,7 @@ defmodule Mix.Tasks.Pleroma.OpenapiSpec do
else
{_, errors} ->
IO.puts(IO.ANSI.format([:red, :bright, "Spec check failed, errors:"]))
Enum.map(errors, &IO.puts/1)
Enum.each(errors, &IO.puts/1)
raise "Spec check failed"
end

2
lib/mix/tasks/pleroma/robots_txt.ex
View file

@ -22,7 +22,7 @@ defmodule Mix.Tasks.Pleroma.RobotsTxt do
static_dir = Pleroma.Config.get([:instance, :static_dir], "instance/static/")
if !File.exists?(static_dir) do
File.mkdir_p!(static_dir)
Pleroma.Backports.mkdir_p!(static_dir)
end
robots_txt_path = Path.join(static_dir, "robots.txt")

58
lib/mix/tasks/pleroma/search/indexer.ex
View file

@ -4,9 +4,6 @@
defmodule Mix.Tasks.Pleroma.Search.Indexer do
import Mix.Pleroma
import Ecto.Query
alias Pleroma.Workers.SearchIndexingWorker
def run(["create_index"]) do
start_pleroma()
@ -35,49 +32,30 @@ defmodule Mix.Tasks.Pleroma.Search.Indexer do
strict: [
chunk: :integer,
limit: :integer,
step: :integer
step: :integer,
before: :string
]
)
start_pleroma()
chunk_size = Keyword.get(options, :chunk, 100)
limit = Keyword.get(options, :limit, 100_000)
per_step = Keyword.get(options, :step, 1000)
result =
Pleroma.Search.Backfill.run(
options ++
[
on_page: fn page ->
IO.puts(
"Queued #{page.count} activities (#{page.enqueued} total), " <>
"checkpoint: #{page.next_cursor}"
)
end
]
)
chunks = max(div(limit, per_step), 1)
IO.puts("Queued #{result.enqueued} activities")
1..chunks
|> Enum.each(fn step ->
q =
from(a in Pleroma.Activity,
limit: ^per_step,
offset: ^per_step * (^step - 1),
select: [:id],
order_by: [desc: :id]
)
{:ok, ids} =
Pleroma.Repo.transaction(fn ->
Pleroma.Repo.stream(q, timeout: :infinity)
|> Enum.map(fn a ->
a.id
end)
end)
IO.puts("Got #{length(ids)} activities, adding to indexer")
ids
|> Enum.chunk_every(chunk_size)
|> Enum.each(fn chunk ->
IO.puts("Adding #{length(chunk)} activities to indexing queue")
chunk
|> Enum.map(fn id ->
SearchIndexingWorker.new(%{"op" => "add_to_index", "activity" => id})
end)
|> Oban.insert_all()
end)
end)
unless result.exhausted do
IO.puts("Resume with --before #{result.next_cursor}")
end
end
end

2
lib/mix/tasks/pleroma/search/meilisearch.ex
View file

@ -72,7 +72,7 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do
query,
timeout: :infinity
)
|> Stream.map(&Pleroma.Search.Meilisearch.object_to_search_data/1)
|> Stream.map(&Pleroma.Search.object_to_search_data/1)
|> Stream.filter(fn o -> not is_nil(o) end)
|> Stream.chunk_every(chunk_size)
|> Stream.transform(0, fn objects, acc ->

4
lib/mix/tasks/pleroma/test_runner.ex
View file

@ -4,7 +4,9 @@ defmodule Mix.Tasks.Pleroma.TestRunner do
use Mix.Task
def run(args \\ []) do
case System.cmd("mix", ["test"] ++ args, into: IO.stream(:stdio, :line)) do
case System.cmd("mix", ["test", "--warnings-as-errors"] ++ args,
into: IO.stream(:stdio, :line)
) do
{_, 0} ->
:ok

2
lib/pleroma/activity/html.ex
View file

@ -38,7 +38,7 @@ defmodule Pleroma.Activity.HTML do
def invalidate_cache_for(activity_id) do
keys = get_cache_keys_for(activity_id)
Enum.map(keys, &@cachex.del(:scrubber_cache, &1))
Enum.each(keys, &@cachex.del(:scrubber_cache, &1))
@cachex.del(:scrubber_management_cache, activity_id)
end

18
lib/pleroma/activity/queries.ex
View file

@ -7,7 +7,7 @@ defmodule Pleroma.Activity.Queries do
Contains queries for Activity.
"""
import Ecto.Query, only: [from: 2, where: 3]
import Ecto.Query, only: [from: 2]
@type query :: Ecto.Queryable.t() | Pleroma.Activity.t()
@ -70,22 +70,6 @@ defmodule Pleroma.Activity.Queries do
)
end
@spec by_object_in_reply_to_id(query, String.t(), keyword()) :: query
def by_object_in_reply_to_id(query, in_reply_to_id, opts \\ []) do
query =
if opts[:skip_preloading] do
Activity.with_joined_object(query)
else
Activity.with_preloaded_object(query)
end
where(
query,
[activity, object: o],
fragment("(?)->>'inReplyTo' = ?", o.data, ^to_string(in_reply_to_id))
)
end
@spec by_type(query, String.t()) :: query
def by_type(query \\ Activity, activity_type) do
from(

101
lib/pleroma/application.ex
View file

@ -43,9 +43,7 @@ defmodule Pleroma.Application do
# every time the application is restarted, so we disable module
# conflicts at runtime
Code.compiler_options(ignore_module_conflict: true)
# Disable warnings_as_errors at runtime, it breaks Phoenix live reload
# due to protocol consolidation warnings
Code.compiler_options(warnings_as_errors: false)
configure_logger()
Pleroma.Telemetry.Logger.attach()
Config.Holder.save_default()
Pleroma.HTML.compile_scrubbers()
@ -56,7 +54,10 @@ defmodule Pleroma.Application do
Pleroma.Web.Plugs.HTTPSecurityPlug.warn_if_disabled()
end
Pleroma.ApplicationRequirements.verify!()
if Config.get(:env) != :test do
Pleroma.ApplicationRequirements.verify!()
end
load_custom_modules()
Pleroma.Docs.JSON.compile()
limiters_setup()
@ -68,26 +69,11 @@ defmodule Pleroma.Application do
Finch.start_link(name: MyFinch)
end
if adapter == Tesla.Adapter.Gun do
if version = Pleroma.OTPVersion.version() do
[major, minor] =
version
|> String.split(".")
|> Enum.map(&String.to_integer/1)
|> Enum.take(2)
if (major == 22 and minor < 2) or major < 22 do
raise "
!!!OTP VERSION WARNING!!!
You are using gun adapter with OTP version #{version}, which doesn't support correct handling of unordered certificates chains. Please update your Erlang/OTP to at least 22.2.
"
end
else
raise "
!!!OTP VERSION WARNING!!!
To support correct handling of unordered certificates chains - OTP version must be > 22.2.
"
end
# Disable warnings_as_errors at runtime, it breaks Phoenix live reload
# due to protocol consolidation warnings
# :warnings_as_errors is deprecated via Code.compiler_options/2 since 1.18
if Version.compare(System.version(), "1.18.0") == :lt do
Code.compiler_options(warnings_as_errors: false)
end
# Define workers and child supervisors to be supervised
@ -95,12 +81,15 @@ defmodule Pleroma.Application do
[
Pleroma.PromEx,
Pleroma.LDAP,
Pleroma.Repo,
Config.TransferTask,
Pleroma.Emoji,
Pleroma.Web.Plugs.RateLimiter.Supervisor,
{Task.Supervisor, name: Pleroma.TaskSupervisor}
Pleroma.Repo
] ++
search_children() ++
[
Config.TransferTask,
Pleroma.Emoji,
Pleroma.Web.Plugs.RateLimiter.Supervisor,
{Task.Supervisor, name: Pleroma.TaskSupervisor}
] ++
cachex_children() ++
http_children(adapter) ++
[
@ -128,6 +117,44 @@ defmodule Pleroma.Application do
Supervisor.start_link(children, opts)
end
def configure_logger do
Config.get([:logger, :backends], [])
|> Enum.each(fn backend ->
case backend_to_logger(backend) do
{:ok, logger} ->
add_logger(logger)
{:error, :console_backend} ->
Logger.warning(":console is no longer considered a backend and is enabled by default")
end
end)
end
defp add_logger(backend) do
case LoggerBackends.add(backend) do
{:ok, _} ->
Logger.debug("Successfully added logger backend: #{inspect(backend)}")
{:error, :already_present} ->
:ok
{:error, reason} ->
Logger.error("Failed to add logger backend #{inspect(backend)}: #{inspect(reason)}")
end
end
defp backend_to_logger(:console), do: {:error, :console_backend}
defp backend_to_logger({:ex_syslogger = backend, name}) do
Logger.warning(
"Configuration {:#{backend}, :#{name}} is incorrect. Use {ExSyslogger, :#{name}} instead!"
)
{:ok, {ExSyslogger, name}}
end
defp backend_to_logger(backend), do: {:ok, backend}
def load_custom_modules do
dir = Config.get([:modules, :runtime_dir])
@ -169,10 +196,21 @@ defmodule Pleroma.Application do
limit: 500_000
),
build_cachex("rel_me", limit: 2500),
build_cachex("host_meta", default_ttl: :timer.minutes(120), limit: 5000)
build_cachex("host_meta", default_ttl: :timer.minutes(120), limit: 5_000),
build_cachex("translations", default_ttl: :timer.hours(24), limit: 5_000)
]
end
defp search_children do
case Config.get([Pleroma.Search, :module]) do
Pleroma.Search.ParadeDB ->
[Pleroma.Search.ParadeDB.Repo]
_ ->
[]
end
end
defp emoji_packs_expiration,
do: expiration(default: :timer.seconds(5 * 60), interval: :timer.seconds(60))
@ -292,7 +330,8 @@ defmodule Pleroma.Application do
config = Config.get(ConcurrentLimiter, [])
[
Pleroma.Search
Pleroma.Search,
Pleroma.Webhook.Notify
]
|> Enum.each(fn module ->
mod_config = Keyword.get(config, module, [])

35
lib/pleroma/application_requirements.ex
View file

@ -189,7 +189,40 @@ defmodule Pleroma.ApplicationRequirements do
false
end
if Enum.all?([preview_proxy_commands_status | filter_commands_statuses], & &1) do
language_detector_commands_status =
if Pleroma.Language.LanguageDetector.missing_dependencies() == [] do
true
else
Logger.error(
"The following dependencies required by the currently enabled " <>
"language detection provider are not installed: " <>
inspect(Pleroma.Language.LanguageDetector.missing_dependencies())
)
false
end
translation_commands_status =
if Pleroma.Language.Translation.missing_dependencies() == [] do
true
else
Logger.error(
"The following dependencies required by the currently enabled " <>
"translation provider are not installed: " <>
inspect(Pleroma.Language.Translation.missing_dependencies())
)
false
end
if Enum.all?(
[
preview_proxy_commands_status,
language_detector_commands_status,
translation_commands_status | filter_commands_statuses
],
& &1
) do
:ok
else
{:error,

72
lib/pleroma/backports.ex Normal file
View file

@ -0,0 +1,72 @@
# Copyright 2012 Plataformatec
# Copyright 2021 The Elixir Team
# SPDX-License-Identifier: Apache-2.0
defmodule Pleroma.Backports do
import File, only: [dir?: 1]
# <https://github.com/elixir-lang/elixir/pull/14242>
# To be removed when we require Elixir 1.19
@doc """
Tries to create the directory `path`.
Missing parent directories are created. Returns `:ok` if successful, or
`{:error, reason}` if an error occurs.
Typical error reasons are:
* `:eacces` - missing search or write permissions for the parent
directories of `path`
* `:enospc` - there is no space left on the device
* `:enotdir` - a component of `path` is not a directory
"""
@spec mkdir_p(Path.t()) :: :ok | {:error, File.posix() | :badarg}
def mkdir_p(path) do
do_mkdir_p(IO.chardata_to_string(path))
end
defp do_mkdir_p("/") do
:ok
end
defp do_mkdir_p(path) do
parent = Path.dirname(path)
if parent == path do
:ok
else
case do_mkdir_p(parent) do
:ok ->
case :file.make_dir(path) do
{:error, :eexist} ->
if dir?(path), do: :ok, else: {:error, :enotdir}
other ->
other
end
e ->
e
end
end
end
@doc """
Same as `mkdir_p/1`, but raises a `File.Error` exception in case of failure.
Otherwise `:ok`.
"""
@spec mkdir_p!(Path.t()) :: :ok
def mkdir_p!(path) do
case mkdir_p(path) do
:ok ->
:ok
{:error, reason} ->
raise File.Error,
reason: reason,
action: "make directory (with -p)",
path: IO.chardata_to_string(path)
end
end
end

11
lib/pleroma/bookmark.ex
View file

@ -19,7 +19,7 @@ defmodule Pleroma.Bookmark do
schema "bookmarks" do
belongs_to(:user, User, type: FlakeId.Ecto.CompatType)
belongs_to(:activity, Activity, type: FlakeId.Ecto.CompatType)
belongs_to(:folder, BookmarkFolder, type: FlakeId.Ecto.CompatType)
belongs_to(:folder, BookmarkFolder, type: FlakeId.Ecto.Type)
timestamps()
end
@ -38,7 +38,7 @@ defmodule Pleroma.Bookmark do
|> validate_required([:user_id, :activity_id])
|> unique_constraint(:activity_id, name: :bookmarks_user_id_activity_id_index)
|> Repo.insert(
on_conflict: [set: [folder_id: folder_id]],
on_conflict: [set: [folder_id: folder_id, updated_at: NaiveDateTime.utc_now()]],
conflict_target: [:user_id, :activity_id]
)
end
@ -76,11 +76,4 @@ defmodule Pleroma.Bookmark do
|> Repo.one()
|> Repo.delete()
end
def set_folder(bookmark, folder_id) do
bookmark
|> cast(%{folder_id: folder_id}, [:folder_id])
|> validate_required([:folder_id])
|> Repo.update()
end
end

2
lib/pleroma/bookmark_folder.ex
View file

@ -14,7 +14,7 @@ defmodule Pleroma.BookmarkFolder do
alias Pleroma.User
@type t :: %__MODULE__{}
@primary_key {:id, FlakeId.Ecto.CompatType, autogenerate: true}
@primary_key {:id, FlakeId.Ecto.Type, autogenerate: true}
schema "bookmark_folders" do
field(:name, :string)

16
lib/pleroma/chat.ex
View file

@ -25,6 +25,8 @@ defmodule Pleroma.Chat do
belongs_to(:user, User, type: FlakeId.Ecto.CompatType)
field(:recipient, :string)
field(:pinned, :boolean)
timestamps()
end
@ -94,4 +96,18 @@ defmodule Pleroma.Chat do
order_by: [desc: c.updated_at]
)
end
def pin(%__MODULE__{} = chat) do
chat
|> cast(%{pinned: true}, [:pinned])
|> Repo.update()
end
def unpin(%__MODULE__{} = chat) do
chat
|> cast(%{pinned: false}, [:pinned])
|> Repo.update()
end
def enabled?, do: Pleroma.Config.get([__MODULE__, :enabled], true)
end

1
lib/pleroma/config.ex
View file

@ -27,6 +27,7 @@ defmodule Pleroma.Config do
Application.get_env(:pleroma, key, default)
end
@impl true
def get!(key) do
value = get(key, nil)

73
lib/pleroma/config/deprecation_warnings.ex
View file

@ -20,6 +20,8 @@ defmodule Pleroma.Config.DeprecationWarnings do
"\n* `config :pleroma, :instance, mrf_transparency_exclusions` is now `config :pleroma, :mrf, transparency_exclusions`"}
]
@logger_formatter_config_knobs [:colors, :format, :metadata, :truncate, :utc_log]
def check_exiftool_filter do
filters = Config.get([Pleroma.Upload]) |> Keyword.get(:filters, [])
@ -218,7 +220,8 @@ defmodule Pleroma.Config.DeprecationWarnings do
check_quarantined_instances_tuples(),
check_transparency_exclusions_tuples(),
check_simple_policy_tuples(),
check_exiftool_filter()
check_exiftool_filter(),
check_deprecated_logger_config()
]
|> Enum.reduce(:ok, fn
:ok, :ok -> :ok
@ -415,4 +418,72 @@ defmodule Pleroma.Config.DeprecationWarnings do
:ok
end
end
defp merge_deprecated_logger_config(config) do
formatter_config = Enum.filter(config, fn {k, _} -> k in @logger_formatter_config_knobs end)
formatter = Logger.default_formatter(formatter_config)
log_level = Keyword.get(config, :level, nil)
Logger.debug("""
Reconfiguring console Logger with deprecated configuration syntax.
Handler configuration:
log_level: #{inspect(log_level)}
Formatter:
#{inspect(formatter)}
""")
if Config.get(:env) != :test do
if log_level, do: :logger.update_handler_config(:default, :level, log_level)
:logger.set_handler_config(:default, :formatter, formatter)
end
end
@spec check_deprecated_logger_config() :: :ok | :error
def check_deprecated_logger_config do
backends_config = Application.get_env(:logger, :backends)
console_config = Application.get_env(:logger, :console)
# NOTE: No need to merge the old backends config since it still works.
# And new configuration will just add new Logger backends.
backend =
if backends_config do
Logger.warning("""
!!!DEPRECATION WARNING!!!
Your configuration is using deprecated syntax for configuring backends of Elixir's logger.
`config :logger, backends: [...]` is deprecated syntax due to changes in Elixir.
Please update your configuration at your earliest convenience to use:
`config :pleroma, :logger, backends: [...]`
Note: `:console` is no longer considered a backend and is used by default, you can disable it using:
`config :logger, :default_handler: false`
""")
true
else
false
end
console =
if console_config do
Logger.warning("""
!!!DEPRECATION WARNING!!!
Your configuration is using deprecated syntax for configuring logging to console.
`config :logger, :console` is deprecated syntax due to changes in Elixir.
Please update your configuration at your earliest convenience to use
`config :logger, default_handler` and `config :logger, :default_formatter`.
Note: `:default_handler` is used only for the `level` setting. All other configurations go under
`:default_formatter`. For more info visit: https://hexdocs.pm/logger/Logger.html#module-backends-and-backwards-compatibility
""")
merge_deprecated_logger_config(console_config)
true
else
false
end
if backend or console, do: :error, else: :ok
end
end

3
lib/pleroma/config/getting.ex
View file

@ -5,10 +5,13 @@
defmodule Pleroma.Config.Getting do
@callback get(any()) :: any()
@callback get(any(), any()) :: any()
@callback get!(any()) :: any()
def get(key), do: get(key, nil)
def get(key, default), do: impl().get(key, default)
def get!(key), do: impl().get!(key)
def impl do
Application.get_env(:pleroma, :config_impl, Pleroma.Config)
end

3
lib/pleroma/config/transfer_task.ex
View file

@ -47,11 +47,12 @@ defmodule Pleroma.Config.TransferTask do
settings =
(Repo.all(ConfigDB) ++ deleted_settings)
|> Enum.reject(&ConfigDB.static?/1)
|> Enum.map(&merge_with_default/1)
started_applications = Application.started_applications()
reject = [nil, :postgrex]
reject = [nil, :postgrex, :logger]
reject =
if restart_pleroma? do

80
lib/pleroma/config_db.ex
View file

@ -10,6 +10,7 @@ defmodule Pleroma.ConfigDB do
import Pleroma.Web.Gettext
alias __MODULE__
alias Pleroma.EctoType.Config.RateLimit
alias Pleroma.Repo
@type t :: %__MODULE__{}
@ -22,6 +23,12 @@ defmodule Pleroma.ConfigDB do
{:pleroma, :mrf_keyword, :replace}
]
@static_config [
{:pleroma, Pleroma.Search},
{:pleroma, Pleroma.Search.ParadeDB},
{:pleroma, Pleroma.Search.ParadeDB.Repo}
]
schema "config" do
field(:key, Pleroma.EctoType.Config.Atom)
field(:group, Pleroma.EctoType.Config.Atom)
@ -31,16 +38,32 @@ defmodule Pleroma.ConfigDB do
timestamps()
end
@spec get_all_as_keyword() :: keyword()
def get_all_as_keyword do
@spec get_all_as_keyword(keyword()) :: keyword()
def get_all_as_keyword(options \\ []) do
ConfigDB
|> select([c], {c.group, c.key, c.value})
|> Repo.all()
|> maybe_reject_static(options)
|> Enum.reduce([], fn {group, key, value}, acc ->
Keyword.update(acc, group, [{key, value}], &Keyword.merge(&1, [{key, value}]))
end)
end
def static?(%{group: group, key: key}), do: static?(group, key)
def static?(_config), do: false
def static?(group, key) do
Enum.any?(@static_config, fn {static_group, static_key} ->
group in [static_group, inspect(static_group)] and key in [static_key, inspect(static_key)]
end)
end
defp maybe_reject_static(configs, exclude_static: true) do
Enum.reject(configs, fn {group, key, _value} -> static?(group, key) end)
end
defp maybe_reject_static(configs, _options), do: configs
@spec get_all_by_group(atom() | String.t()) :: [t()]
def get_all_by_group(group) do
from(c in ConfigDB, where: c.group == ^group) |> Repo.all()
@ -60,8 +83,59 @@ defmodule Pleroma.ConfigDB do
|> cast(params, [:key, :group, :value])
|> validate_required([:key, :group, :value])
|> unique_constraint(:key, name: :config_group_key_index)
|> validate_rate_limit()
end
defp validate_rate_limit(changeset) do
group = get_field(changeset, :group)
key = get_field(changeset, :key)
if group == :pleroma and key == :rate_limit do
value = get_field(changeset, :value)
case normalize_rate_limit(value) do
{:ok, normalized_value} ->
put_change(changeset, :value, normalized_value)
{:error, {limiter_name, reason}} ->
add_error(
changeset,
:value,
"invalid :rate_limit value for #{inspect(limiter_name)}: #{reason}"
)
end
else
changeset
end
end
defp normalize_rate_limit(nil), do: {:ok, nil}
defp normalize_rate_limit(%{} = value), do: normalize_rate_limit(Map.to_list(value))
defp normalize_rate_limit(value) when is_list(value) do
if Keyword.keyword?(value) do
value
|> Enum.reduce_while({:ok, []}, fn {limiter_name, limiter_value}, {:ok, acc} ->
case RateLimit.cast_with_error(limiter_value) do
{:ok, normalized_limiter_value} ->
{:cont, {:ok, [{limiter_name, normalized_limiter_value} | acc]}}
{:error, reason} ->
{:halt, {:error, {limiter_name, reason}}}
end
end)
|> case do
{:ok, acc} -> {:ok, Enum.reverse(acc)}
{:error, _} = error -> error
end
else
{:error, {:rate_limit, "must be a keyword list"}}
end
end
defp normalize_rate_limit(_), do: {:error, {:rate_limit, "must be a keyword list"}}
defp create(params) do
%ConfigDB{}
|> changeset(params)
@ -302,7 +376,7 @@ defmodule Pleroma.ConfigDB do
end
def to_elixir_types(%{"tuple" => entity}) do
Enum.reduce(entity, {}, &Tuple.append(&2, to_elixir_types(&1)))
Enum.reduce(entity, {}, &Tuple.insert_at(&2, tuple_size(&2), to_elixir_types(&1)))
end
def to_elixir_types(entity) when is_map(entity) do

23
lib/pleroma/constants.ex
View file

@ -20,13 +20,16 @@ defmodule Pleroma.Constants do
"deleted_activity_id",
"pleroma_internal",
"generator",
"rules"
"rules",
"language",
"voters",
"assigned_account"
]
)
const(static_only_files,
do:
~w(index.html robots.txt static static-fe finmoji emoji packs sounds images instance sw.js sw-pleroma.js favicon.png schemas doc embed.js embed.css)
~w(index.html robots.txt static static-fe finmoji emoji packs sounds images instance sw.js sw-pleroma.js schemas doc embed.js embed.css)
)
const(status_updatable_fields,
@ -36,10 +39,12 @@ defmodule Pleroma.Constants do
"updated",
"emoji",
"content",
"contentMap",
"summary",
"sensitive",
"attachment",
"generator"
"generator",
"language"
]
)
@ -97,10 +102,12 @@ defmodule Pleroma.Constants do
"Add",
"Remove",
"Like",
"Dislike",
"Announce",
"Undo",
"Flag",
"EmojiReact"
"EmojiReact",
"Listen"
]
)
@ -111,6 +118,7 @@ defmodule Pleroma.Constants do
"Flag",
"Follow",
"Like",
"Dislike",
"EmojiReact",
"Announce"
]
@ -126,6 +134,13 @@ defmodule Pleroma.Constants do
do: ~r/^[^[:cntrl:] ()<>@,;:\\"\/\[\]?=]+\/[^[:cntrl:] ()<>@,;:\\"\/\[\]?=]+(; .*)?$/
)
# List of allowed chars in the path segment of a URI
# unreserved, sub-delims, ":", "@" and "/" allowed as the separator in path
# https://datatracker.ietf.org/doc/html/rfc3986
const(uri_path_allowed_reserved_chars,
do: ~c"!$&'()*+,;=/:@"
)
const(upload_object_types, do: ["Document", "Image"])
const(activity_json_canonical_mime_type,

3
lib/pleroma/date_time.ex Normal file
View file

@ -0,0 +1,3 @@
defmodule Pleroma.DateTime do
@callback utc_now() :: NaiveDateTime.t()
end

6
lib/pleroma/date_time/impl.ex Normal file
View file

@ -0,0 +1,6 @@
defmodule Pleroma.DateTime.Impl do
@behaviour Pleroma.DateTime
@impl true
def utc_now, do: NaiveDateTime.utc_now()
end

23
lib/pleroma/docs/generator.ex
View file

@ -12,13 +12,27 @@ defmodule Pleroma.Docs.Generator do
@spec list_behaviour_implementations(behaviour :: module()) :: [module()]
def list_behaviour_implementations(behaviour) do
:code.all_loaded()
|> Enum.filter(fn {module, _} ->
# :code.all_loaded() no longer works on Elixir 1.19 and newer since modules are lazily loaded.
# We can get a complete list of modules by interating over Application.loaded_applications/0,
# but that includes every module which would get loaded below, which is very slow.
# Hardcode the Pleroma application instead, which is good enough for now.
application_modules =
case :application.get_key(:pleroma, :modules) do
{:ok, modules} -> modules
:undefined -> []
end
# At compile time or in tests some modules will be outside of the Pleroma application.
loaded_modules = Enum.map(:code.all_loaded(), fn {module, _} -> module end)
(application_modules ++ loaded_modules)
|> Enum.uniq()
|> Enum.filter(fn module ->
Code.ensure_loaded(module)
# This shouldn't be needed as all modules are expected to have module_info/1,
# but in test environments some transient modules `:elixir_compiler_XX`
# are loaded for some reason (where XX is a random integer).
Code.ensure_loaded(module)
if function_exported?(module, :module_info, 1) do
module.module_info(:attributes)
|> Keyword.get_values(:behaviour)
@ -26,7 +40,6 @@ defmodule Pleroma.Docs.Generator do
|> Enum.member?(behaviour)
end
end)
|> Enum.map(fn {module, _} -> module end)
end
@doc """

49
lib/pleroma/ecto_type/activity_pub/object_validators/content_language_map.ex Normal file
View file

@ -0,0 +1,49 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2023 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.EctoType.ActivityPub.ObjectValidators.ContentLanguageMap do
use Ecto.Type
import Pleroma.EctoType.ActivityPub.ObjectValidators.LanguageCode,
only: [good_locale_code?: 1]
def type, do: :map
def cast(%{} = object) do
with {status, %{} = data} when status in [:modified, :ok] <- validate_map(object) do
{:ok, data}
else
{_, nil} -> {:ok, nil}
{:error, _} -> :error
end
end
def cast(_), do: :error
def dump(data), do: {:ok, data}
def load(data), do: {:ok, data}
defp validate_map(%{} = object) do
{status, data} =
object
|> Enum.reduce({:ok, %{}}, fn
{lang, value}, {status, acc} when is_binary(lang) and is_binary(value) ->
if good_locale_code?(lang) do
{status, Map.put(acc, lang, value)}
else
{:modified, acc}
end
_, {_status, acc} ->
{:modified, acc}
end)
if data == %{} do
{status, nil}
else
{status, data}
end
end
end

27
lib/pleroma/ecto_type/activity_pub/object_validators/language_code.ex Normal file
View file

@ -0,0 +1,27 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2023 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.EctoType.ActivityPub.ObjectValidators.LanguageCode do
use Ecto.Type
def type, do: :string
def cast(language) when is_binary(language) do
if good_locale_code?(language) do
{:ok, language}
else
{:error, :invalid_language}
end
end
def cast(_), do: :error
def dump(data), do: {:ok, data}
def load(data), do: {:ok, data}
def good_locale_code?(code) when is_binary(code), do: code =~ ~r<^[a-zA-Z0-9\-]+\z$>
def good_locale_code?(_code), do: false
end

71
lib/pleroma/ecto_type/config/rate_limit.ex Normal file
View file

@ -0,0 +1,71 @@
# Pleroma: A lightweight social networking server
# Copyright © Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.EctoType.Config.RateLimit do
@moduledoc false
use Ecto.Type
@type t ::
nil
| {non_neg_integer(), non_neg_integer()}
| [{non_neg_integer(), non_neg_integer()}]
@impl true
def type, do: :term
@impl true
def cast(value) do
case cast_with_error(value) do
{:ok, normalized} -> {:ok, normalized}
{:error, _reason} -> :error
end
end
@impl true
def load(value), do: cast(value)
@impl true
def dump(value), do: cast(value)
@spec cast_with_error(term()) :: {:ok, t()} | {:error, String.t()}
def cast_with_error(nil), do: {:ok, nil}
def cast_with_error({scale, limit}) do
with {:ok, scale} <- parse_integer(scale, "scale"),
{:ok, limit} <- parse_integer(limit, "limit"),
true <- scale >= 1 and limit >= 1 do
{:ok, {scale, limit}}
else
false -> {:error, "scale and limit must be >= 1"}
{:error, reason} -> {:error, reason}
end
end
def cast_with_error([{_, _} = unauth, {_, _} = auth]) do
with {:ok, unauth} <- cast_with_error(unauth),
{:ok, auth} <- cast_with_error(auth) do
{:ok, [unauth, auth]}
else
{:error, reason} -> {:error, reason}
end
end
def cast_with_error(_),
do:
{:error, "must be a {scale, limit} tuple, a [{scale, limit}, {scale, limit}] list, or nil"}
defp parse_integer(value, _label) when is_integer(value), do: {:ok, value}
defp parse_integer(value, label) when is_binary(value) do
value = String.trim(value)
case Integer.parse(value) do
{number, ""} -> {:ok, number}
_ -> {:error, "#{label} must be an integer"}
end
end
defp parse_integer(_value, label), do: {:error, "#{label} must be an integer"}
end

30
lib/pleroma/emoji.ex
View file

@ -11,6 +11,8 @@ defmodule Pleroma.Emoji do
alias Pleroma.Emoji.Combinations
alias Pleroma.Emoji.Loader
alias Pleroma.Utils.URIEncoding
alias Pleroma.Web.Endpoint
require Logger
@ -189,6 +191,34 @@ defmodule Pleroma.Emoji do
def emoji_url(_), do: nil
@spec local_url(String.t() | nil) :: String.t() | nil
def local_url(nil), do: nil
def local_url("http" <> _ = url) do
URIEncoding.encode_url(url)
end
def local_url("/" <> _ = path) do
path = URIEncoding.encode_url(path, bypass_parse: true, bypass_decode: true)
Endpoint.url() <> path
end
def local_url(path) when is_binary(path) do
local_url("/" <> path)
end
def build_emoji_tag({name, url}) do
url = URIEncoding.encode_url(url)
%{
"icon" => %{"url" => "#{url}", "type" => "Image"},
"name" => ":" <> name <> ":",
"type" => "Emoji",
"updated" => "1970-01-01T00:00:00Z",
"id" => url
}
end
def emoji_name_with_instance(name, url) do
url = url |> URI.parse() |> Map.get(:host)
"#{name}@#{url}"

3
lib/pleroma/emoji/formatter.ex
View file

@ -5,7 +5,6 @@
defmodule Pleroma.Emoji.Formatter do
alias Pleroma.Emoji
alias Pleroma.HTML
alias Pleroma.Web.Endpoint
alias Pleroma.Web.MediaProxy
def emojify(text) do
@ -44,7 +43,7 @@ defmodule Pleroma.Emoji.Formatter do
Emoji.get_all()
|> Enum.filter(fn {emoji, %Emoji{}} -> String.contains?(text, ":#{emoji}:") end)
|> Enum.reduce(%{}, fn {name, %Emoji{file: file}}, acc ->
Map.put(acc, name, to_string(URI.merge(Endpoint.url(), file)))
Map.put(acc, name, Emoji.local_url(file))
end)
end

198
lib/pleroma/emoji/pack.ex
View file

@ -24,12 +24,13 @@ defmodule Pleroma.Emoji.Pack do
alias Pleroma.Emoji
alias Pleroma.Emoji.Pack
alias Pleroma.SafeZip
alias Pleroma.Utils
@spec create(String.t()) :: {:ok, t()} | {:error, File.posix()} | {:error, :empty_values}
def create(name) do
with :ok <- validate_not_empty([name]),
dir <- Path.join(emoji_path(), name),
dir <- path_join_name_safe(emoji_path(), name),
:ok <- File.mkdir(dir) do
save_pack(%__MODULE__{pack_file: Path.join(dir, "pack.json")})
end
@ -65,43 +66,21 @@ defmodule Pleroma.Emoji.Pack do
{:ok, [binary()]} | {:error, File.posix(), binary()} | {:error, :empty_values}
def delete(name) do
with :ok <- validate_not_empty([name]),
pack_path <- Path.join(emoji_path(), name) do
pack_path <- path_join_name_safe(emoji_path(), name) do
File.rm_rf(pack_path)
end
end
@spec unpack_zip_emojies(list(tuple())) :: list(map())
defp unpack_zip_emojies(zip_files) do
Enum.reduce(zip_files, [], fn
{_, path, s, _, _, _}, acc when elem(s, 2) == :regular ->
with(
filename <- Path.basename(path),
shortcode <- Path.basename(filename, Path.extname(filename)),
false <- Emoji.exist?(shortcode)
) do
[%{path: path, filename: path, shortcode: shortcode} | acc]
else
_ -> acc
end
_, acc ->
acc
end)
end
@spec add_file(t(), String.t(), Path.t(), Plug.Upload.t()) ::
{:ok, t()}
| {:error, File.posix() | atom()}
def add_file(%Pack{} = pack, _, _, %Plug.Upload{content_type: "application/zip"} = file) do
with {:ok, zip_files} <- :zip.table(to_charlist(file.path)),
[_ | _] = emojies <- unpack_zip_emojies(zip_files),
with {:ok, zip_files} <- SafeZip.list_dir_file(file.path),
[_ | _] = emojies <- map_zip_emojies(zip_files),
{:ok, tmp_dir} <- Utils.tmp_dir("emoji") do
try do
{:ok, _emoji_files} =
:zip.unzip(
to_charlist(file.path),
[{:file_list, Enum.map(emojies, & &1[:path])}, {:cwd, String.to_charlist(tmp_dir)}]
)
SafeZip.unzip_file(file.path, tmp_dir, Enum.map(emojies, & &1[:path]))
{_, updated_pack} =
Enum.map_reduce(emojies, pack, fn item, emoji_pack ->
@ -246,6 +225,97 @@ defmodule Pleroma.Emoji.Pack do
end
end
def download_zip(name, opts \\ %{}) do
with :ok <- validate_not_empty([name]),
:ok <- validate_new_pack(name),
{:ok, archive_data} <- fetch_archive_data(opts),
pack_path <- path_join_name_safe(emoji_path(), name),
:ok <- create_pack_dir(pack_path),
:ok <- safe_unzip(archive_data, pack_path) do
ensure_pack_json(pack_path, archive_data, opts)
else
{:error, :empty_values} -> {:error, "Pack name cannot be empty"}
{:error, reason} when is_binary(reason) -> {:error, reason}
_ -> {:error, "Could not process pack"}
end
end
defp create_pack_dir(pack_path) do
case File.mkdir_p(pack_path) do
:ok -> :ok
{:error, _} -> {:error, "Could not create the pack directory"}
end
end
defp safe_unzip(archive_data, pack_path) do
case SafeZip.unzip_data(archive_data, pack_path) do
{:ok, _} -> :ok
{:error, reason} when is_binary(reason) -> {:error, reason}
_ -> {:error, "Could not unzip pack"}
end
end
defp validate_new_pack(name) do
pack_path = path_join_name_safe(emoji_path(), name)
if File.exists?(pack_path) do
{:error, "Pack already exists, refusing to import #{name}"}
else
:ok
end
end
defp fetch_archive_data(%{url: url}) do
case Pleroma.HTTP.get(url) do
{:ok, %{status: 200, body: data}} -> {:ok, data}
_ -> {:error, "Could not download pack"}
end
end
defp fetch_archive_data(%{file: %Plug.Upload{path: path}}) do
case File.read(path) do
{:ok, data} -> {:ok, data}
_ -> {:error, "Could not read the uploaded pack file"}
end
end
defp fetch_archive_data(_) do
{:error, "Neither file nor URL was present in the request"}
end
defp ensure_pack_json(pack_path, archive_data, opts) do
pack_json_path = Path.join(pack_path, "pack.json")
if not File.exists?(pack_json_path) do
create_pack_json(pack_path, pack_json_path, archive_data, opts)
end
:ok
end
defp create_pack_json(pack_path, pack_json_path, archive_data, opts) do
emoji_map =
Pleroma.Emoji.Loader.make_shortcode_to_file_map(
pack_path,
Map.get(opts, :exts, [".png", ".gif", ".jpg"])
)
archive_sha = :crypto.hash(:sha256, archive_data) |> Base.encode16()
pack_json = %{
pack: %{
license: Map.get(opts, :license, ""),
homepage: Map.get(opts, :homepage, ""),
description: Map.get(opts, :description, ""),
src: Map.get(opts, :url),
src_sha256: archive_sha
},
files: emoji_map
}
File.write!(pack_json_path, Jason.encode!(pack_json, pretty: true))
end
@spec download(String.t(), String.t(), String.t()) :: {:ok, t()} | {:error, atom()}
def download(name, url, as) do
uri = url |> String.trim() |> URI.parse()
@ -292,7 +362,7 @@ defmodule Pleroma.Emoji.Pack do
@spec load_pack(String.t()) :: {:ok, t()} | {:error, :file.posix()}
def load_pack(name) do
name = Path.basename(name)
pack_file = Path.join([emoji_path(), name, "pack.json"])
pack_file = path_join_name_safe(emoji_path(), name) |> Path.join("pack.json")
with {:ok, _} <- File.stat(pack_file),
{:ok, pack_data} <- File.read(pack_file) do
@ -416,10 +486,9 @@ defmodule Pleroma.Emoji.Pack do
end
defp create_archive_and_cache(pack, hash) do
files = [~c"pack.json" | Enum.map(pack.files, fn {_, file} -> to_charlist(file) end)]
{:ok, {_, result}} =
:zip.zip(~c"#{pack.name}.zip", files, [:memory, cwd: to_charlist(pack.path)])
pack_file_list = Enum.into(pack.files, [], fn {_, f} -> f end)
files = ["pack.json" | pack_file_list]
{:ok, {_, result}} = SafeZip.zip("#{pack.name}.zip", files, pack.path, true)
ttl_per_file = Pleroma.Config.get!([:emoji, :shared_pack_cache_seconds_per_file])
overall_ttl = :timer.seconds(ttl_per_file * Enum.count(files))
@ -478,7 +547,7 @@ defmodule Pleroma.Emoji.Pack do
end
defp save_file(%Plug.Upload{path: upload_path}, pack, filename) do
file_path = Path.join(pack.path, filename)
file_path = path_join_safe(pack.path, filename)
create_subdirs(file_path)
with {:ok, _} <- File.copy(upload_path, file_path) do
@ -497,8 +566,8 @@ defmodule Pleroma.Emoji.Pack do
end
defp rename_file(pack, filename, new_filename) do
old_path = Path.join(pack.path, filename)
new_path = Path.join(pack.path, new_filename)
old_path = path_join_safe(pack.path, filename)
new_path = path_join_safe(pack.path, new_filename)
create_subdirs(new_path)
with :ok <- File.rename(old_path, new_path) do
@ -510,13 +579,13 @@ defmodule Pleroma.Emoji.Pack do
with true <- String.contains?(file_path, "/"),
path <- Path.dirname(file_path),
false <- File.exists?(path) do
File.mkdir_p!(path)
Pleroma.Backports.mkdir_p!(path)
end
end
defp remove_file(pack, shortcode) do
with {:ok, filename} <- get_filename(pack, shortcode),
emoji <- Path.join(pack.path, filename),
emoji <- path_join_safe(pack.path, filename),
:ok <- File.rm(emoji) do
remove_dir_if_empty(emoji, filename)
end
@ -534,7 +603,7 @@ defmodule Pleroma.Emoji.Pack do
defp get_filename(pack, shortcode) do
with %{^shortcode => filename} when is_binary(filename) <- pack.files,
file_path <- Path.join(pack.path, filename),
file_path <- path_join_safe(pack.path, filename),
{:ok, _} <- File.stat(file_path) do
{:ok, filename}
else
@ -558,7 +627,7 @@ defmodule Pleroma.Emoji.Pack do
emoji_path = emoji_path()
# Create the directory first if it does not exist. This is probably the first request made
# with the API so it should be sufficient
with {:create_dir, :ok} <- {:create_dir, File.mkdir_p(emoji_path)},
with {:create_dir, :ok} <- {:create_dir, Pleroma.Backports.mkdir_p(emoji_path)},
{:ls, {:ok, results}} <- {:ls, File.ls(emoji_path)} do
{:ok, Enum.sort(results)}
else
@ -583,12 +652,11 @@ defmodule Pleroma.Emoji.Pack do
end
defp unzip(archive, pack_info, remote_pack, local_pack) do
with :ok <- File.mkdir_p!(local_pack.path) do
files = Enum.map(remote_pack["files"], fn {_, path} -> to_charlist(path) end)
with :ok <- Pleroma.Backports.mkdir_p!(local_pack.path) do
files = Enum.map(remote_pack["files"], fn {_, path} -> path end)
# Fallback cannot contain a pack.json file
files = if pack_info[:fallback], do: files, else: [~c"pack.json" | files]
:zip.unzip(archive, cwd: to_charlist(local_pack.path), file_list: files)
files = if pack_info[:fallback], do: files, else: ["pack.json" | files]
SafeZip.unzip_data(archive, local_pack.path, files)
end
end
@ -649,13 +717,43 @@ defmodule Pleroma.Emoji.Pack do
end
defp validate_has_all_files(pack, zip) do
with {:ok, f_list} <- :zip.unzip(zip, [:memory]) do
# Check if all files from the pack.json are in the archive
pack.files
|> Enum.all?(fn {_, from_manifest} ->
List.keyfind(f_list, to_charlist(from_manifest), 0)
# Check if all files from the pack.json are in the archive
eset =
Enum.reduce(pack.files, MapSet.new(), fn
{_, file}, s -> MapSet.put(s, to_charlist(file))
end)
|> if(do: :ok, else: {:error, :incomplete})
if SafeZip.contains_all_data?(zip, eset),
do: :ok,
else: {:error, :incomplete}
end
defp path_join_name_safe(dir, name) do
if to_string(name) != Path.basename(name) or name in ["..", ".", ""] do
raise "Invalid or malicious pack name: #{name}"
else
Path.join(dir, name)
end
end
defp path_join_safe(dir, path) do
{:ok, safe_path} = Path.safe_relative(path)
Path.join(dir, safe_path)
end
defp map_zip_emojies(zip_files) do
Enum.reduce(zip_files, [], fn path, acc ->
with(
filename <- Path.basename(path),
shortcode <- Path.basename(filename, Path.extname(filename)),
# note: this only checks the shortcode, if an emoji already exists on the same path, but
# with a different shortcode, the existing one will be degraded to an alias of the new
false <- Emoji.exist?(shortcode)
) do
[%{path: path, filename: path, shortcode: shortcode} | acc]
else
_ -> acc
end
end)
end
end

18
lib/pleroma/following_relationship.ex
View file

@ -147,14 +147,22 @@ defmodule Pleroma.FollowingRelationship do
|> Repo.aggregate(:count, :id)
end
def get_follow_requests(%User{id: id}) do
def get_follow_requests_query(%User{id: id}) do
__MODULE__
|> join(:inner, [r], f in assoc(r, :follower))
|> join(:inner, [r], f in assoc(r, :follower), as: :follower)
|> where([r], r.state == ^:follow_pending)
|> where([r], r.following_id == ^id)
|> where([r, f], f.is_active == true)
|> select([r, f], f)
|> Repo.all()
|> where([r, follower: f], f.is_active == true)
|> select([r, follower: f], f)
end
def get_outgoing_follow_requests_query(%User{id: id}) do
__MODULE__
|> join(:inner, [r], f in assoc(r, :following), as: :following)
|> where([r], r.state == ^:follow_pending)
|> where([r], r.follower_id == ^id)
|> where([r, following: f], f.is_active == true)
|> select([r, following: f], f)
end
def following?(%User{id: follower_id}, %User{id: followed_id}) do

11
lib/pleroma/formatter.ex
View file

@ -127,6 +127,13 @@ defmodule Pleroma.Formatter do
Earmark.as_html!(text, %Earmark.Options{compact_output: true, smartypants: false})
end
def markdown_to_html(text, opts) do
Earmark.as_html!(
text,
%Earmark.Options{compact_output: true, smartypants: false} |> Map.merge(opts)
)
end
def html_escape({text, mentions, hashtags}, type) do
{html_escape(text, type), mentions, hashtags}
end
@ -135,6 +142,10 @@ defmodule Pleroma.Formatter do
HTML.filter_tags(text)
end
def html_escape(text, "text/x.misskeymarkdown") do
HTML.filter_tags(text)
end
def html_escape(text, "text/plain") do
Regex.split(@link_regex, text, include_captures: true)
|> Enum.map_every(2, fn chunk ->

26
lib/pleroma/frontend.ex
View file

@ -65,30 +65,18 @@ defmodule Pleroma.Frontend do
end
def unzip(zip, dest) do
with {:ok, unzipped} <- :zip.unzip(zip, [:memory]) do
File.rm_rf!(dest)
File.mkdir_p!(dest)
File.rm_rf!(dest)
Pleroma.Backports.mkdir_p!(dest)
Enum.each(unzipped, fn {filename, data} ->
path = filename
new_file_path = Path.join(dest, path)
path
|> Path.dirname()
|> then(&Path.join(dest, &1))
|> File.mkdir_p!()
if not File.dir?(new_file_path) do
File.write!(new_file_path, data)
end
end)
case Pleroma.SafeZip.unzip_data(zip, dest) do
{:ok, _} -> :ok
error -> error
end
end
defp download_build(frontend_info, dest) do
Logger.info("Downloading pre-built bundle for #{frontend_info["name"]}")
url = String.replace(frontend_info["build_url"], "${ref}", frontend_info["ref"])
Logger.info("Downloading pre-built bundle for #{frontend_info["name"]} from #{url}")
with {:ok, %{status: 200, body: zip_body}} <-
Pleroma.HTTP.get(url, [], pool: :media, recv_timeout: 120_000) do
@ -102,7 +90,7 @@ defmodule Pleroma.Frontend do
defp install_frontend(frontend_info, source, dest) do
from = frontend_info["build_dir"] || "dist"
File.rm_rf!(dest)
File.mkdir_p!(dest)
Pleroma.Backports.mkdir_p!(dest)
File.cp_r!(Path.join([source, from]), dest)
:ok
end

37
lib/pleroma/gopher/server.ex
View file

@ -21,17 +21,28 @@ defmodule Pleroma.Gopher.Server do
def init([ip, port]) do
Logger.info("Starting gopher server on #{port}")
Process.flag(:trap_exit, true)
:ranch.start_listener(
:gopher,
100,
:ranch_tcp,
[ip: ip, port: port],
__MODULE__.ProtocolHandler,
[]
)
listener = :gopher
{:ok, %{ip: ip, port: port}}
{:ok, _pid} =
:ranch.start_listener(
listener,
:ranch_tcp,
%{
num_acceptors: 100,
max_connections: 100,
socket_opts: [ip: ip, port: port]
},
__MODULE__.ProtocolHandler,
[]
)
{:ok, %{ip: ip, port: port, listener: listener}}
end
def terminate(_reason, state) do
:ranch.stop_listener(state.listener)
end
end
@ -43,13 +54,13 @@ defmodule Pleroma.Gopher.Server.ProtocolHandler do
alias Pleroma.Web.ActivityPub.ActivityPub
alias Pleroma.Web.ActivityPub.Visibility
def start_link(ref, socket, transport, opts) do
pid = spawn_link(__MODULE__, :init, [ref, socket, transport, opts])
def start_link(ref, transport, opts) do
pid = spawn_link(__MODULE__, :init, [ref, transport, opts])
{:ok, pid}
end
def init(ref, socket, transport, [] = _Opts) do
:ok = :ranch.accept_ack(ref)
def init(ref, transport, opts \\ []) do
{:ok, socket} = :ranch.handshake(ref, opts)
loop(socket, transport)
end

9
lib/pleroma/gun.ex
View file

@ -6,9 +6,12 @@ defmodule Pleroma.Gun do
@callback open(charlist(), pos_integer(), map()) :: {:ok, pid()}
@callback info(pid()) :: map()
@callback close(pid()) :: :ok
@callback cancel(pid(), reference() | [reference()]) :: :ok
@callback await_up(pid, pos_integer()) :: {:ok, atom()} | {:error, atom()}
@callback connect(pid(), map()) :: reference()
@callback await(pid(), reference()) :: {:response, :fin, 200, []}
@callback await(pid(), reference()) :: tuple()
@callback await_tunnel_up(pid(), reference() | :undefined, timeout()) ::
{:ok, atom()} | {:error, term()}
@callback set_owner(pid(), pid()) :: :ok
defp api, do: Pleroma.Config.get([Pleroma.Gun], Pleroma.Gun.API)
@ -19,11 +22,15 @@ defmodule Pleroma.Gun do
def close(pid), do: api().close(pid)
def cancel(pid, stream), do: api().cancel(pid, stream)
def await_up(pid, timeout \\ 5_000), do: api().await_up(pid, timeout)
def connect(pid, opts), do: api().connect(pid, opts)
def await(pid, ref), do: api().await(pid, ref)
def await_tunnel_up(pid, ref, timeout), do: api().await_tunnel_up(pid, ref, timeout)
def set_owner(pid, owner), do: api().set_owner(pid, owner)
end

25
lib/pleroma/gun/api.ex
View file

@ -32,6 +32,9 @@ defmodule Pleroma.Gun.API do
@impl Gun
defdelegate close(pid), to: :gun
@impl Gun
defdelegate cancel(pid, stream), to: :gun
@impl Gun
defdelegate await_up(pid, timeout \\ 5_000), to: :gun
@ -41,6 +44,28 @@ defmodule Pleroma.Gun.API do
@impl Gun
defdelegate await(pid, ref), to: :gun
@impl Gun
def await_tunnel_up(pid, stream_ref, timeout) do
monitor_ref = Process.monitor(pid)
result =
receive do
{:gun_tunnel_up, ^pid, ^stream_ref, protocol} ->
{:ok, protocol}
{:gun_down, ^pid, _protocol, reason, _killed_streams} ->
{:error, {:down, reason}}
{:DOWN, ^monitor_ref, :process, ^pid, reason} ->
{:error, {:down, reason}}
after
timeout -> {:error, :timeout}
end
Process.demonitor(monitor_ref, [:flush])
result
end
@impl Gun
defdelegate set_owner(pid, owner), to: :gun
end

139
lib/pleroma/gun/conn.ex
View file

@ -42,18 +42,34 @@ defmodule Pleroma.Gun.Conn do
Map.put(opts, :tls_opts, tls_opts)
end
defp do_open(%URI{scheme: "http"} = uri, %{proxy: {proxy_host, proxy_port}} = opts) do
with open_opts <- opts |> proxy_open_opts() |> Map.put(:protocols, [:http]),
{:ok, conn} <- Gun.open(proxy_host, proxy_port, open_opts),
{:ok, protocol} <- await_up(conn, opts[:connect_timeout]) do
{:ok, conn, protocol, :forward_proxy}
else
error ->
Logger.warning(
"Opening forward proxy connection to #{compose_uri_log(uri)} failed with error #{inspect(error)}"
)
error
end
end
defp do_open(uri, %{proxy: {proxy_host, proxy_port}} = opts) do
connect_opts =
uri
|> destination_opts()
|> add_http2_opts(uri.scheme, Map.get(opts, :tls_opts, []))
|> add_proxy_auth(opts)
with open_opts <- Map.delete(opts, :tls_opts),
with open_opts <- proxy_open_opts(opts),
{:ok, conn} <- Gun.open(proxy_host, proxy_port, open_opts),
{:ok, protocol} <- Gun.await_up(conn, opts[:connect_timeout]),
stream <- Gun.connect(conn, connect_opts),
{:response, :fin, 200, _} <- Gun.await(conn, stream) do
{:ok, conn, protocol}
{:ok, _proxy_protocol} <- await_up(conn, opts[:connect_timeout]) do
conn
|> Gun.connect(connect_opts)
|> await_proxy_connect(conn, opts[:connect_timeout])
else
error ->
Logger.warning(
@ -64,30 +80,29 @@ defmodule Pleroma.Gun.Conn do
end
end
defp do_open(uri, %{proxy: {proxy_type, proxy_host, proxy_port}} = opts) do
version =
proxy_type
|> to_string()
|> String.last()
|> case do
"4" -> 4
_ -> 5
end
defp do_open(_uri, %{proxy: {:socks4, _proxy_host, _proxy_port}}) do
{:error, :socks4_unsupported}
end
defp do_open(uri, %{proxy: {proxy_type, proxy_host, proxy_port}} = opts)
when proxy_type in [:socks, :socks5] do
socks_opts =
uri
|> destination_opts()
|> add_http2_opts(uri.scheme, Map.get(opts, :tls_opts, []))
|> Map.put(:version, version)
|> Map.put(:version, 5)
|> add_socks_proxy_auth(opts)
opts =
open_opts =
opts
|> proxy_open_opts()
|> Map.put(:protocols, [:socks])
|> Map.put(:socks_opts, socks_opts)
with {:ok, conn} <- Gun.open(proxy_host, proxy_port, opts),
{:ok, protocol} <- Gun.await_up(conn, opts[:connect_timeout]) do
{:ok, conn, protocol}
with {:ok, conn} <- Gun.open(proxy_host, proxy_port, open_opts),
{:ok, :socks} <- await_up(conn, opts[:connect_timeout]),
{:ok, protocol} <- await_tunnel_up(conn, :undefined, opts[:connect_timeout]) do
{:ok, conn, protocol, nil}
else
error ->
Logger.warning(
@ -100,10 +115,11 @@ defmodule Pleroma.Gun.Conn do
defp do_open(%URI{host: host, port: port} = uri, opts) do
host = Pleroma.HTTP.AdapterHelper.parse_host(host)
opts = Map.put(opts, :transport, transport(uri.scheme))
with {:ok, conn} <- Gun.open(host, port, opts),
{:ok, protocol} <- Gun.await_up(conn, opts[:connect_timeout]) do
{:ok, conn, protocol}
{:ok, protocol} <- await_up(conn, opts[:connect_timeout]) do
{:ok, conn, protocol, nil}
else
error ->
Logger.warning(
@ -120,11 +136,90 @@ defmodule Pleroma.Gun.Conn do
end
defp add_http2_opts(opts, "https", tls_opts) do
Map.merge(opts, %{protocols: [:http2], transport: :tls, tls_opts: tls_opts})
Map.merge(opts, %{protocols: [:http2, :http], transport: :tls, tls_opts: tls_opts})
end
defp add_http2_opts(opts, _, _), do: opts
defp add_proxy_auth(connect_opts, %{proxy_auth: {username, password}})
when is_binary(username) and is_binary(password) do
Map.merge(connect_opts, %{username: username, password: password})
end
defp add_proxy_auth(connect_opts, _), do: connect_opts
defp add_socks_proxy_auth(socks_opts, %{proxy_auth: {username, password}})
when is_binary(username) and is_binary(password) do
Map.put(socks_opts, :auth, [{:username_password, username, password}])
end
defp add_socks_proxy_auth(socks_opts, _), do: socks_opts
defp proxy_open_opts(opts) do
proxy_tls_opts = Map.get(opts, :proxy_tls_opts, [])
opts =
opts
|> Map.drop([:proxy, :proxy_auth, :proxy_tls_opts, :tls_opts])
|> Map.put_new(:transport, :tcp)
if opts.transport == :tls do
Map.put(opts, :tls_opts, proxy_tls_opts)
else
opts
end
end
defp await_up(conn, timeout) do
case Gun.await_up(conn, timeout) do
{:ok, protocol} ->
{:ok, protocol}
error ->
Gun.close(conn)
error
end
end
defp await_proxy_connect(stream, conn, timeout) do
case Gun.await(conn, stream) do
{:response, :fin, 200, _headers} ->
case await_tunnel_up(conn, stream, timeout) do
{:ok, protocol} -> {:ok, conn, protocol, stream}
error -> error
end
{:response, _fin, 407, _headers} ->
close_with_error(conn, :proxy_auth_failed)
{:response, _fin, status, _headers} when is_integer(status) ->
close_with_error(conn, {:proxy_connect_failed, status})
error ->
Gun.close(conn)
error
end
end
defp close_with_error(conn, reason) do
Gun.close(conn)
{:error, reason}
end
defp transport("https"), do: :tls
defp transport(_), do: :tcp
defp await_tunnel_up(conn, stream_ref, timeout) do
case Gun.await_tunnel_up(conn, stream_ref, timeout) do
{:ok, protocol} ->
{:ok, protocol}
error ->
Gun.close(conn)
error
end
end
def compose_uri_log(%URI{scheme: scheme, host: host, path: path}) do
"#{scheme}://#{host}#{path}"
end

225
lib/pleroma/gun/connection_pool.ex
View file

@ -4,53 +4,132 @@
defmodule Pleroma.Gun.ConnectionPool do
@registry __MODULE__
@locks Module.concat(__MODULE__, Locks)
alias Pleroma.Gun.ConnectionPool.WorkerSupervisor
@connection_option_keys [
:http_opts,
:http2_opts,
:protocols,
:proxy,
:proxy_auth,
:proxy_tls_opts,
:retry,
:retry_timeout,
:socks_opts,
:tcp_opts,
:tls_opts,
:transport,
:ws_opts
]
def children do
[
{Registry, keys: :unique, name: @registry},
{Registry, keys: :duplicate, name: @registry},
{Registry, keys: :unique, name: @locks},
Pleroma.Gun.ConnectionPool.WorkerSupervisor
]
end
@spec get_conn(URI.t(), keyword()) :: {:ok, pid()} | {:error, term()}
def get_conn(uri, opts) do
key = "#{uri.scheme}:#{uri.host}:#{uri.port}"
key = connection_key(uri, opts)
case Registry.lookup(@registry, key) do
# The key has already been registered, but connection is not up yet
[{worker_pid, nil}] ->
get_gun_pid_from_worker(worker_pid, true)
[{worker_pid, {gun_pid, _used_by, _crf, _last_reference}}] ->
GenServer.call(worker_pid, :add_client)
case checkout_existing(Registry.lookup(@registry, key)) do
{:ok, gun_pid} ->
{:ok, gun_pid}
[] ->
# :gun.set_owner fails in :connected state for whatevever reason,
# so we open the connection in the process directly and send it's pid back
# We trust gun to handle timeouts by itself
case WorkerSupervisor.start_worker([key, uri, opts, self()]) do
{:ok, worker_pid} ->
get_gun_pid_from_worker(worker_pid, false)
:not_available ->
provision_or_wait(key, uri, opts)
end
end
{:error, {:already_started, worker_pid}} ->
get_gun_pid_from_worker(worker_pid, true)
defp checkout_existing([]), do: :not_available
err ->
err
defp checkout_existing([{_worker_pid, nil} | workers]), do: checkout_existing(workers)
defp checkout_existing([{worker_pid, _value} | workers]) do
case checkout_worker(worker_pid) do
{:ok, gun_pid} -> {:ok, gun_pid}
_ -> checkout_existing(workers)
end
end
defp provision_or_wait(key, uri, opts) do
result = with_route_lock(key, fn -> provision(key, uri, opts) end)
case result do
{:ok, gun_pid} ->
{:ok, gun_pid}
{:new, worker_pid} ->
get_gun_pid_from_worker(worker_pid)
{:wait, worker_pid} ->
case checkout_worker(worker_pid, :infinity) do
{:ok, gun_pid} -> {:ok, gun_pid}
:busy -> get_conn(uri, opts)
error -> error
end
error ->
error
end
end
defp provision(key, uri, opts) do
workers = Registry.lookup(@registry, key)
case checkout_existing(workers) do
{:ok, gun_pid} ->
{:ok, gun_pid}
:not_available ->
case Enum.find(workers, fn {_worker_pid, value} -> is_nil(value) end) do
{worker_pid, nil} -> {:wait, worker_pid}
nil -> start_worker(key, uri, opts)
end
end
end
defp get_gun_pid_from_worker(worker_pid, register) do
defp with_route_lock(key, function) do
case Registry.register(@locks, key, nil) do
{:ok, _} ->
try do
function.()
after
Registry.unregister(@locks, key)
end
{:error, {:already_registered, _owner}} ->
Process.sleep(1)
with_route_lock(key, function)
end
end
defp start_worker(key, uri, opts) do
# :gun.set_owner fails in :connected state, so the worker opens the connection itself.
case WorkerSupervisor.start_worker([key, uri, opts, self()]) do
{:ok, worker_pid} -> {:new, worker_pid}
error -> error
end
end
defp checkout_worker(worker_pid, timeout \\ 5_000) do
try do
GenServer.call(worker_pid, :checkout, timeout)
catch
:exit, reason -> {:error, {:connection_worker_exit, reason}}
end
end
defp get_gun_pid_from_worker(worker_pid) do
# GenServer.call will block the process for timeout length if
# the server crashes on startup (which will happen if gun fails to connect)
# so instead we use cast + monitor
ref = Process.monitor(worker_pid)
if register, do: GenServer.cast(worker_pid, {:add_client, self()})
receive do
{:conn_pid, pid} ->
@ -68,19 +147,115 @@ defmodule Pleroma.Gun.ConnectionPool do
@spec release_conn(pid()) :: :ok
def release_conn(conn_pid) do
# :ets.fun2ms(fn {_, {worker_pid, {gun_pid, _, _, _}}} when gun_pid == conn_pid ->
# :ets.fun2ms(fn {_, {worker_pid, {gun_pid, _tunnel}}} when gun_pid == conn_pid ->
# worker_pid end)
query_result =
Registry.select(@registry, [
{{:_, :"$1", {:"$2", :_, :_, :_}}, [{:==, :"$2", conn_pid}], [:"$1"]}
{{:_, :"$1", {:"$2", :_}}, [{:==, :"$2", conn_pid}], [:"$1"]}
])
case query_result do
[worker_pid] ->
GenServer.call(worker_pid, :remove_client)
try do
GenServer.call(worker_pid, :remove_client)
catch
:exit, _ -> :ok
end
[] ->
:ok
end
end
@spec discard_conn(pid()) :: :ok
def discard_conn(conn_pid) do
case worker_for_conn(conn_pid) do
nil ->
:ok
worker_pid ->
DynamicSupervisor.terminate_child(WorkerSupervisor, worker_pid)
end
end
@spec cancel_stream(pid(), reference() | [reference()]) :: :ok
def cancel_stream(conn_pid, stream) do
case worker_for_conn(conn_pid) do
nil ->
Pleroma.Gun.cancel(conn_pid, stream)
worker_pid ->
try do
GenServer.call(worker_pid, {:cancel_stream, stream})
catch
:exit, _ -> :ok
end
end
end
@spec register_stream(pid(), reference() | [reference()]) :: :ok
def register_stream(conn_pid, stream) do
call_worker(conn_pid, {:register_stream, stream})
end
@spec finish_stream(pid(), reference() | [reference()]) :: :ok
def finish_stream(conn_pid, stream) do
call_worker(conn_pid, {:finish_stream, stream})
end
@spec release_stream(pid(), reference() | [reference()]) :: :ok
def release_stream(conn_pid, stream) do
call_worker(conn_pid, {:release_stream, stream})
end
@spec tunnel_ref(pid()) :: reference() | :forward_proxy | nil
def tunnel_ref(conn_pid) do
case Registry.select(@registry, [
{{:_, :_, {:"$1", :"$2"}}, [{:==, :"$1", conn_pid}], [:"$2"]}
]) do
[tunnel] -> tunnel
[] -> nil
end
end
defp worker_for_conn(conn_pid) do
case Registry.select(@registry, [
{{:_, :"$1", {:"$2", :_}}, [{:==, :"$2", conn_pid}], [:"$1"]}
]) do
[worker_pid] -> worker_pid
[] -> nil
end
end
defp call_worker(conn_pid, message) do
case worker_for_conn(conn_pid) do
nil ->
:ok
worker_pid ->
try do
GenServer.call(worker_pid, message)
catch
:exit, _ -> :ok
end
end
end
defp connection_key(uri, opts) do
origin = {
String.downcase(uri.scheme),
String.downcase(uri.host),
uri.port
}
profile =
opts
|> Map.new()
|> Map.take(@connection_option_keys)
|> :erlang.term_to_binary()
|> then(&:crypto.hash(:sha256, &1))
fingerprint = Base.encode16(profile, case: :lower)
Enum.join(Tuple.to_list(origin) ++ [fingerprint], ":")
end
end

51
lib/pleroma/gun/connection_pool/reclaimer.ex
View file

@ -9,11 +9,11 @@ defmodule Pleroma.Gun.ConnectionPool.Reclaimer do
def start_monitor do
pid =
case GenServer.start(__MODULE__, [], name: {:via, Registry, {registry(), "reclaimer"}}) do
case GenServer.start(__MODULE__, [], name: __MODULE__) do
{:ok, pid} ->
pid
{:error, {:already_registered, pid}} ->
{:error, {:already_started, pid}} ->
pid
end
@ -41,17 +41,16 @@ defmodule Pleroma.Gun.ConnectionPool.Reclaimer do
reclaim_max: reclaim_max
})
# :ets.fun2ms(
# fn {_, {worker_pid, {_, used_by, crf, last_reference}}} when used_by == [] ->
# {worker_pid, crf, last_reference} end)
unused_conns =
workers =
Registry.select(
registry(),
[
{{:_, :"$1", {:_, :"$2", :"$3", :"$4"}}, [{:==, :"$2", []}], [{{:"$1", :"$3", :"$4"}}]}
{{:_, :"$1", {:_, :_}}, [], [:"$1"]}
]
)
unused_conns = Enum.flat_map(workers, &reclaim_info/1)
case unused_conns do
[] ->
:telemetry.execute(
@ -65,25 +64,43 @@ defmodule Pleroma.Gun.ConnectionPool.Reclaimer do
{:stop, :no_unused_conns, nil}
unused_conns ->
reclaimed =
reclaimed_count =
unused_conns
|> Enum.sort(fn {_pid1, crf1, last_reference1}, {_pid2, crf2, last_reference2} ->
crf1 <= crf2 and last_reference1 <= last_reference2
end)
|> Enum.take(reclaim_max)
|> Enum.sort_by(fn {_pid, crf, last_reference} -> {crf, last_reference} end)
|> Enum.reduce_while(0, fn
_worker, count when count == reclaim_max ->
{:halt, count}
reclaimed
|> Enum.each(fn {pid, _, _} ->
DynamicSupervisor.terminate_child(Pleroma.Gun.ConnectionPool.WorkerSupervisor, pid)
end)
{worker_pid, _crf, _last_reference}, count ->
if reclaim(worker_pid), do: {:cont, count + 1}, else: {:cont, count}
end)
:telemetry.execute(
[:pleroma, :connection_pool, :reclaim, :stop],
%{reclaimed_count: Enum.count(reclaimed)},
%{reclaimed_count: reclaimed_count},
%{max_connections: max_connections}
)
{:stop, :normal, nil}
end
end
defp reclaim_info(worker_pid) do
try do
case GenServer.call(worker_pid, :reclaim_info) do
{:idle, crf, last_reference} -> [{worker_pid, crf, last_reference}]
:in_use -> []
end
catch
:exit, _ -> []
end
end
defp reclaim(worker_pid) do
try do
GenServer.call(worker_pid, :reclaim) == :reclaimed
catch
:exit, _ -> false
end
end
end

268
lib/pleroma/gun/connection_pool/worker.ex
View file

@ -8,25 +8,22 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do
defp registry, do: Pleroma.Gun.ConnectionPool
def start_link([key | _] = opts) do
GenServer.start_link(__MODULE__, opts, name: {:via, Registry, {registry(), key}})
end
def start_link(opts), do: GenServer.start_link(__MODULE__, opts)
@impl true
def init([_key, _uri, _opts, _client_pid] = opts) do
def init([key, _uri, _opts, _client_pid] = opts) do
{:ok, _} = Registry.register(registry(), key, nil)
{:ok, nil, {:continue, {:connect, opts}}}
end
@impl true
def handle_continue({:connect, [key, uri, opts, client_pid]}, _) do
with {:ok, conn_pid, protocol} <- Gun.Conn.open(uri, opts),
with {:ok, conn_pid, protocol, tunnel} <- Gun.Conn.open(uri, opts),
Process.link(conn_pid) do
time = :erlang.monotonic_time(:millisecond)
{_, _} =
Registry.update_value(registry(), key, fn _ ->
{conn_pid, [client_pid], 1, time}
end)
Registry.unregister(registry(), key)
{:ok, _} = Registry.register(registry(), key, {conn_pid, tunnel})
send(client_pid, {:conn_pid, conn_pid})
@ -34,8 +31,14 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do
%{
key: key,
timer: nil,
client_monitors: %{client_pid => Process.monitor(client_pid)},
protocol: protocol
clients: %{
client_pid => %{count: 1, monitor: Process.monitor(client_pid), streams: MapSet.new()}
},
conn_pid: conn_pid,
protocol: protocol,
proxied: Keyword.has_key?(opts, :proxy),
crf: 1,
last_reference: time
}, :hibernate}
else
err ->
@ -44,106 +47,225 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do
end
@impl true
def handle_cast({:add_client, client_pid}, state) do
case handle_call(:add_client, {client_pid, nil}, state) do
{:reply, conn_pid, state, :hibernate} ->
send(client_pid, {:conn_pid, conn_pid})
{:noreply, state, :hibernate}
end
def handle_call(:checkout, _from, %{protocol: protocol, clients: clients} = state)
when protocol != :http2 and map_size(clients) > 0 do
{:reply, :busy, state, :hibernate}
end
@impl true
def handle_cast({:remove_client, client_pid}, state) do
case handle_call(:remove_client, {client_pid, nil}, state) do
{:reply, _, state, :hibernate} ->
{:noreply, state, :hibernate}
end
end
@impl true
def handle_call(:add_client, {client_pid, _}, %{key: key, protocol: protocol} = state) do
def handle_call(:checkout, {client_pid, _}, %{key: key, protocol: protocol} = state) do
time = :erlang.monotonic_time(:millisecond)
{{conn_pid, used_by, _, _}, _} =
Registry.update_value(registry(), key, fn {conn_pid, used_by, crf, last_reference} ->
{conn_pid, [client_pid | used_by], crf(time - last_reference, crf), time}
end)
[{_, {conn_pid, _tunnel}}] =
Registry.lookup(registry(), key) |> Enum.filter(&(elem(&1, 0) == self()))
state =
state
|> cancel_idle_timer()
|> add_client(client_pid)
|> Map.put(:crf, crf(time - state.last_reference, state.crf))
|> Map.put(:last_reference, time)
:telemetry.execute(
[:pleroma, :connection_pool, :client, :add],
%{client_pid: client_pid, clients: used_by},
%{client_pid: client_pid, clients: Map.keys(state.clients)},
%{key: state.key, protocol: protocol}
)
state =
if state.timer != nil do
Process.cancel_timer(state[:timer])
%{state | timer: nil}
else
state
end
ref = Process.monitor(client_pid)
state = put_in(state.client_monitors[client_pid], ref)
{:reply, conn_pid, state, :hibernate}
{:reply, {:ok, conn_pid}, state, :hibernate}
end
@impl true
def handle_call(:remove_client, {client_pid, _}, %{key: key} = state) do
{{_conn_pid, used_by, _crf, _last_reference}, _} =
Registry.update_value(registry(), key, fn {conn_pid, used_by, crf, last_reference} ->
{conn_pid, List.delete(used_by, client_pid), crf, last_reference}
end)
def handle_call(:remove_client, {client_pid, _}, state) do
case active_streams(state, client_pid) do
[] ->
{:reply, :ok, remove_client(state, client_pid, true), :hibernate}
{ref, state} = pop_in(state.client_monitors[client_pid])
streams when state.protocol == :http2 ->
Enum.each(streams, &(:ok = Gun.cancel(state.conn_pid, &1)))
{:reply, :ok, remove_client(state, client_pid, true), :hibernate}
Process.demonitor(ref, [:flush])
_streams ->
{:stop, :normal, :ok, state}
end
end
timer =
if used_by == [] do
max_idle = Pleroma.Config.get([:connections_pool, :max_idle_time], 30_000)
Process.send_after(self(), :idle_close, max_idle)
else
nil
end
def handle_call({:register_stream, stream}, {client_pid, _}, state) do
{:reply, :ok, update_stream(state, client_pid, &MapSet.put(&1, stream)), :hibernate}
end
{:reply, :ok, %{state | timer: timer}, :hibernate}
def handle_call({:finish_stream, stream}, {client_pid, _}, state) do
{:reply, :ok, update_stream(state, client_pid, &MapSet.delete(&1, stream)), :hibernate}
end
def handle_call({:release_stream, stream}, {client_pid, _}, state) do
state = update_stream(state, client_pid, &MapSet.delete(&1, stream))
{:reply, :ok, remove_client(state, client_pid, true), :hibernate}
end
def handle_call({:cancel_stream, stream}, {client_pid, _}, %{protocol: :http2} = state) do
:ok = Gun.cancel(state.conn_pid, stream)
state = update_stream(state, client_pid, &MapSet.delete(&1, stream))
{:reply, :ok, remove_client(state, client_pid, true), :hibernate}
end
def handle_call({:cancel_stream, _stream}, _from, state) do
{:stop, :normal, :ok, state}
end
def handle_call(:reclaim_info, _from, %{clients: clients} = state)
when map_size(clients) == 0 do
{:reply, {:idle, state.crf, state.last_reference}, state, :hibernate}
end
def handle_call(:reclaim_info, _from, state) do
{:reply, :in_use, state, :hibernate}
end
def handle_call(:reclaim, _from, %{clients: clients} = state) when map_size(clients) == 0 do
{:stop, :normal, :reclaimed, state}
end
def handle_call(:reclaim, _from, state) do
{:reply, :in_use, state, :hibernate}
end
@impl true
def handle_info(:idle_close, state) do
# Gun monitors the owner process, and will close the connection automatically
# when it's terminated
def handle_info({:idle_close, token}, %{timer: {_timer_ref, token}, clients: clients} = state)
when map_size(clients) == 0 do
# Gun monitors the owner process, and will close the connection automatically.
{:stop, :normal, state}
end
def handle_info({:idle_close, _token}, state), do: {:noreply, state, :hibernate}
@impl true
def handle_info({:gun_up, _pid, _protocol}, state) do
{:noreply, state, :hibernate}
def handle_info({:gun_up, _pid, protocol}, state) do
{:noreply, %{state | protocol: protocol}, :hibernate}
end
# Gracefully shutdown if the connection got closed without any streams left
@impl true
def handle_info({:gun_error, conn_pid, stream, _reason}, %{conn_pid: conn_pid} = state) do
{:noreply, delete_stream(state, stream), :hibernate}
end
@impl true
def handle_info({:gun_down, _pid, _protocol, _reason, _streams}, %{proxied: true} = state) do
{:stop, :normal, state}
end
# Gracefully shutdown if the connection got closed without any streams left.
@impl true
def handle_info({:gun_down, _pid, _protocol, _reason, []}, state) do
{:stop, :normal, state}
end
# Otherwise, wait for retry
# Otherwise, wait for retry.
@impl true
def handle_info({:gun_down, _pid, _protocol, _reason, _killed_streams}, state) do
{:noreply, state, :hibernate}
end
@impl true
def handle_info({:DOWN, _ref, :process, pid, reason}, state) do
:telemetry.execute(
[:pleroma, :connection_pool, :client, :dead],
%{client_pid: pid, reason: reason},
%{key: state.key}
)
def handle_info({:DOWN, ref, :process, pid, reason}, state) do
case state.clients[pid] do
%{monitor: ^ref} ->
:telemetry.execute(
[:pleroma, :connection_pool, :client, :dead],
%{client_pid: pid, reason: reason},
%{key: state.key}
)
handle_cast({:remove_client, pid}, state)
handle_dead_client(state, pid)
_ ->
{:noreply, state, :hibernate}
end
end
defp add_client(state, client_pid) do
update_in(state.clients, fn clients ->
case clients[client_pid] do
nil ->
Map.put(clients, client_pid, %{
count: 1,
monitor: Process.monitor(client_pid),
streams: MapSet.new()
})
client ->
Map.put(clients, client_pid, %{client | count: client.count + 1})
end
end)
end
defp handle_dead_client(state, client_pid) do
streams = active_streams(state, client_pid)
cond do
streams == [] ->
{:noreply, remove_client(state, client_pid, false, true), :hibernate}
state.protocol == :http2 ->
Enum.each(streams, &(:ok = Gun.cancel(state.conn_pid, &1)))
{:noreply, remove_client(state, client_pid, false, true), :hibernate}
true ->
{:stop, :normal, state}
end
end
defp active_streams(state, client_pid) do
case state.clients[client_pid] do
%{streams: streams} -> MapSet.to_list(streams)
nil -> []
end
end
defp update_stream(state, client_pid, update) do
update_in(state.clients, fn clients ->
case clients[client_pid] do
nil -> clients
client -> Map.put(clients, client_pid, %{client | streams: update.(client.streams)})
end
end)
end
defp delete_stream(state, stream) do
update_in(state.clients, fn clients ->
Map.new(clients, fn {pid, client} ->
{pid, %{client | streams: MapSet.delete(client.streams, stream)}}
end)
end)
end
defp remove_client(state, client_pid, demonitor, all \\ false) do
case state.clients[client_pid] do
nil ->
state
%{count: count} = client when count > 1 and not all ->
put_in(state.clients[client_pid], %{client | count: count - 1})
client ->
if demonitor, do: Process.demonitor(client.monitor, [:flush])
state = update_in(state.clients, &Map.delete(&1, client_pid))
if map_size(state.clients) == 0, do: schedule_idle_close(state), else: state
end
end
defp cancel_idle_timer(%{timer: nil} = state), do: state
defp cancel_idle_timer(%{timer: {timer_ref, _token}} = state) do
Process.cancel_timer(timer_ref)
%{state | timer: nil}
end
defp schedule_idle_close(state) do
max_idle = Pleroma.Config.get([:connections_pool, :max_idle_time], 30_000)
token = make_ref()
timer_ref = Process.send_after(self(), {:idle_close, token}, max_idle)
%{state | timer: {timer_ref, token}}
end
# LRFU policy: https://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.55.1478

9
lib/pleroma/gun/connection_pool/worker_supervisor.ex
View file

@ -26,7 +26,14 @@ defmodule Pleroma.Gun.ConnectionPool.WorkerSupervisor do
def start_worker(opts, true) do
case DynamicSupervisor.start_child(__MODULE__, {Worker, opts}) do
{:error, :max_children} ->
:telemetry.execute([:pleroma, :connection_pool, :provision_failure], %{opts: opts})
[key | _] = opts
:telemetry.execute(
[:pleroma, :connection_pool, :provision_failure],
%{},
%{key: key}
)
{:error, :pool_full}
res ->

89
lib/pleroma/hashtag.ex
View file

@ -12,6 +12,7 @@ defmodule Pleroma.Hashtag do
alias Pleroma.Hashtag
alias Pleroma.Object
alias Pleroma.Repo
alias Pleroma.User.HashtagFollow
schema "hashtags" do
field(:name, :string)
@ -27,6 +28,14 @@ defmodule Pleroma.Hashtag do
|> String.trim()
end
def get_by_id(id) do
Repo.get(Hashtag, id)
end
def get_by_name(name) do
Repo.get_by(Hashtag, name: normalize_name(name))
end
def get_or_create_by_name(name) do
changeset = changeset(%Hashtag{}, %{name: name})
@ -103,4 +112,84 @@ defmodule Pleroma.Hashtag do
{:ok, deleted_count}
end
end
def get_followers(%Hashtag{id: hashtag_id}) do
from(hf in HashtagFollow)
|> where([hf], hf.hashtag_id == ^hashtag_id)
|> join(:inner, [hf], u in assoc(hf, :user))
|> select([hf, u], u.id)
|> Repo.all()
end
def get_recipients_for_activity(%Pleroma.Activity{object: %{hashtags: tags}})
when is_list(tags) do
tags
|> Enum.map(&get_followers/1)
|> List.flatten()
|> Enum.uniq()
end
def get_recipients_for_activity(_activity), do: []
def search(query, options \\ []) do
limit = Keyword.get(options, :limit, 20)
offset = Keyword.get(options, :offset, 0)
search_terms =
query
|> String.downcase()
|> String.trim()
|> String.split(~r/\s+/)
|> Enum.filter(&(&1 != ""))
|> Enum.map(&String.trim_leading(&1, "#"))
|> Enum.filter(&(&1 != ""))
if Enum.empty?(search_terms) do
[]
else
# Use PostgreSQL's ANY operator with array for efficient multi-term search
# This is much more efficient than multiple OR clauses
search_patterns = Enum.map(search_terms, &"%#{&1}%")
# Create ranking query that prioritizes exact matches and closer matches
# Use a subquery to properly handle computed columns in ORDER BY
base_query =
from(ht in Hashtag,
where: fragment("LOWER(?) LIKE ANY(?)", ht.name, ^search_patterns),
select: %{
name: ht.name,
# Ranking: exact matches get highest priority (0)
# then prefix matches (1), then contains (2)
match_rank:
fragment(
"""
CASE
WHEN LOWER(?) = ANY(?) THEN 0
WHEN LOWER(?) LIKE ANY(?) THEN 1
ELSE 2
END
""",
ht.name,
^search_terms,
ht.name,
^Enum.map(search_terms, &"#{&1}%")
),
# Secondary sort by name length (shorter names first)
name_length: fragment("LENGTH(?)", ht.name)
}
)
from(result in subquery(base_query),
order_by: [
asc: result.match_rank,
asc: result.name_length,
asc: result.name
],
limit: ^limit,
offset: ^offset
)
|> Repo.all()
|> Enum.map(& &1.name)
end
end
end

30
lib/pleroma/http.ex
View file

@ -105,20 +105,30 @@ defmodule Pleroma.HTTP do
end
defp adapter_middlewares(Tesla.Adapter.Gun, extra_middleware) do
[Tesla.Middleware.FollowRedirects, Pleroma.Tesla.Middleware.ConnectionPool] ++
default_middleware() ++
[Pleroma.Tesla.Middleware.ConnectionPool] ++
extra_middleware
end
defp adapter_middlewares({Tesla.Adapter.Finch, _}, extra_middleware) do
[Tesla.Middleware.FollowRedirects] ++ extra_middleware
end
defp adapter_middlewares(_, extra_middleware) do
if Pleroma.Config.get(:env) == :test do
# Emulate redirects in test env, which are handled by adapters in other environments
[Tesla.Middleware.FollowRedirects]
else
extra_middleware
# A lot of tests are written expecting unencoded URLs
# and the burden of fixing that is high. Also it makes
# them hard to read. Tests will opt-in when we want to validate
# the encoding is being done correctly.
cond do
Pleroma.Config.get(:env) == :test and Pleroma.Config.get(:test_url_encoding) ->
default_middleware()
Pleroma.Config.get(:env) == :test ->
# Emulate redirects in test env, which are handled by adapters in other environments
[Tesla.Middleware.FollowRedirects]
# Hackney and Finch
true ->
default_middleware() ++ extra_middleware
end
end
defp default_middleware,
do: [Tesla.Middleware.FollowRedirects, Pleroma.Tesla.Middleware.EncodeUrl]
end

253
lib/pleroma/http/adapter/gun.ex Normal file
View file

@ -0,0 +1,253 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.HTTP.Adapter.Gun do
@behaviour Tesla.Adapter
alias Pleroma.Gun.ConnectionPool
alias Tesla.Multipart
@default_timeout 1_000
@impl Tesla.Adapter
def call(env, opts) do
adapter_opts =
Tesla.Adapter.opts(
[close_conn: true, body_as: :plain, send_body: :at_once, receive: true],
env,
opts
)
|> Map.new()
with {:ok, status, headers, body} <- request(env, adapter_opts) do
env = %{env | status: status, headers: format_headers(headers), body: body}
{:ok, put_stream_ref(env, body)}
end
end
defp put_stream_ref(env, %{stream: stream}) do
put_in(env.opts[:adapter][:stream], stream)
end
defp put_stream_ref(env, _body), do: env
defp request(env, %{conn: conn, tunnel: tunnel} = opts) do
uri = URI.parse(Tesla.build_url(env))
path = request_path(uri, tunnel)
method = Tesla.Adapter.Shared.format_method(env.method)
{headers, body, send_body} = prepare_body(env.headers, env.body, opts[:send_body])
headers = maybe_add_forward_proxy_headers(headers, uri, opts, tunnel)
request_opts = request_opts(opts[:reply_to], tunnel)
stream =
try do
open_stream(conn, method, path, headers, body, request_opts, send_body)
catch
kind, reason ->
ConnectionPool.release_conn(conn)
:erlang.raise(kind, reason, __STACKTRACE__)
end
:ok = ConnectionPool.register_stream(conn, stream)
:ok = send_stream_body(conn, stream, body, send_body)
response = read_response(conn, stream, opts)
case response do
{:error, reason} ->
ConnectionPool.cancel_stream(conn, stream)
{:error, {:gun_stream_error, reason}}
response ->
unless opts[:body_as] in [:chunks, :stream] and stream_body?(response) do
:ok = ConnectionPool.finish_stream(conn, stream)
end
if opts[:close_conn] and opts[:body_as] not in [:stream, :chunks] do
:gun.close(conn)
end
response
end
end
defp stream_body?({:ok, _status, _headers, %{stream: _stream}}), do: true
defp stream_body?({:ok, _status, _headers, %Stream{}}), do: true
defp stream_body?(_response), do: false
defp request_opts(reply_to, nil), do: %{reply_to: reply_to || self()}
defp request_opts(reply_to, :forward_proxy), do: %{reply_to: reply_to || self()}
defp request_opts(reply_to, tunnel) do
%{reply_to: reply_to || self(), tunnel: tunnel}
end
defp request_path(uri, :forward_proxy) do
path = Tesla.Adapter.Shared.prepare_path(uri.path, uri.query)
"#{uri.scheme}://#{authority(uri)}#{path}"
end
defp request_path(uri, _tunnel) do
Tesla.Adapter.Shared.prepare_path(uri.path, uri.query)
end
defp maybe_add_forward_proxy_headers(headers, uri, opts, :forward_proxy) do
headers
|> put_header_new("host", authority(uri))
|> maybe_put_proxy_authorization(opts)
end
defp maybe_add_forward_proxy_headers(headers, _uri, _opts, _tunnel), do: headers
defp maybe_put_proxy_authorization(headers, %{proxy_auth: {username, password}})
when is_binary(username) and is_binary(password) do
value = "Basic " <> Base.encode64(username <> ":" <> password)
List.keystore(headers, "proxy-authorization", 0, {"proxy-authorization", value})
end
defp maybe_put_proxy_authorization(headers, _opts), do: headers
defp put_header_new(headers, name, value) do
if List.keymember?(headers, name, 0), do: headers, else: [{name, value} | headers]
end
defp authority(%URI{scheme: scheme, host: host, port: port}) do
host = if String.contains?(host, ":"), do: "[#{host}]", else: host
if port == URI.default_port(scheme), do: host, else: "#{host}:#{port}"
end
defp prepare_body(headers, %Multipart{} = multipart, _send_body) do
{format_headers(headers ++ Multipart.headers(multipart)), Multipart.body(multipart), :stream}
end
defp prepare_body(headers, body, _send_body)
when is_struct(body, Stream) or is_function(body) do
{format_headers(headers), body, :stream}
end
defp prepare_body(headers, body, send_body) do
{format_headers(headers), body || "", send_body}
end
defp open_stream(conn, method, path, headers, _body, request_opts, :stream) do
:gun.headers(conn, method, path, headers, request_opts)
end
defp open_stream(conn, method, path, headers, body, request_opts, :at_once) do
:gun.request(conn, method, path, headers, body, request_opts)
end
defp send_stream_body(conn, stream, body, :stream) do
try do
for data <- body, do: :ok = :gun.data(conn, stream, :nofin, data)
:gun.data(conn, stream, :fin, "")
catch
kind, reason ->
ConnectionPool.cancel_stream(conn, stream)
:erlang.raise(kind, reason, __STACKTRACE__)
end
end
defp send_stream_body(_conn, _stream, _body, :at_once), do: :ok
defp read_response(conn, stream, opts) do
receive? = opts[:receive]
receive do
{:gun_response, ^conn, ^stream, :fin, status, headers} ->
{:ok, status, headers, ""}
{:gun_response, ^conn, ^stream, :nofin, status, headers} ->
format_response(conn, stream, opts, status, headers, opts[:body_as])
{:gun_up, ^conn, _protocol} when receive? ->
read_response(conn, stream, opts)
{:gun_error, ^conn, reason} ->
{:error, reason}
{:gun_error, ^conn, ^stream, reason} ->
{:error, reason}
{:gun_down, ^conn, _protocol, _reason, _killed_streams} when receive? ->
read_response(conn, stream, opts)
{:DOWN, _ref, :process, ^conn, reason} ->
{:error, reason}
after
opts[:timeout] || @default_timeout -> {:error, :recv_response_timeout}
end
end
defp format_response(conn, stream, opts, status, headers, :plain) do
case read_body(conn, stream, opts) do
{:ok, body} ->
{:ok, status, headers, body}
{:error, error} ->
:ok = :gun.flush(stream)
{:error, error}
end
end
defp format_response(conn, stream, opts, status, headers, :stream) do
body =
Stream.resource(
fn -> :reading end,
fn
:reading ->
case Tesla.Adapter.Gun.read_chunk(conn, stream, opts) do
{:nofin, part} -> {[part], :reading}
{:fin, part} -> {[part], :done}
{:error, reason} -> raise "Gun stream failed: #{inspect(reason)}"
end
:done ->
{:halt, :done}
end,
fn _state ->
if opts[:close_conn], do: :gun.close(conn)
end
)
{:ok, status, headers, body}
end
defp format_response(conn, stream, opts, status, headers, :chunks) do
{:ok, status, headers, %{pid: conn, stream: stream, opts: Map.to_list(opts)}}
end
defp read_body(conn, stream, opts, acc \\ "") do
receive do
{:gun_data, ^conn, ^stream, :fin, body} ->
append_body(acc, body, opts[:max_body])
{:gun_data, ^conn, ^stream, :nofin, body} ->
with {:ok, acc} <- append_body(acc, body, opts[:max_body]) do
read_body(conn, stream, opts, acc)
end
{:gun_error, ^conn, ^stream, reason} ->
{:error, reason}
{:DOWN, _ref, :process, ^conn, reason} ->
{:error, reason}
after
opts[:timeout] || @default_timeout -> {:error, :recv_body_timeout}
end
end
defp append_body(acc, part, nil), do: {:ok, acc <> part}
defp append_body(acc, part, limit) do
body = acc <> part
if byte_size(body) <= limit, do: {:ok, body}, else: {:error, :body_too_large}
end
defp format_headers(headers) do
Enum.map(headers, fn {key, value} ->
{String.downcase(to_string(key)), to_string(value)}
end)
end
end

4
lib/pleroma/http/adapter_helper/gun.ex
View file

@ -48,10 +48,10 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do
Keyword.put(opts, :timeout, recv_timeout)
end
# Gun uses [body_as: :stream]
# Keep the pool lease until the caller consumes or cancels the body.
defp maybe_stream(opts) do
case Keyword.pop(opts, :stream, nil) do
{true, opts} -> Keyword.put(opts, :body_as, :stream)
{true, opts} -> Keyword.put(opts, :body_as, :chunks)
{_, opts} -> opts
end
end

10
lib/pleroma/http/adapter_helper/hackney.ex
View file

@ -6,8 +6,9 @@ defmodule Pleroma.HTTP.AdapterHelper.Hackney do
@behaviour Pleroma.HTTP.AdapterHelper
@defaults [
follow_redirect: true,
force_redirect: true
follow_redirect: false,
force_redirect: false,
with_body: true
]
@spec options(keyword(), URI.t()) :: keyword()
@ -16,7 +17,12 @@ defmodule Pleroma.HTTP.AdapterHelper.Hackney do
config_opts = Pleroma.Config.get([:http, :adapter], [])
url_encoding =
Keyword.new()
|> Keyword.put(:path_encode_fun, fn path -> path end)
@defaults
|> Keyword.merge(url_encoding)
|> Keyword.merge(config_opts)
|> Keyword.merge(connection_opts)
|> add_scheme_opts(uri)

37
lib/pleroma/instances.ex
View file

@ -15,25 +15,7 @@ defmodule Pleroma.Instances do
defdelegate set_unreachable(url_or_host, unreachable_since \\ nil), to: Instance
defdelegate get_consistently_unreachable, to: Instance
def set_consistently_unreachable(url_or_host),
do: set_unreachable(url_or_host, reachability_datetime_threshold())
def reachability_datetime_threshold do
federation_reachability_timeout_days =
Pleroma.Config.get([:instance, :federation_reachability_timeout_days], 0)
if federation_reachability_timeout_days > 0 do
NaiveDateTime.add(
NaiveDateTime.utc_now(),
-federation_reachability_timeout_days * 24 * 3600,
:second
)
else
~N[0000-01-01 00:00:00]
end
end
defdelegate get_unreachable, to: Instance
def host(url_or_host) when is_binary(url_or_host) do
if url_or_host =~ ~r/^http/i do
@ -42,4 +24,21 @@ defmodule Pleroma.Instances do
url_or_host
end
end
@doc "Schedules reachability checks for all unreachable instances"
def check_all_unreachable do
get_unreachable()
|> Enum.each(fn {domain, _} ->
Pleroma.Workers.ReachabilityWorker.new(%{"domain" => domain})
|> Oban.insert()
end)
end
@doc "Deletes all users and activities for unreachable instances"
def delete_all_unreachable do
get_unreachable()
|> Enum.each(fn {domain, _} ->
Instance.delete(domain)
end)
end
end

48
lib/pleroma/instances/instance.ex
View file

@ -9,7 +9,6 @@ defmodule Pleroma.Instances.Instance do
alias Pleroma.Instances.Instance
alias Pleroma.Maps
alias Pleroma.Repo
alias Pleroma.User
alias Pleroma.Workers.DeleteWorker
use Ecto.Schema
@ -51,7 +50,7 @@ defmodule Pleroma.Instances.Instance do
|> cast(params, [:software_name, :software_version, :software_repository])
end
def filter_reachable([]), do: %{}
def filter_reachable([]), do: []
def filter_reachable(urls_or_hosts) when is_list(urls_or_hosts) do
hosts =
@ -68,19 +67,15 @@ defmodule Pleroma.Instances.Instance do
)
|> Map.new(& &1)
reachability_datetime_threshold = Instances.reachability_datetime_threshold()
for entry <- Enum.filter(urls_or_hosts, &is_binary/1) do
host = host(entry)
unreachable_since = unreachable_since_by_host[host]
if !unreachable_since ||
NaiveDateTime.compare(unreachable_since, reachability_datetime_threshold) == :gt do
{entry, unreachable_since}
if is_nil(unreachable_since) do
entry
end
end
|> Enum.filter(& &1)
|> Map.new(& &1)
end
def reachable?(url_or_host) when is_binary(url_or_host) do
@ -88,7 +83,7 @@ defmodule Pleroma.Instances.Instance do
from(i in Instance,
where:
i.host == ^host(url_or_host) and
i.unreachable_since <= ^Instances.reachability_datetime_threshold(),
not is_nil(i.unreachable_since),
select: true
)
)
@ -97,9 +92,16 @@ defmodule Pleroma.Instances.Instance do
def reachable?(url_or_host) when is_binary(url_or_host), do: true
def set_reachable(url_or_host) when is_binary(url_or_host) do
%Instance{host: host(url_or_host)}
|> changeset(%{unreachable_since: nil})
|> Repo.insert(on_conflict: {:replace, [:unreachable_since]}, conflict_target: :host)
host = host(url_or_host)
result =
%Instance{host: host}
|> changeset(%{unreachable_since: nil})
|> Repo.insert(on_conflict: {:replace, [:unreachable_since]}, conflict_target: :host)
Pleroma.Workers.ReachabilityWorker.delete_jobs_for_host(host)
result
end
def set_reachable(_), do: {:error, nil}
@ -132,11 +134,9 @@ defmodule Pleroma.Instances.Instance do
def set_unreachable(_, _), do: {:error, nil}
def get_consistently_unreachable do
reachability_datetime_threshold = Instances.reachability_datetime_threshold()
def get_unreachable do
from(i in Instance,
where: ^reachability_datetime_threshold > i.unreachable_since,
where: not is_nil(i.unreachable_since),
order_by: i.unreachable_since,
select: {i.host, i.unreachable_since}
)
@ -296,20 +296,14 @@ defmodule Pleroma.Instances.Instance do
Deletes all users from an instance in a background task, thus also deleting
all of those users' activities and notifications.
"""
def delete_users_and_activities(host) when is_binary(host) do
def delete(host) when is_binary(host) do
DeleteWorker.new(%{"op" => "delete_instance", "host" => host})
|> Oban.insert()
end
def perform(:delete_instance, host) when is_binary(host) do
User.Query.build(%{nickname: "@#{host}"})
|> Repo.chunk_stream(100, :batches)
|> Stream.each(fn users ->
users
|> Enum.each(fn user ->
User.perform(:delete, user)
end)
end)
|> Stream.run()
@doc "Schedules reachability check for instance"
def check_unreachable(domain) when is_binary(domain) do
Pleroma.Workers.ReachabilityWorker.new(%{"domain" => domain})
|> Oban.insert()
end
end

59
lib/pleroma/language/language_detector.ex Normal file
View file

@ -0,0 +1,59 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Language.LanguageDetector do
import Pleroma.EctoType.ActivityPub.ObjectValidators.LanguageCode,
only: [good_locale_code?: 1]
@words_threshold 4
@config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config)
def configured? do
provider = get_provider()
!!provider and provider.configured?()
end
def missing_dependencies do
provider = get_provider()
if provider do
provider.missing_dependencies()
else
[]
end
end
# Strip tags from text, etc.
defp prepare_text(text) do
text
|> Floki.parse_fragment!()
|> Floki.filter_out(
".h-card, .mention, .hashtag, .u-url, .quote-inline, .recipients-inline, code, pre"
)
|> Floki.text()
end
def detect(text) do
provider = get_provider()
text = prepare_text(text)
word_count = text |> String.split(~r/\s+/) |> Enum.count()
if word_count < @words_threshold or !provider or !provider.configured?() do
nil
else
with language <- provider.detect(text),
true <- good_locale_code?(language) do
language
else
_ -> nil
end
end
end
defp get_provider do
@config_impl.get([__MODULE__, :provider])
end
end

47
lib/pleroma/language/language_detector/fasttext.ex Normal file
View file

@ -0,0 +1,47 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Language.LanguageDetector.Fasttext do
import Pleroma.Web.Utils.Guards, only: [not_empty_string: 1]
alias Pleroma.Language.LanguageDetector.Provider
@behaviour Provider
@impl Provider
def missing_dependencies do
if Pleroma.Utils.command_available?("fasttext") do
[]
else
["fasttext"]
end
end
@impl Provider
def configured?, do: not_empty_string(get_model())
@impl Provider
def detect(text) do
text_path = Path.join(System.tmp_dir!(), "fasttext-#{Ecto.UUID.generate()}")
File.write(text_path, text |> String.replace(~r/\s+/, " "))
detected_language =
case System.cmd("fasttext", ["predict", get_model(), text_path]) do
{"__label__" <> language, _} ->
language |> String.trim()
_ ->
nil
end
File.rm(text_path)
detected_language
end
defp get_model do
Pleroma.Config.get([__MODULE__, :model])
end
end

11
lib/pleroma/language/language_detector/provider.ex Normal file
View file

@ -0,0 +1,11 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Language.LanguageDetector.Provider do
@callback missing_dependencies() :: [String.t()]
@callback configured?() :: boolean()
@callback detect(text :: String.t()) :: String.t() | nil
end

127
lib/pleroma/language/translation.ex Normal file
View file

@ -0,0 +1,127 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Language.Translation do
@cachex Pleroma.Config.get([:cachex, :provider], Cachex)
def configured? do
provider = get_provider()
!!provider and provider.configured?()
end
def missing_dependencies do
provider = get_provider()
if provider do
provider.missing_dependencies()
else
[]
end
end
def translate(text, source_language, target_language) do
cache_key = get_cache_key(text, source_language, target_language)
case @cachex.get(:translations_cache, cache_key) do
{:ok, nil} ->
provider = get_provider()
result =
if !configured?() do
{:error, :not_found}
else
provider.translate(text, source_language, target_language)
|> scrub_html()
end
store_result(result, cache_key)
result
{:ok, result} ->
{:ok, result}
{:error, error} ->
{:error, error}
end
end
def supported_languages(type) when type in [:source, :target] do
provider = get_provider()
cache_key = "#{type}_languages/#{provider.name()}"
case @cachex.get(:translations_cache, cache_key) do
{:ok, nil} ->
result =
if !configured?() do
{:error, :not_found}
else
provider.supported_languages(type)
end
store_result(result, cache_key)
result
{:ok, result} ->
{:ok, result}
{:error, error} ->
{:error, error}
end
end
def languages_matrix do
provider = get_provider()
cache_key = "languages_matrix/#{provider.name()}"
case @cachex.get(:translations_cache, cache_key) do
{:ok, nil} ->
result =
if !configured?() do
{:error, :not_found}
else
provider.languages_matrix()
end
store_result(result, cache_key)
result
{:ok, result} ->
{:ok, result}
{:error, error} ->
{:error, error}
end
end
defp get_provider, do: Pleroma.Config.get([__MODULE__, :provider])
defp get_cache_key(text, source_language, target_language) do
"#{source_language}/#{target_language}/#{content_hash(text)}"
end
defp store_result({:ok, result}, cache_key) do
@cachex.put(:translations_cache, cache_key, result)
end
defp store_result(_, _), do: nil
defp content_hash(text), do: :crypto.hash(:sha256, text) |> Base.encode64()
defp scrub_html({:ok, %{content: content} = result}) when is_binary(content) do
scrubbers = Pleroma.Config.get([:markup, :scrub_policy])
content
|> Pleroma.HTML.filter_tags(scrubbers)
{:ok, %{result | content: content}}
end
defp scrub_html(result), do: result
end

121
lib/pleroma/language/translation/deepl.ex Normal file
View file

@ -0,0 +1,121 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Language.Translation.Deepl do
import Pleroma.Web.Utils.Guards, only: [not_empty_string: 1]
alias Pleroma.Language.Translation.Provider
use Provider
@behaviour Provider
@name "DeepL"
@impl Provider
def configured?, do: not_empty_string(base_url()) and not_empty_string(api_key())
@impl Provider
def translate(content, source_language, target_language) do
endpoint =
base_url()
|> URI.merge("/v2/translate")
|> URI.to_string()
case Pleroma.HTTP.post(
endpoint,
Jason.encode!(%{
text: [content],
source_lang: source_language |> String.upcase(),
target_lang: target_language,
tag_handling: "html"
}),
[
{"Content-Type", "application/json"},
{"Authorization", "DeepL-Auth-Key #{api_key()}"}
]
) do
{:ok, %{status: 429}} ->
{:error, :too_many_requests}
{:ok, %{status: 456}} ->
{:error, :quota_exceeded}
{:ok, %{status: 200} = res} ->
%{
"translations" => [
%{"text" => content, "detected_source_language" => detected_source_language}
]
} = Jason.decode!(res.body)
{:ok,
%{
content: content,
detected_source_language: detected_source_language,
provider: @name
}}
_ ->
{:error, :internal_server_error}
end
end
@impl Provider
def supported_languages(type) when type in [:source, :target] do
endpoint =
base_url()
|> URI.merge("/v2/languages")
|> URI.to_string()
case Pleroma.HTTP.post(
endpoint <> "?" <> URI.encode_query(%{type: type}),
"",
[
{"Content-Type", "application/x-www-form-urlencoded"},
{"Authorization", "DeepL-Auth-Key #{api_key()}"}
]
) do
{:ok, %{status: 200} = res} ->
languages =
Jason.decode!(res.body)
|> Enum.map(fn %{"language" => language} -> language |> String.downcase() end)
|> Enum.map(fn language ->
if String.contains?(language, "-") do
[language, language |> String.split("-") |> Enum.at(0)]
else
language
end
end)
|> List.flatten()
|> Enum.uniq()
{:ok, languages}
_ ->
{:error, :internal_server_error}
end
end
@impl Provider
def languages_matrix do
with {:ok, source_languages} <- supported_languages(:source),
{:ok, target_languages} <- supported_languages(:target) do
{:ok,
Map.new(source_languages, fn language -> {language, target_languages -- [language]} end)}
else
{:error, error} -> {:error, error}
end
end
@impl Provider
def name, do: @name
defp base_url do
Pleroma.Config.get([__MODULE__, :base_url])
end
defp api_key do
Pleroma.Config.get([__MODULE__, :api_key])
end
end

93
lib/pleroma/language/translation/libretranslate.ex Normal file
View file

@ -0,0 +1,93 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Language.Translation.Libretranslate do
import Pleroma.Web.Utils.Guards, only: [not_empty_string: 1]
alias Pleroma.Language.Translation.Provider
use Provider
@behaviour Provider
@name "LibreTranslate"
@impl Provider
def configured?, do: not_empty_string(base_url()) and not_empty_string(api_key())
@impl Provider
def translate(content, source_language, target_language) do
case Pleroma.HTTP.post(
base_url() <> "/translate",
Jason.encode!(%{
q: content,
source: source_language |> String.upcase(),
target: target_language,
format: "html",
api_key: api_key()
}),
[
{"Content-Type", "application/json"}
]
) do
{:ok, %{status: 429}} ->
{:error, :too_many_requests}
{:ok, %{status: 403}} ->
{:error, :quota_exceeded}
{:ok, %{status: 200} = res} ->
%{
"translatedText" => content
} = Jason.decode!(res.body)
{:ok,
%{
content: content,
detected_source_language: source_language,
provider: @name
}}
_ ->
{:error, :internal_server_error}
end
end
@impl Provider
def supported_languages(_) do
case Pleroma.HTTP.get(base_url() <> "/languages") do
{:ok, %{status: 200} = res} ->
languages =
Jason.decode!(res.body)
|> Enum.map(fn %{"code" => code} -> code end)
{:ok, languages}
_ ->
{:error, :internal_server_error}
end
end
@impl Provider
def languages_matrix do
with {:ok, source_languages} <- supported_languages(:source),
{:ok, target_languages} <- supported_languages(:target) do
{:ok,
Map.new(source_languages, fn language -> {language, target_languages -- [language]} end)}
else
{:error, error} -> {:error, error}
end
end
@impl Provider
def name, do: @name
defp base_url do
Pleroma.Config.get([__MODULE__, :base_url])
end
defp api_key do
Pleroma.Config.get([__MODULE__, :api_key], "")
end
end

109
lib/pleroma/language/translation/mozhi.ex Normal file
View file

@ -0,0 +1,109 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2024 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Language.Translation.Mozhi do
import Pleroma.Web.Utils.Guards, only: [not_empty_string: 1]
alias Pleroma.Language.Translation.Provider
use Provider
@behaviour Provider
@name "Mozhi"
@impl Provider
def configured?, do: not_empty_string(base_url()) and not_empty_string(engine())
@impl Provider
def translate(content, source_language, target_language) do
endpoint =
base_url()
|> URI.merge("/api/translate")
|> URI.to_string()
case Pleroma.HTTP.get(
endpoint <>
"?" <>
URI.encode_query(%{
engine: engine(),
text: content,
from: source_language,
to: target_language
}),
[{"Accept", "application/json"}]
) do
{:ok, %{status: 200} = res} ->
%{
"translated-text" => content,
"source_language" => source_language
} = Jason.decode!(res.body)
{:ok,
%{
content: content,
detected_source_language: source_language,
provider: @name
}}
_ ->
{:error, :internal_server_error}
end
end
@impl Provider
def supported_languages(type) when type in [:source, :target] do
path =
case type do
:source -> "/api/source_languages"
:target -> "/api/target_languages"
end
endpoint =
base_url()
|> URI.merge(path)
|> URI.to_string()
case Pleroma.HTTP.get(
endpoint <>
"?" <>
URI.encode_query(%{
engine: engine()
}),
[{"Accept", "application/json"}]
) do
{:ok, %{status: 200} = res} ->
languages =
Jason.decode!(res.body)
|> Enum.map(fn %{"Id" => language} -> language end)
{:ok, languages}
_ ->
{:error, :internal_server_error}
end
end
@impl Provider
def languages_matrix do
with {:ok, source_languages} <- supported_languages(:source),
{:ok, target_languages} <- supported_languages(:target) do
{:ok,
Map.new(source_languages, fn language -> {language, target_languages -- [language]} end)}
else
{:error, error} -> {:error, error}
end
end
@impl Provider
def name, do: @name
defp base_url do
Pleroma.Config.get([__MODULE__, :base_url])
end
defp engine do
Pleroma.Config.get([__MODULE__, :engine])
end
end

40
lib/pleroma/language/translation/provider.ex Normal file
View file

@ -0,0 +1,40 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Language.Translation.Provider do
alias Pleroma.Language.Translation.Provider
@callback missing_dependencies() :: [String.t()]
@callback configured?() :: boolean()
@callback translate(
content :: String.t(),
source_language :: String.t(),
target_language :: String.t()
) ::
{:ok,
%{
content: String.t(),
detected_source_language: String.t(),
provider: String.t()
}}
| {:error, atom()}
@callback supported_languages(type :: :string | :target) ::
{:ok, [String.t()]} | {:error, atom()}
@callback languages_matrix() :: {:ok, map()} | {:error, atom()}
@callback name() :: String.t()
defmacro __using__(_opts) do
quote do
@impl Provider
def missing_dependencies, do: []
defoverridable missing_dependencies: 0
end
end
end

129
lib/pleroma/language/translation/translate_locally.ex Normal file
View file

@ -0,0 +1,129 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2024 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Language.Translation.TranslateLocally do
alias Pleroma.Language.Translation.Provider
use Provider
@behaviour Provider
@name "translateLocally"
@impl Provider
def missing_dependencies do
if Pleroma.Utils.command_available?("translateLocally") do
[]
else
["translateLocally"]
end
end
@impl Provider
def configured?, do: is_map(models())
@impl Provider
def translate(content, source_language, target_language) do
model =
models()
|> Map.get(source_language, %{})
|> Map.get(target_language)
models =
if model do
[model]
else
[
models()
|> Map.get(source_language, %{})
|> Map.get(intermediary_language()),
models()
|> Map.get(intermediary_language(), %{})
|> Map.get(target_language)
]
end
translated_content =
Enum.reduce(models, content, fn model, content ->
text_path = Path.join(System.tmp_dir!(), "translateLocally-#{Ecto.UUID.generate()}")
File.write(text_path, content)
translated_content =
case System.cmd("translateLocally", ["-m", model, "-i", text_path, "--html"]) do
{content, _} -> content
_ -> nil
end
File.rm(text_path)
translated_content
end)
{:ok,
%{
content: translated_content,
detected_source_language: source_language,
provider: @name
}}
end
@impl Provider
def supported_languages(:source) do
languages =
languages_matrix()
|> elem(1)
|> Map.keys()
{:ok, languages}
end
@impl Provider
def supported_languages(:target) do
languages =
languages_matrix()
|> elem(1)
|> Map.values()
|> List.flatten()
|> Enum.uniq()
{:ok, languages}
end
@impl Provider
def languages_matrix do
languages =
models()
|> Map.to_list()
|> Enum.map(fn {key, value} -> {key, Map.keys(value)} end)
|> Enum.into(%{})
matrix =
if intermediary_language() do
languages
|> Map.to_list()
|> Enum.map(fn {key, value} ->
with_intermediary =
(((value ++ languages[intermediary_language()])
|> Enum.uniq()) --
[key])
|> Enum.sort()
{key, with_intermediary}
end)
|> Enum.into(%{})
else
languages
end
{:ok, matrix}
end
@impl Provider
def name, do: @name
defp models, do: Pleroma.Config.get([__MODULE__, :models])
defp intermediary_language, do: Pleroma.Config.get([__MODULE__, :intermediary_language])
end

63
lib/pleroma/list.ex
View file

@ -9,6 +9,7 @@ defmodule Pleroma.List do
import Ecto.Changeset
alias Pleroma.Activity
alias Pleroma.Emoji
alias Pleroma.Repo
alias Pleroma.User
@ -17,14 +18,54 @@ defmodule Pleroma.List do
field(:title, :string)
field(:following, {:array, :string}, default: [])
field(:ap_id, :string)
field(:exclusive, :boolean, default: false)
field(:emoji, :string)
timestamps()
end
def title_changeset(list, attrs \\ %{}) do
def update_changeset(list, attrs \\ %{}) do
list
|> cast(attrs, [:title])
|> cast(attrs, [:title, :exclusive, :emoji])
|> validate_required([:title])
|> fix_emoji()
|> validate_emoji()
end
defp fix_emoji(changeset) do
with {:emoji_field, emoji} when is_binary(emoji) <-
{:emoji_field, get_field(changeset, :emoji)},
{:fixed_emoji, emoji} <-
{:fixed_emoji,
emoji
|> Emoji.fully_qualify_emoji()
|> Emoji.maybe_quote()} do
put_change(changeset, :emoji, emoji)
else
{:emoji_field, _} -> changeset
end
end
defp validate_emoji(changeset) do
validate_change(changeset, :emoji, fn
:emoji, nil ->
[]
:emoji, emoji ->
if Emoji.unicode?(emoji) or valid_local_custom_emoji?(emoji) do
[]
else
[emoji: "Invalid emoji"]
end
end)
end
defp valid_local_custom_emoji?(emoji) do
with %{file: _path} <- Emoji.get(emoji) do
true
else
_ -> false
end
end
def follow_changeset(list, attrs \\ %{}) do
@ -91,14 +132,14 @@ defmodule Pleroma.List do
|> Repo.all()
end
def rename(%Pleroma.List{} = list, title) do
def update(%Pleroma.List{} = list, params) do
list
|> title_changeset(%{title: title})
|> update_changeset(params)
|> Repo.update()
end
def create(title, %User{} = creator) do
changeset = title_changeset(%Pleroma.List{user_id: creator.id}, %{title: title})
def create(params, %User{} = creator) do
changeset = update_changeset(%Pleroma.List{user_id: creator.id}, params)
if changeset.valid? do
Repo.transaction(fn ->
@ -149,4 +190,14 @@ defmodule Pleroma.List do
end
def member?(_, _), do: false
def get_exclusive_list_members(%User{id: user_id}) do
Pleroma.List
|> where([l], l.user_id == ^user_id)
|> where([l], l.exclusive == true)
|> select([l], l.following)
|> Repo.all()
|> List.flatten()
|> Enum.uniq()
end
end

2
lib/pleroma/marker.ex
View file

@ -78,7 +78,7 @@ defmodule Pleroma.Marker do
defp get_marker(user, timeline) do
case Repo.find_resource(get_query(user, timeline)) do
{:ok, marker} -> %__MODULE__{marker | user: user}
{:ok, %__MODULE__{} = marker} -> %{marker | user: user}
_ -> %__MODULE__{timeline: timeline, user_id: user.id}
end
end

13
lib/pleroma/mfa/changeset.ex
View file

@ -8,7 +8,8 @@ defmodule Pleroma.MFA.Changeset do
alias Pleroma.User
def disable(%Ecto.Changeset{} = changeset, force \\ false) do
settings =
%Settings{} =
settings =
changeset
|> Ecto.Changeset.apply_changes()
|> MFA.fetch_settings()
@ -20,20 +21,20 @@ defmodule Pleroma.MFA.Changeset do
end
end
def disable_totp(%User{multi_factor_authentication_settings: settings} = user) do
def disable_totp(%User{multi_factor_authentication_settings: %Settings{} = settings} = user) do
user
|> put_change(%Settings{settings | totp: %Settings.TOTP{}})
end
def confirm_totp(%User{multi_factor_authentication_settings: settings} = user) do
totp_settings = %Settings.TOTP{settings.totp | confirmed: true}
def confirm_totp(%User{multi_factor_authentication_settings: %Settings{} = settings} = user) do
totp_settings = %Settings.TOTP{(%Settings.TOTP{} = settings.totp) | confirmed: true}
user
|> put_change(%Settings{settings | totp: totp_settings, enabled: true})
end
def setup_totp(%User{} = user, attrs) do
mfa_settings = MFA.fetch_settings(user)
%Settings{} = mfa_settings = MFA.fetch_settings(user)
totp_settings =
%Settings.TOTP{}
@ -46,7 +47,7 @@ defmodule Pleroma.MFA.Changeset do
def cast_backup_codes(%User{} = user, codes) do
user
|> put_change(%Settings{
user.multi_factor_authentication_settings
(%Settings{} = user.multi_factor_authentication_settings)
| backup_codes: codes
})
end

54
lib/pleroma/moderation_log.ex
View file

@ -132,11 +132,18 @@ defmodule Pleroma.ModerationLog do
end
def insert_log(%{actor: %User{}, action: action, subject: %Activity{} = subject} = attrs)
when action in ["report_note_delete", "report_update", "report_note"] do
when action in [
"report_note_delete",
"report_update",
"report_note",
"report_unassigned",
"report_assigned"
] do
data =
attrs
|> prepare_log_data
|> Pleroma.Maps.put_if_present("text", attrs[:text])
|> Pleroma.Maps.put_if_present("assigned_account", attrs[:assigned_account])
|> Map.merge(%{"subject" => report_to_map(subject)})
insert_log_entry_with_message(%ModerationLog{data: data})
@ -339,6 +346,16 @@ defmodule Pleroma.ModerationLog do
"@#{actor_nickname} approved users: #{users_to_nicknames_string(users)}"
end
def get_log_entry_message(%ModerationLog{
data: %{
"actor" => %{"nickname" => actor_nickname},
"action" => "reject",
"subject" => users
}
}) do
"@#{actor_nickname} rejected users: #{users_to_nicknames_string(users)}"
end
def get_log_entry_message(%ModerationLog{
data: %{
"actor" => %{"nickname" => actor_nickname},
@ -441,6 +458,35 @@ defmodule Pleroma.ModerationLog do
" with '#{state}' state"
end
def get_log_entry_message(
%ModerationLog{
data: %{
"actor" => %{"nickname" => actor_nickname},
"action" => "report_assigned",
"subject" => %{"id" => subject_id, "type" => "report"},
"assigned_account" => assigned_account
}
} = log
) do
"@#{actor_nickname} assigned report ##{subject_id}" <>
subject_actor_nickname(log, " (on user ", ")") <>
" to user #{assigned_account}"
end
def get_log_entry_message(
%ModerationLog{
data: %{
"actor" => %{"nickname" => actor_nickname},
"action" => "report_unassigned",
"subject" => %{"id" => subject_id, "type" => "report"}
}
} = log
) do
"@#{actor_nickname} unassigned report ##{subject_id}" <>
subject_actor_nickname(log, " (on user ", ")") <>
" from a user"
end
def get_log_entry_message(
%ModerationLog{
data: %{
@ -575,6 +621,12 @@ defmodule Pleroma.ModerationLog do
"@#{actor_nickname} requested account backup for @#{user_nickname}"
end
def get_log_entry_message(%ModerationLog{data: data}) do
actor_name = get_in(data, ["actor", "nickname"]) || "unknown"
action = data["action"] || "unknown"
"@#{actor_name} performed action #{action}"
end
defp nicknames_to_string(nicknames) do
nicknames
|> Enum.map(&"@#{&1}")

15
lib/pleroma/mogrify_behaviour.ex Normal file
View file

@ -0,0 +1,15 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.MogrifyBehaviour do
@moduledoc """
Behaviour for Mogrify operations.
This module defines the interface for Mogrify operations that can be mocked in tests.
"""
@callback open(binary()) :: map()
@callback custom(map(), binary()) :: map()
@callback custom(map(), binary(), binary()) :: map()
@callback save(map(), keyword()) :: map()
end

30
lib/pleroma/mogrify_wrapper.ex Normal file
View file

@ -0,0 +1,30 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.MogrifyWrapper do
@moduledoc """
Default implementation of MogrifyBehaviour that delegates to Mogrify.
"""
@behaviour Pleroma.MogrifyBehaviour
@impl true
def open(file) do
Mogrify.open(file)
end
@impl true
def custom(image, action) do
Mogrify.custom(image, action)
end
@impl true
def custom(image, action, options) do
Mogrify.custom(image, action, options)
end
@impl true
def save(image, opts) do
Mogrify.save(image, opts)
end
end

119
lib/pleroma/notification.ex
View file

@ -35,6 +35,7 @@ defmodule Pleroma.Notification do
# remember to add a migration to add it to the `notifications_type` enum
# as well.
field(:type, :string)
field(:group_key, :string)
belongs_to(:user, User, type: FlakeId.Ecto.CompatType)
belongs_to(:activity, Activity, type: FlakeId.Ecto.CompatType)
@ -49,7 +50,7 @@ defmodule Pleroma.Notification do
|> type_from_activity()
notification
|> changeset(%{type: type})
|> changeset(%{type: type, group_key: grouped_notification_key(type, activity)})
|> Repo.update()
end
end
@ -62,6 +63,99 @@ defmodule Pleroma.Notification do
|> Repo.aggregate(:count, :id)
end
@groupable_notification_types ~w{favourite follow reblog}
@group_bucket_seconds 12 * 60 * 60
def groupable_notification_types, do: @groupable_notification_types
def group_notifications(notifications, grouped_types \\ nil) do
grouped_types = normalize_grouped_types(grouped_types)
{group_keys, grouped_notifications} =
Enum.reduce(notifications, {[], %{}}, fn notification,
{group_keys, grouped_notifications} ->
group_key = group_key(notification, grouped_types)
if Map.has_key?(grouped_notifications, group_key) do
{group_keys, Map.update!(grouped_notifications, group_key, &[notification | &1])}
else
{[group_key | group_keys], Map.put(grouped_notifications, group_key, [notification])}
end
end)
group_keys
|> Enum.reverse()
|> Enum.map(fn group_key ->
grouped_notifications
|> Map.fetch!(group_key)
|> Enum.reverse()
end)
end
def group_key(notification, grouped_types \\ nil)
def group_key(%Notification{} = notification, nil) do
group_key(notification, @groupable_notification_types)
end
def group_key(%Notification{type: type, group_key: group_key} = notification, grouped_types) do
grouped_types = normalize_grouped_types(grouped_types)
if type in @groupable_notification_types and type in grouped_types and is_binary(group_key) do
group_key
else
ungrouped_group_key(notification)
end
end
def normalize_grouped_types(nil), do: @groupable_notification_types
def normalize_grouped_types(types) when is_list(types), do: types
def normalize_grouped_types(type), do: [type]
defp ungrouped_group_key(%Notification{id: id}), do: "ungrouped-#{id}"
defp grouped_notification_key(type, activity) when type in @groupable_notification_types do
with target_id when is_binary(target_id) <- group_target_id(type, activity) do
# Mastodon uses Redis to reuse a recent bucket for rolling 12h windows. Pleroma keeps keys
# deterministic and self-contained; clients must treat group_key as opaque either way.
"#{type}-#{target_id}-#{group_time_bucket(activity)}"
end
end
defp grouped_notification_key(_type, _activity), do: nil
defp group_time_bucket(%Activity{inserted_at: inserted_at}) when not is_nil(inserted_at) do
inserted_at
|> NaiveDateTime.to_erl()
|> :calendar.datetime_to_gregorian_seconds()
|> div(@group_bucket_seconds)
end
defp group_time_bucket(_),
do: group_time_bucket(%Activity{inserted_at: NaiveDateTime.utc_now()})
defp group_target_id(type, activity) when type in ["favourite", "reblog"] do
with object_id when is_binary(object_id) <- object_id_for(activity),
%Activity{id: id} <- Activity.get_create_by_object_ap_id(object_id) do
to_string(id)
else
_ -> nil
end
end
defp group_target_id("follow", %{data: %{"object" => ap_id}}) do
case User.get_cached_by_ap_id(ap_id) do
%User{id: id} -> to_string(id)
_ -> nil
end
end
defp group_target_id(_, _), do: nil
defp object_id_for(%{data: %{"object" => %{"id" => id}}}) when is_binary(id), do: id
defp object_id_for(%{data: %{"object" => id}}) when is_binary(id), do: id
defp object_id_for(_), do: nil
@notification_types ~w{
favourite
follow
@ -74,11 +168,12 @@ defmodule Pleroma.Notification do
reblog
poll
status
update
}
def changeset(%Notification{} = notification, attrs) do
notification
|> cast(attrs, [:seen, :type])
|> cast(attrs, [:seen, :type, :group_key])
|> validate_inclusion(:type, @notification_types)
end
@ -281,10 +376,15 @@ defmodule Pleroma.Notification do
select: n.id
)
Multi.new()
|> Multi.update_all(:ids, query, set: [seen: true, updated_at: NaiveDateTime.utc_now()])
|> Marker.multi_set_last_read_id(user, "notifications")
|> Repo.transaction()
{:ok, %{marker: marker}} =
Multi.new()
|> Multi.update_all(:ids, query, set: [seen: true, updated_at: NaiveDateTime.utc_now()])
|> Marker.multi_set_last_read_id(user, "notifications")
|> Repo.transaction()
Streamer.stream(["user", "user:notification"], marker)
{:ok, %{marker: marker}}
end
@spec read_one(User.t(), String.t()) ::
@ -452,7 +552,8 @@ defmodule Pleroma.Notification do
user_id: user.id,
activity: activity,
seen: mark_as_read?(activity, user),
type: type
type: type,
group_key: grouped_notification_key(type, activity)
})
|> Marker.multi_set_last_read_id(user, "notifications")
|> Repo.transaction()
@ -525,9 +626,7 @@ defmodule Pleroma.Notification do
%Activity{data: %{"type" => "Create"}} = activity,
local_only
) do
notification_enabled_ap_ids =
[]
|> Utils.maybe_notify_subscribers(activity)
notification_enabled_ap_ids = Utils.get_notified_subscribers(activity)
potential_receivers =
User.get_users_from_set(notification_enabled_ap_ids, local_only: local_only)

45
lib/pleroma/object.ex
View file

@ -126,7 +126,7 @@ defmodule Pleroma.Object do
Logger.debug("Backtrace: #{inspect(Process.info(:erlang.self(), :current_stacktrace))}")
end
def normalize(_, options \\ [fetch: false, id_only: false])
def normalize(_, options \\ [fetch: false])
# If we pass an Activity to Object.normalize(), we can try to use the preloaded object.
# Use this whenever possible, especially when walking graphs in an O(N) loop!
@ -155,9 +155,6 @@ defmodule Pleroma.Object do
def normalize(ap_id, options) when is_binary(ap_id) do
cond do
Keyword.get(options, :id_only) ->
ap_id
Keyword.get(options, :fetch) ->
case Fetcher.fetch_object_from_id(ap_id, options) do
{:ok, object} -> object
@ -375,12 +372,28 @@ defmodule Pleroma.Object do
option
end)
voters = [actor | object.data["voters"] || []] |> Enum.uniq()
existing_voters = object.data["voters"] || []
voters = [actor | existing_voters] |> Enum.uniq()
new_voter? = actor not in existing_voters
existing_voters_count = object.data["votersCount"]
voters_count =
cond do
is_integer(existing_voters_count) and new_voter? ->
existing_voters_count + 1
is_integer(existing_voters_count) ->
existing_voters_count
true ->
length(voters)
end
data =
object.data
|> Map.put(key, options)
|> Map.put("voters", voters)
|> Map.put("votersCount", voters_count)
object
|> Object.change(%{data: data})
@ -401,28 +414,6 @@ defmodule Pleroma.Object do
String.starts_with?(id, Pleroma.Web.Endpoint.url() <> "/")
end
def replies(object, opts \\ []) do
object = Object.normalize(object, fetch: false)
query =
Object
|> where(
[o],
fragment("(?)->>'inReplyTo' = ?", o.data, ^object.data["id"])
)
|> order_by([o], asc: o.id)
if opts[:self_only] do
actor = object.data["actor"]
where(query, [o], fragment("(?)->>'actor' = ?", o.data, ^actor))
else
query
end
end
def self_replies(object, opts \\ []),
do: replies(object, Keyword.put(opts, :self_only, true))
def tags(%Object{data: %{"tag" => tags}}) when is_list(tags), do: tags
def tags(_), do: []

49
lib/pleroma/object/containment.ex
View file

@ -15,12 +15,28 @@ defmodule Pleroma.Object.Containment do
actor
end
def get_actor(%{"actor" => nil, "attributedTo" => actor}) when not is_nil(actor) do
get_actor(%{"actor" => actor})
end
def get_actor(%{"actor" => nil}), do: nil
def get_actor(%{"actor" => []}), do: nil
def get_actor(%{"actor" => actor}) when is_list(actor) do
if is_binary(Enum.at(actor, 0)) do
Enum.at(actor, 0)
else
Enum.find(actor, fn %{"type" => type} -> type in ["Person", "Service", "Application"] end)
|> Map.get("id")
case actor do
[first | _] when is_binary(first) ->
first
_ ->
Enum.find_value(actor, fn
%{"type" => type, "id" => id}
when type in ["Person", "Service", "Application"] and is_binary(id) ->
id
_ ->
nil
end)
end
end
@ -28,10 +44,6 @@ defmodule Pleroma.Object.Containment do
id
end
def get_actor(%{"actor" => nil, "attributedTo" => actor}) when not is_nil(actor) do
get_actor(%{"actor" => actor})
end
def get_object(%{"object" => id}) when is_binary(id) do
id
end
@ -47,6 +59,19 @@ defmodule Pleroma.Object.Containment do
defp compare_uris(%URI{host: host} = _id_uri, %URI{host: host} = _other_uri), do: :ok
defp compare_uris(_id_uri, _other_uri), do: :error
@doc """
Checks whether an URL to fetch from is from the local server.
We never want to fetch from ourselves; if it's not in the database
it can't be authentic and must be a counterfeit.
"""
def contain_local_fetch(id) do
case compare_uris(URI.parse(id), Pleroma.Web.Endpoint.struct_url()) do
:ok -> :error
_ -> :ok
end
end
@doc """
Checks that an imported AP object's actor matches the host it came from.
"""
@ -54,9 +79,11 @@ defmodule Pleroma.Object.Containment do
def contain_origin(id, %{"actor" => _actor} = params) do
id_uri = URI.parse(id)
actor_uri = URI.parse(get_actor(params))
compare_uris(actor_uri, id_uri)
case get_actor(params) do
actor when is_binary(actor) -> compare_uris(URI.parse(actor), id_uri)
_ -> :error
end
end
def contain_origin(id, %{"attributedTo" => actor} = params),

60
lib/pleroma/object/fetcher.ex
View file

@ -4,7 +4,6 @@
defmodule Pleroma.Object.Fetcher do
alias Pleroma.HTTP
alias Pleroma.Instances
alias Pleroma.Maps
alias Pleroma.Object
alias Pleroma.Object.Containment
@ -146,18 +145,18 @@ defmodule Pleroma.Object.Fetcher do
with {:scheme, true} <- {:scheme, String.starts_with?(id, "http")},
{_, true} <- {:mrf, MRF.id_filter(id)},
{_, :ok} <- {:local_fetch, Containment.contain_local_fetch(id)},
{:ok, body} <- get_object(id),
{:ok, data} <- safe_json_decode(body),
:ok <- Containment.contain_origin_from_id(id, data) do
if not Instances.reachable?(id) do
Instances.set_reachable(id)
end
{:ok, data}
else
{:scheme, _} ->
{:error, "Unsupported URI scheme"}
{:local_fetch, _} ->
{:error, "Trying to fetch local resource"}
{:error, e} ->
{:error, e}
@ -172,6 +171,14 @@ defmodule Pleroma.Object.Fetcher do
def fetch_and_contain_remote_object_from_id(_id),
do: {:error, "id must be a string"}
defp check_crossdomain_redirect(final_host, _original_url) when is_nil(final_host) do
{:cross_domain_redirect, false}
end
defp check_crossdomain_redirect(final_host, original_url) do
{:cross_domain_redirect, final_host != URI.parse(original_url).host}
end
defp get_object(id) do
date = Pleroma.Signature.signed_date()
@ -181,19 +188,29 @@ defmodule Pleroma.Object.Fetcher do
|> sign_fetch(id, date)
case HTTP.get(id, headers) do
{:ok, %{body: body, status: code, headers: headers, url: final_url}}
when code in 200..299 ->
remote_host = if final_url, do: URI.parse(final_url).host, else: nil
with {:cross_domain_redirect, false} <- check_crossdomain_redirect(remote_host, id),
{_, content_type} <- List.keyfind(headers, "content-type", 0),
{:ok, _media_type} <- verify_content_type(content_type) do
{:ok, body}
else
{:cross_domain_redirect, true} ->
{:error, {:cross_domain_redirect, true}}
error ->
error
end
# Handle the case where URL is not in the response (older HTTP library versions)
{:ok, %{body: body, status: code, headers: headers}} when code in 200..299 ->
case List.keyfind(headers, "content-type", 0) do
{_, content_type} ->
case Plug.Conn.Utils.media_type(content_type) do
{:ok, "application", "activity+json", _} ->
{:ok, body}
{:ok, "application", "ld+json",
%{"profile" => "https://www.w3.org/ns/activitystreams"}} ->
{:ok, body}
_ ->
{:error, {:content_type, content_type}}
case verify_content_type(content_type) do
{:ok, _} -> {:ok, body}
error -> error
end
_ ->
@ -216,4 +233,17 @@ defmodule Pleroma.Object.Fetcher do
defp safe_json_decode(nil), do: {:ok, nil}
defp safe_json_decode(json), do: Jason.decode(json)
defp verify_content_type(content_type) do
case Plug.Conn.Utils.media_type(content_type) do
{:ok, "application", "activity+json", _} ->
{:ok, :activity_json}
{:ok, "application", "ld+json", %{"profile" => "https://www.w3.org/ns/activitystreams"}} ->
{:ok, :ld_json}
_ ->
{:error, {:content_type, content_type}}
end
end
end

2
lib/pleroma/object/updater.ex
View file

@ -5,6 +5,7 @@
defmodule Pleroma.Object.Updater do
require Pleroma.Constants
alias Pleroma.Maps
alias Pleroma.Object
alias Pleroma.Repo
@ -115,6 +116,7 @@ defmodule Pleroma.Object.Updater do
# Choices are the same, but counts are different
to_be_updated
|> Map.put(key, updated_object[key])
|> Maps.put_if_present("votersCount", updated_object["votersCount"])
else
# Choices (or vote type) have changed, do not allow this
_ -> to_be_updated

28
lib/pleroma/otp_version.ex
View file

@ -1,28 +0,0 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.OTPVersion do
@spec version() :: String.t() | nil
def version do
# OTP Version https://erlang.org/doc/system_principles/versions.html#otp-version
[
Path.join(:code.root_dir(), "OTP_VERSION"),
Path.join([:code.root_dir(), "releases", :erlang.system_info(:otp_release), "OTP_VERSION"])
]
|> get_version_from_files()
end
@spec get_version_from_files([Path.t()]) :: String.t() | nil
def get_version_from_files([]), do: nil
def get_version_from_files([path | paths]) do
if File.exists?(path) do
path
|> File.read!()
|> String.replace(~r/\r|\n|\s/, "")
else
get_version_from_files(paths)
end
end
end

52
lib/pleroma/pagination.ex
View file

@ -89,19 +89,36 @@ defmodule Pleroma.Pagination do
defp cast_params(params) do
param_types = %{
min_id: :string,
since_id: :string,
max_id: :string,
min_id: params[:id_type] || :string,
since_id: params[:id_type] || :string,
max_id: params[:id_type] || :string,
offset: :integer,
limit: :integer,
skip_extra_order: :boolean,
skip_order: :boolean
skip_order: :boolean,
order_asc: :boolean
}
changeset = cast({%{}, param_types}, params, Map.keys(param_types))
changeset.changes
end
defp order_statement(query, table_binding, :asc) do
order_by(
query,
[{u, table_position(query, table_binding)}],
fragment("? asc nulls last", u.id)
)
end
defp order_statement(query, table_binding, :desc) do
order_by(
query,
[{u, table_position(query, table_binding)}],
fragment("? desc nulls last", u.id)
)
end
defp restrict(query, :min_id, %{min_id: min_id}, table_binding) do
where(query, [{q, table_position(query, table_binding)}], q.id > ^min_id)
end
@ -119,19 +136,16 @@ defmodule Pleroma.Pagination do
defp restrict(%{order_bys: [_ | _]} = query, :order, %{skip_extra_order: true}, _), do: query
defp restrict(query, :order, %{min_id: _}, table_binding) do
order_by(
query,
[{u, table_position(query, table_binding)}],
fragment("? asc nulls last", u.id)
)
order_statement(query, table_binding, :asc)
end
defp restrict(query, :order, _options, table_binding) do
order_by(
query,
[{u, table_position(query, table_binding)}],
fragment("? desc nulls last", u.id)
)
defp restrict(query, :order, %{max_id: _}, table_binding) do
order_statement(query, table_binding, :desc)
end
defp restrict(query, :order, options, table_binding) do
dir = if options[:order_asc], do: :asc, else: :desc
order_statement(query, table_binding, dir)
end
defp restrict(query, :offset, %{offset: offset}, _table_binding) do
@ -151,11 +165,9 @@ defmodule Pleroma.Pagination do
defp restrict(query, _, _, _), do: query
defp enforce_order(result, %{min_id: _}) do
result
|> Enum.reverse()
end
defp enforce_order(result, %{min_id: _, order_asc: true}), do: result
defp enforce_order(result, %{min_id: _}), do: Enum.reverse(result)
defp enforce_order(result, %{max_id: _, order_asc: true}), do: Enum.reverse(result)
defp enforce_order(result, _), do: result
defp table_position(%Ecto.Query{} = query, binding_name) do

19
lib/pleroma/release_tasks.ex
View file

@ -5,7 +5,10 @@
defmodule Pleroma.ReleaseTasks do
@repo Pleroma.Repo
def run(args) do
# TODO: Kept for some backwards compatibility with buggy pleroma_ctl,
# if a mismatch between pleroma_ctl and Pleroma accidentaly happens.
# Remove in the future.
def run(args) when is_binary(args) do
[task | args] = String.split(args)
case task do
@ -16,6 +19,20 @@ defmodule Pleroma.ReleaseTasks do
end
end
# HACK: Script arguments need to be received as a list, otherwise (quoted) arguments with
# whitespace will be broken. Previously the broken string form above was used,
# escaping in the shell does not help.
def run(args) when is_list(args) do
[task | args] = args
case task do
"migrate" -> migrate(args)
"create" -> create()
"rollback" -> rollback(args)
task -> mix_task(task, args)
end
end
def find_module(task) do
module_name =
task

236
lib/pleroma/reverse_proxy.ex
View file

@ -3,6 +3,8 @@
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.ReverseProxy do
alias Pleroma.Utils.URIEncoding
@range_headers ~w(range if-range)
@keep_req_headers ~w(accept accept-encoding cache-control if-modified-since) ++
~w(if-unmodified-since if-none-match) ++ @range_headers
@ -10,13 +12,16 @@ defmodule Pleroma.ReverseProxy do
@keep_resp_headers @resp_cache_headers ++
~w(content-length content-type content-disposition content-encoding) ++
~w(content-range accept-ranges vary)
@default_cache_control_header "public, max-age=1209600"
@default_cache_control_header "public, max-age=1209600, immutable"
@valid_resp_codes [200, 206, 304]
@max_read_duration :timer.seconds(30)
@max_body_length :infinity
@failed_request_ttl :timer.seconds(60)
@sniff_bytes 8 * 1024
@methods ~w(GET HEAD)
@allowed_mime_types Pleroma.Config.get([Pleroma.Upload, :allowed_mime_types], [])
@cachex Pleroma.Config.get([:cachex, :provider], Cachex)
def max_read_duration_default, do: @max_read_duration
@ -55,7 +60,12 @@ defmodule Pleroma.ReverseProxy do
* `inline_content_types`:
* `true` will not alter `content-disposition` (up to the upstream),
* `false` will add `content-disposition: attachment` to any request,
* a list of whitelisted content types
* a list of whitelisted content types for which `content-disposition: inline`
is always set (overriding any upstream header) so the media can be embedded
in pages; the filename is derived from the content type
* `sniff_content_type` (default `false`): detects an image MIME type from the first
response chunk when the upstream type is missing or `application/octet-stream`.
* `req_headers`, `resp_headers` additional headers.
@ -65,11 +75,15 @@ defmodule Pleroma.ReverseProxy do
@default_options [pool: :media]
@inline_content_types [
"image/apng",
"image/avif",
"image/bmp",
"image/gif",
"image/jpeg",
"image/jpg",
"image/png",
"image/svg+xml",
"image/webp",
"audio/mpeg",
"audio/mp3",
"video/webm",
@ -88,6 +102,7 @@ defmodule Pleroma.ReverseProxy do
| {:req_headers, [{String.t(), String.t()}]}
| {:resp_headers, [{String.t(), String.t()}]}
| {:inline_content_types, boolean() | list(String.t())}
| {:sniff_content_type, boolean()}
| {:redirect_on_failure, boolean()}
@spec call(Plug.Conn.t(), String.t(), list(option())) :: Plug.Conn.t()
@ -106,12 +121,8 @@ defmodule Pleroma.ReverseProxy do
end
with {:ok, nil} <- @cachex.get(:failed_proxy_url_cache, url),
{:ok, code, headers, client} <- request(method, url, req_headers, client_opts),
:ok <-
header_length_constraint(
headers,
Keyword.get(opts, :max_body_length, @max_body_length)
) do
{:ok, code, headers, client} <-
request_with_constraints(method, url, req_headers, client_opts, opts) do
response(conn, client, url, code, headers, opts)
else
{:ok, true} ->
@ -153,9 +164,12 @@ defmodule Pleroma.ReverseProxy do
end
defp request(method, url, headers, opts) do
Logger.debug("#{__MODULE__} #{method} #{url} #{inspect(headers)}")
method = method |> String.downcase() |> String.to_existing_atom()
url = maybe_encode_url(url)
Logger.debug("#{__MODULE__} #{method} #{url} #{inspect(headers)}")
case client().request(method, url, headers, "", opts) do
{:ok, code, headers, client} when code in @valid_resp_codes ->
{:ok, code, downcase_headers(headers), client}
@ -163,7 +177,8 @@ defmodule Pleroma.ReverseProxy do
{:ok, code, headers} when code in @valid_resp_codes ->
{:ok, code, downcase_headers(headers)}
{:ok, code, _, _} ->
{:ok, code, _, client} ->
client().close(client)
{:error, {:invalid_http_response, code}}
{:ok, code, _} ->
@ -174,15 +189,37 @@ defmodule Pleroma.ReverseProxy do
end
end
defp request_with_constraints(method, url, headers, client_opts, opts) do
case request(method, url, headers, client_opts) do
{:ok, _code, headers, client} = response ->
case header_length_constraint(
headers,
Keyword.get(opts, :max_body_length, @max_body_length)
) do
:ok ->
response
error ->
client().close(client)
error
end
response ->
response
end
end
defp response(conn, client, url, status, headers, opts) do
Logger.debug("#{__MODULE__} #{status} #{url} #{inspect(headers)}")
{headers, client, prefetched} = maybe_prefetch_for_content_type(headers, client, opts)
result =
conn
|> put_resp_headers(build_resp_headers(headers, opts))
|> streaming_compat
|> send_chunked(status)
|> chunk_reply(client, opts)
|> chunk_reply(client, opts, prefetched)
case result do
{:ok, conn} ->
@ -202,18 +239,28 @@ defmodule Pleroma.ReverseProxy do
end
end
defp chunk_reply(conn, client, opts) do
defp chunk_reply(conn, client, opts, :none) do
chunk_reply(conn, client, opts, 0, 0)
end
defp chunk_reply(conn, client, opts, sent_so_far, duration) do
with {:ok, duration} <-
check_read_duration(
duration,
Keyword.get(opts, :max_read_duration, @max_read_duration)
defp chunk_reply(conn, _client, _opts, :done), do: {:ok, conn}
defp chunk_reply(conn, _client, _opts, {:error, error}), do: {:error, error, conn}
defp chunk_reply(conn, client, opts, {:ok, data, duration}) do
with :ok <-
body_size_constraint(
byte_size(data),
Keyword.get(opts, :max_body_length, @max_body_length)
),
{:ok, data, client} <- client().stream_body(client),
{:ok, duration} <- increase_read_duration(duration),
{:ok, conn} <- chunk(conn, data) do
chunk_reply(conn, client, opts, byte_size(data), duration)
else
{:error, error} -> {:error, error, conn}
end
end
defp chunk_reply(conn, client, opts, sent_so_far, duration) do
with {:ok, data, client, duration} <- read_chunk(client, duration, opts),
sent_so_far = sent_so_far + byte_size(data),
:ok <-
body_size_constraint(
@ -228,6 +275,66 @@ defmodule Pleroma.ReverseProxy do
end
end
defp read_chunk(client, duration, opts) do
with {:ok, timer} <-
check_read_duration(
duration,
Keyword.get(opts, :max_read_duration, @max_read_duration)
),
{:ok, data, client} <- client().stream_body(client),
{:ok, duration} <- increase_read_duration(timer) do
{:ok, data, client, duration}
end
end
defp maybe_prefetch_for_content_type(headers, client, opts) do
if Keyword.get(opts, :sniff_content_type, false) and generic_content_type?(headers) do
case read_chunk(client, 0, opts) do
{:ok, data, client, duration} ->
{maybe_put_image_content_type(headers, data), client, {:ok, data, duration}}
:done ->
{headers, client, :done}
{:error, error} ->
{headers, client, {:error, error}}
end
else
{headers, client, :none}
end
end
defp generic_content_type?(headers) do
headers
|> get_content_type()
|> String.trim()
|> String.downcase()
|> then(&(&1 in ["", "application/octet-stream"]))
end
defp maybe_put_image_content_type(headers, data) do
with false <- data == "",
{:ok, %{mime_type: "image/" <> _ = content_type}} <-
Majic.perform({:bytes, binary_part(data, 0, min(byte_size(data), @sniff_bytes))},
pool: Pleroma.MajicPool
) do
[
{"content-type", content_type}
| Enum.reject(headers, fn {key, _} -> key == "content-type" end)
]
else
_ -> headers
end
rescue
error ->
Logger.debug("#{__MODULE__}: content-type sniffing failed: #{Exception.message(error)}")
headers
catch
kind, reason ->
Logger.debug("#{__MODULE__}: content-type sniffing failed: #{kind}: #{inspect(reason)}")
headers
end
defp head_response(conn, url, code, headers, opts) do
Logger.debug("#{__MODULE__} #{code} #{url} #{inspect(headers)}")
@ -274,17 +381,34 @@ defmodule Pleroma.ReverseProxy do
|> Enum.filter(fn {k, _} -> k in @keep_req_headers end)
|> build_req_range_or_encoding_header(opts)
|> build_req_user_agent_header(opts)
|> Keyword.merge(Keyword.get(opts, :req_headers, []))
|> merge_headers(Keyword.get(opts, :req_headers, []))
|> maybe_force_identity_encoding(opts)
end
defp merge_headers(headers, extra_headers) do
Enum.reduce(extra_headers, headers, fn {key, value}, headers ->
List.keystore(headers, String.downcase(key), 0, {String.downcase(key), value})
end)
end
defp maybe_force_identity_encoding(headers, opts) do
if Keyword.get(opts, :sniff_content_type, false) do
List.keystore(headers, "accept-encoding", 0, {"accept-encoding", "identity"})
else
headers
end
end
# Disable content-encoding if any @range_headers are requested (see #1823).
defp build_req_range_or_encoding_header(headers, _opts) do
range? = Enum.any?(headers, fn {header, _} -> Enum.member?(@range_headers, header) end)
if range? && List.keymember?(headers, "accept-encoding", 0) do
List.keydelete(headers, "accept-encoding", 0)
else
headers
cond do
range? && List.keymember?(headers, "accept-encoding", 0) ->
List.keydelete(headers, "accept-encoding", 0)
true ->
headers
end
end
@ -301,10 +425,26 @@ defmodule Pleroma.ReverseProxy do
headers
|> Enum.filter(fn {k, _} -> k in @keep_resp_headers end)
|> build_resp_cache_headers(opts)
|> sanitise_content_type()
|> build_resp_content_disposition_header(opts)
|> Keyword.merge(Keyword.get(opts, :resp_headers, []))
end
defp sanitise_content_type(headers) do
original_ct = get_content_type(headers)
safe_ct =
Pleroma.Web.Plugs.Utils.get_safe_mime_type(
%{allowed_mime_types: @allowed_mime_types},
original_ct
)
[
{"content-type", safe_ct}
| Enum.filter(headers, fn {k, _v} -> k != "content-type" end)
]
end
defp build_resp_cache_headers(headers, _opts) do
has_cache? = Enum.any?(headers, fn {k, _} -> k in @resp_cache_headers end)
@ -361,9 +501,33 @@ defmodule Pleroma.ReverseProxy do
disposition = "attachment; filename=\"#{name}\""
List.keystore(headers, "content-disposition", 0, {"content-disposition", disposition})
replace_header(headers, "content-disposition", disposition)
else
headers
if opt == true do
headers
else
name = inline_filename(content_type)
disposition =
if name do
"inline; filename=\"#{name}\""
else
"inline"
end
replace_header(headers, "content-disposition", disposition)
end
end
end
defp replace_header(headers, key, value) do
[{key, value} | Enum.reject(headers, fn {header, _} -> header == key end)]
end
defp inline_filename(content_type) do
case MIME.extensions(content_type) do
[ext | _] when ext != "" -> "inline.#{ext}"
_ -> nil
end
end
@ -383,7 +547,7 @@ defmodule Pleroma.ReverseProxy do
defp header_length_constraint(_, _), do: :ok
defp body_size_constraint(size, limit) when is_integer(limit) and limit > 0 and size >= limit do
defp body_size_constraint(size, limit) when is_integer(limit) and limit > 0 and size > limit do
{:error, :body_too_large}
end
@ -398,7 +562,9 @@ defmodule Pleroma.ReverseProxy do
end
end
defp check_read_duration(_, _), do: {:ok, :no_duration_limit, :no_duration_limit}
defp check_read_duration(_, _), do: {:ok, :no_duration_limit}
defp increase_read_duration(:no_duration_limit), do: {:ok, :no_duration_limit}
defp increase_read_duration({previous_duration, started})
when is_integer(previous_duration) and is_integer(started) do
@ -431,4 +597,18 @@ defmodule Pleroma.ReverseProxy do
_ -> delete_resp_header(conn, "content-length")
end
end
# Only when Tesla adapter is Hackney or Finch does the URL
# need encoding before Reverse Proxying as both end up
# using the raw Hackney client and cannot leverage our
# EncodeUrl Tesla middleware
# Also do it for test environment
defp maybe_encode_url(url) do
case Application.get_env(:tesla, :adapter) do
Tesla.Adapter.Hackney -> URIEncoding.encode_url(url)
{Tesla.Adapter.Finch, _} -> URIEncoding.encode_url(url)
Tesla.Mock -> URIEncoding.encode_url(url)
_ -> url
end
end
end

72
lib/pleroma/reverse_proxy/client/hackney.ex
View file

@ -4,10 +4,52 @@
defmodule Pleroma.ReverseProxy.Client.Hackney do
@behaviour Pleroma.ReverseProxy.Client
@redirect_limit 5
require Logger
# In-app redirect handler to avoid Hackney redirect bugs:
# - https://github.com/benoitc/hackney/issues/527 (relative/protocol-less redirects can crash Hackney)
# - https://github.com/benoitc/hackney/issues/273 (redirects not followed when using HTTP proxy)
#
# Based on a redirect handler from Pleb, slightly modified to work with Hackney:
# https://declin.eu/objects/d4f38e62-5429-4614-86d1-e8fc16e6bf33
@redirect_statuses [301, 302, 303, 307, 308]
defp absolute_redirect_url(original_url, resp_headers) do
location =
Enum.find(resp_headers, fn {header, _location} ->
String.downcase(header) == "location"
end)
URI.merge(original_url, elem(location, 1))
|> URI.to_string()
end
@impl true
def request(method, url, headers, body, opts \\ []) do
:hackney.request(method, url, headers, body, opts)
opts =
Keyword.put_new(opts, :path_encode_fun, fn path ->
path
end)
if opts[:follow_redirect] != false do
{_state, req_opts} = Access.get_and_update(opts, :follow_redirect, fn a -> {a, false} end)
env = %{method: method, headers: headers, body: body, req_opts: req_opts}
res = :hackney.request(method, url, headers, body, req_opts)
case res do
{:ok, code, resp_headers, _client} when code in @redirect_statuses ->
redirect(url, resp_headers, env, @redirect_limit)
{:ok, code, resp_headers} when code in @redirect_statuses ->
redirect(url, resp_headers, env, @redirect_limit)
_ ->
res
end
else
:hackney.request(method, url, headers, body, opts)
end
end
@impl true
@ -21,4 +63,32 @@ defmodule Pleroma.ReverseProxy.Client.Hackney do
@impl true
def close(ref), do: :hackney.close(ref)
defp redirect(url, resp_headers, env, limit) when limit == 0 do
new_url = absolute_redirect_url(url, resp_headers)
Logger.debug(
"#{__MODULE__}: Handling redirect #{url} -> #{new_url}; redirect limit was reached - returning response after final redirect"
)
:hackney.request(env.method, new_url, env.headers, env.body, env.req_opts)
end
defp redirect(url, resp_headers, env, limit) do
new_url = absolute_redirect_url(url, resp_headers)
Logger.debug("#{__MODULE__}: handling redirect #{url} -> #{new_url}; limit = #{limit}")
res = :hackney.request(env.method, new_url, env.headers, env.body, env.req_opts)
case res do
{:ok, code, new_resp_headers, _client} when code in @redirect_statuses ->
redirect(new_url, new_resp_headers, env, limit - 1)
{:ok, code, new_resp_headers} when code in @redirect_statuses ->
redirect(new_url, new_resp_headers, env, limit - 1)
_ ->
res
end
end
end

16
lib/pleroma/reverse_proxy/client/tesla.ex
View file

@ -6,6 +6,7 @@ defmodule Pleroma.ReverseProxy.Client.Tesla do
@behaviour Pleroma.ReverseProxy.Client
alias Pleroma.Gun.ConnectionPool
alias Pleroma.HTTP.AdapterHelper
@type headers() :: [{String.t(), String.t()}]
@type status() :: pos_integer()
@ -20,6 +21,13 @@ defmodule Pleroma.ReverseProxy.Client.Tesla do
def request(method, url, headers, body, opts \\ []) do
check_adapter()
# MediaProxyController inserts MediaProxy proxy configuration into the ReverseProxy call.
# This config can be unformatted and in String representation, while this works for Hackney,
# it does not work for Tesla and the proxy needs to be reformatted.
# Gun AdapterHelper doesn't do it, since it does not overwrite proxy config.
proxy = AdapterHelper.format_proxy(opts[:proxy])
opts = if proxy, do: Keyword.put(opts, :proxy, proxy), else: opts
opts = Keyword.put(opts, :body_as, :chunks)
with {:ok, response} <-
@ -45,8 +53,8 @@ defmodule Pleroma.ReverseProxy.Client.Tesla do
@impl true
@spec stream_body(map()) ::
{:ok, binary(), map()} | {:error, atom() | String.t()} | :done | no_return()
def stream_body(%{pid: pid, fin: true}) do
ConnectionPool.release_conn(pid)
def stream_body(%{pid: pid, stream: stream, fin: true}) do
ConnectionPool.release_stream(pid, stream)
:done
end
@ -70,6 +78,10 @@ defmodule Pleroma.ReverseProxy.Client.Tesla do
@impl true
@spec close(map) :: :ok | no_return()
def close(%{pid: pid, stream: stream}) do
ConnectionPool.cancel_stream(pid, stream)
end
def close(%{pid: pid}) do
ConnectionPool.release_conn(pid)
end

212
lib/pleroma/safe_zip.ex Normal file
View file

@ -0,0 +1,212 @@
# Akkoma: Magically expressive social media
# Copyright © 2024 Akkoma Authors <https://akkoma.dev/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.SafeZip do
@moduledoc """
Wraps the subset of Erlang's zip module we’d like to use
but enforces path-traversal safety everywhere and other checks.
For convenience almost all functions accept both elixir strings and charlists,
but output elixir strings themselves. However, this means the input parameter type
can no longer be used to distinguish archive file paths from archive binary data in memory,
thus where needed both a _data and _file variant are provided.
"""
@type text() :: String.t() | [char()]
defp safe_path?(path) do
# Path accepts elixir’s chardata()
case Path.safe_relative(path) do
{:ok, _} -> true
_ -> false
end
end
defp safe_type?(file_type) do
if file_type in [:regular, :directory] do
true
else
false
end
end
defp maybe_add_file(_type, _path_charlist, nil), do: nil
defp maybe_add_file(:regular, path_charlist, file_list),
do: [to_string(path_charlist) | file_list]
defp maybe_add_file(_type, _path_charlist, file_list), do: file_list
@spec check_safe_archive_and_maybe_list_files(binary() | [char()], [term()], boolean()) ::
{:ok, [String.t()]} | {:error, reason :: term()}
defp check_safe_archive_and_maybe_list_files(archive, opts, list) do
acc = if list, do: [], else: nil
with {:ok, table} <- :zip.table(archive, opts) do
Enum.reduce_while(table, {:ok, acc}, fn
# ZIP comment
{:zip_comment, _}, acc ->
{:cont, acc}
# File entry
{:zip_file, path, info, _comment, _offset, _comp_size}, {:ok, fl} ->
with {_, type} <- {:get_type, elem(info, 2)},
{_, true} <- {:type, safe_type?(type)},
{_, true} <- {:safe_path, safe_path?(path)} do
{:cont, {:ok, maybe_add_file(type, path, fl)}}
else
{:type, _} ->
{:halt, {:error, "Potentially unsafe file type in ZIP at: #{path}"}}
{:safe_path, _} ->
{:halt, {:error, "Unsafe path in ZIP: #{path}"}}
end
# new OTP version?
_, _acc ->
{:halt, {:error, "Unknown ZIP record type"}}
end)
end
end
@spec check_safe_archive_and_list_files(binary() | [char()], [term()]) ::
{:ok, [String.t()]} | {:error, reason :: term()}
defp check_safe_archive_and_list_files(archive, opts \\ []) do
check_safe_archive_and_maybe_list_files(archive, opts, true)
end
@spec check_safe_archive(binary() | [char()], [term()]) :: :ok | {:error, reason :: term()}
defp check_safe_archive(archive, opts \\ []) do
case check_safe_archive_and_maybe_list_files(archive, opts, false) do
{:ok, _} -> :ok
error -> error
end
end
@spec check_safe_file_list([text()], text()) :: :ok | {:error, term()}
defp check_safe_file_list([], _), do: :ok
defp check_safe_file_list([path | tail], cwd) do
with {_, true} <- {:path, safe_path?(path)},
{_, {:ok, fstat}} <- {:stat, File.stat(Path.expand(path, cwd))},
{_, true} <- {:type, safe_type?(fstat.type)} do
check_safe_file_list(tail, cwd)
else
{:path, _} ->
{:error, "Unsafe path escaping cwd: #{path}"}
{:stat, e} ->
{:error, "Unable to check file type of #{path}: #{inspect(e)}"}
{:type, _} ->
{:error, "Unsafe type at #{path}"}
end
end
defp check_safe_file_list(_, _), do: {:error, "Malformed file_list"}
@doc """
Checks whether the archive data contais file entries for all paths from fset
Note this really only accepts entries corresponding to regular _files_,
if a path is contained as for example an directory, this does not count as a match.
"""
@spec contains_all_data?(binary(), MapSet.t()) :: true | false
def contains_all_data?(archive_data, fset) do
with {:ok, table} <- :zip.table(archive_data) do
remaining =
Enum.reduce(table, fset, fn
{:zip_file, path, info, _comment, _offset, _comp_size}, fset ->
if elem(info, 2) == :regular do
MapSet.delete(fset, path)
else
fset
end
_, _ ->
fset
end)
|> MapSet.size()
if remaining == 0, do: true, else: false
else
_ -> false
end
end
@doc """
List all file entries in ZIP, or error if invalid or unsafe.
Note this really only lists regular files, no directories, ZIP comments or other types!
"""
@spec list_dir_file(text()) :: {:ok, [String.t()]} | {:error, reason :: term()}
def list_dir_file(archive) do
path = to_charlist(archive)
check_safe_archive_and_list_files(path)
end
defp stringify_zip({:ok, {fname, data}}), do: {:ok, {to_string(fname), data}}
defp stringify_zip({:ok, fname}), do: {:ok, to_string(fname)}
defp stringify_zip(ret), do: ret
@spec zip(text(), text(), [text()], boolean()) ::
{:ok, file_name :: String.t()}
| {:ok, {file_name :: String.t(), file_data :: binary()}}
| {:error, reason :: term()}
def zip(name, file_list, cwd, memory \\ false) do
opts = [{:cwd, to_charlist(cwd)}]
opts = if memory, do: [:memory | opts], else: opts
with :ok <- check_safe_file_list(file_list, cwd) do
file_list = for f <- file_list, do: to_charlist(f)
name = to_charlist(name)
stringify_zip(:zip.zip(name, file_list, opts))
end
end
@spec unzip_file(text(), text(), [text()] | nil) ::
{:ok, [String.t()]}
| {:error, reason :: term()}
| {:error, {name :: text(), reason :: term()}}
def unzip_file(archive, target_dir, file_list \\ nil) do
do_unzip(to_charlist(archive), to_charlist(target_dir), file_list)
end
@spec unzip_data(binary(), text(), [text()] | nil) ::
{:ok, [String.t()]}
| {:error, reason :: term()}
| {:error, {name :: text(), reason :: term()}}
def unzip_data(archive, target_dir, file_list \\ nil) do
do_unzip(archive, to_charlist(target_dir), file_list)
end
defp stringify_unzip({:ok, [{_fname, _data} | _] = filebinlist}),
do: {:ok, Enum.map(filebinlist, fn {fname, data} -> {to_string(fname), data} end)}
defp stringify_unzip({:ok, [_fname | _] = filelist}),
do: {:ok, Enum.map(filelist, fn fname -> to_string(fname) end)}
defp stringify_unzip({:error, {fname, term}}), do: {:error, {to_string(fname), term}}
defp stringify_unzip(ret), do: ret
@spec do_unzip(binary() | [char()], text(), [text()] | nil) ::
{:ok, [String.t()]}
| {:error, reason :: term()}
| {:error, {name :: text(), reason :: term()}}
defp do_unzip(archive, target_dir, file_list) do
opts =
if file_list != nil do
[
file_list: for(f <- file_list, do: to_charlist(f)),
cwd: target_dir
]
else
[cwd: target_dir]
end
with :ok <- check_safe_archive(archive) do
stringify_unzip(:zip.unzip(archive, opts))
end
end
end

82
lib/pleroma/search.ex
View file

@ -1,11 +1,27 @@
defmodule Pleroma.Search do
alias Pleroma.Activity
alias Pleroma.Object
alias Pleroma.Web.ActivityPub.Visibility
alias Pleroma.Workers.SearchIndexingWorker
def add_to_index(%Pleroma.Activity{id: activity_id}) do
SearchIndexingWorker.new(%{"op" => "add_to_index", "activity" => activity_id})
|> Oban.insert()
@spec add_to_index(Activity.t()) :: {:ok, Oban.Job.t() | :noop} | {:error, Oban.Job.changeset()}
def add_to_index(%Activity{id: activity_id, object: %Object{}} = activity) do
if indexable?(activity) do
SearchIndexingWorker.new(%{"op" => "add_to_index", "activity" => activity_id})
|> Oban.insert()
else
{:ok, :noop}
end
end
def add_to_index(%Activity{id: activity_id}) do
case Activity.get_by_id_with_object(activity_id) do
%Activity{} = preloaded -> add_to_index(preloaded)
_ -> {:ok, :noop}
end
end
@spec remove_from_index(Object.t()) :: {:ok, Oban.Job.t()} | {:error, Oban.Job.changeset()}
def remove_from_index(%Pleroma.Object{id: object_id}) do
SearchIndexingWorker.new(%{"op" => "remove_from_index", "object" => object_id})
|> Oban.insert()
@ -20,4 +36,64 @@ defmodule Pleroma.Search do
search_module = Pleroma.Config.get([Pleroma.Search, :module])
search_module.healthcheck_endpoints()
end
def object_to_search_data(%Object{} = object) do
data = object.data
content =
data
|> search_texts()
|> Enum.map(&sanitize_search_text/1)
|> Enum.reject(&(&1 in ["", "."]))
|> Enum.join(" ")
with true <- content != "",
published when is_binary(published) <- data["published"],
{:ok, published, _} <- DateTime.from_iso8601(published) do
%{
id: object.id,
content: content,
ap: data["id"],
published: published |> DateTime.to_unix()
}
else
_ -> nil
end
end
def indexable?(%Activity{
data: %{"type" => "Create"},
object: %Object{data: %{"published" => published, "type" => "Note"}} = object
})
when not is_nil(published) do
Visibility.get_visibility(object) in ["public", "unlisted"] and
not is_nil(object_to_search_data(object))
end
def indexable?(_), do: false
defp search_texts(data) do
[data["content"], data["summary"] | attachment_names(data["attachment"])]
end
defp attachment_names(attachments) when is_list(attachments) do
Enum.map(attachments, fn
%{"summary" => summary} when is_binary(summary) -> summary
%{"name" => name} -> name
_ -> nil
end)
end
defp attachment_names(_), do: []
defp sanitize_search_text([nil | rest]), do: sanitize_search_text(to_string(rest))
defp sanitize_search_text(text) when is_binary(text) do
case FastSanitize.Sanitizer.scrub(text, Pleroma.HTML.Scrubber.SearchIndexing) do
{:ok, scrubbed} -> String.trim(scrubbed)
_ -> ""
end
end
defp sanitize_search_text(_), do: ""
end

137
lib/pleroma/search/backfill.ex Normal file
View file

@ -0,0 +1,137 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2026 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Search.Backfill do
import Ecto.Query
alias Pleroma.Activity
alias Pleroma.Workers.SearchIndexingWorker
@default_chunk_size 100
@default_step_size 1_000
@default_limit 100_000
@doc """
Enqueues existing Create activities for search indexing.
The returned `next_cursor` can be passed as `before` to resume a bounded run.
Pass `limit: :infinity` to continue until the oldest activity is reached.
"""
def run(options \\ []) do
options =
Keyword.validate!(options,
chunk: @default_chunk_size,
step: @default_step_size,
limit: @default_limit,
before: nil,
on_page: nil
)
chunk_size = positive_option!(options, :chunk, @default_chunk_size)
step_size = positive_option!(options, :step, @default_step_size)
limit = limit_option!(options)
before = cursor_option!(options[:before])
on_page = callback_option!(options[:on_page])
do_run(before, limit, step_size, chunk_size, on_page, 0)
end
defp do_run(before, remaining, step_size, chunk_size, on_page, enqueued) do
page_size = page_size(remaining, step_size)
ids = fetch_ids(before, page_size)
case ids do
[] ->
result(enqueued, before, true)
ids ->
enqueue(ids, chunk_size)
page_count = length(ids)
next_cursor = List.last(ids)
total = enqueued + page_count
remaining = decrement(remaining, page_count)
on_page.(%{count: page_count, enqueued: total, next_cursor: next_cursor})
cond do
remaining == 0 -> result(total, next_cursor, false)
page_count < page_size -> result(total, next_cursor, true)
true -> do_run(next_cursor, remaining, step_size, chunk_size, on_page, total)
end
end
end
defp fetch_ids(before, limit) do
Activity
|> where([a], fragment("?->>'type' = 'Create'", a.data))
|> maybe_before(before)
|> order_by([a], desc: a.id)
|> limit(^limit)
|> select([a], a.id)
|> Pleroma.Repo.all()
end
defp maybe_before(query, nil), do: query
defp maybe_before(query, before), do: where(query, [a], a.id < ^before)
defp enqueue(ids, chunk_size) do
ids
|> Enum.chunk_every(chunk_size)
|> Enum.each(fn chunk ->
chunk
|> Enum.map(&SearchIndexingWorker.new(%{"op" => "add_to_index", "activity" => &1}))
|> Oban.insert_all()
end)
end
defp page_size(:infinity, step_size), do: step_size
defp page_size(remaining, step_size), do: min(remaining, step_size)
defp decrement(:infinity, _count), do: :infinity
defp decrement(remaining, count), do: remaining - count
defp result(enqueued, next_cursor, exhausted) do
%{enqueued: enqueued, next_cursor: next_cursor, exhausted: exhausted}
end
defp positive_option!(options, key, default) do
case Keyword.get(options, key, default) do
value when is_integer(value) and value > 0 -> value
value -> raise ArgumentError, "#{key} must be a positive integer, got: #{inspect(value)}"
end
end
defp limit_option!(options) do
case Keyword.fetch!(options, :limit) do
:infinity -> :infinity
value when is_integer(value) and value > 0 -> value
value -> raise ArgumentError, "limit must be a positive integer, got: #{inspect(value)}"
end
end
defp cursor_option!(nil), do: nil
defp cursor_option!(cursor) do
valid? =
try do
match?({:ok, <<_::binary-size(16)>>}, FlakeId.Ecto.CompatType.dump(cursor))
rescue
_ -> false
end
if valid? do
cursor
else
raise ArgumentError, "before must be a valid activity id, got: #{inspect(cursor)}"
end
end
defp callback_option!(nil), do: fn _ -> :ok end
defp callback_option!(callback) when is_function(callback, 1), do: callback
defp callback_option!(callback) do
raise ArgumentError, "on_page must be a function of arity 1, got: #{inspect(callback)}"
end
end

3
lib/pleroma/search/database_search.ex
View file

@ -102,7 +102,8 @@ defmodule Pleroma.Search.DatabaseSearch do
^tsc,
o.data,
^search_query
)
),
order_by: [desc: :inserted_at]
)
end

72
lib/pleroma/search/meilisearch.ex
View file

@ -4,6 +4,8 @@ defmodule Pleroma.Search.Meilisearch do
alias Pleroma.Activity
alias Pleroma.Config.Getting, as: Config
alias Pleroma.Object
alias Pleroma.Search
import Pleroma.Search.DatabaseSearch
import Ecto.Query
@ -118,66 +120,24 @@ defmodule Pleroma.Search.Meilisearch do
end
end
def object_to_search_data(object) do
# Only index public or unlisted Notes
if not is_nil(object) and object.data["type"] == "Note" and
not is_nil(object.data["content"]) and
not is_nil(object.data["published"]) and
(Pleroma.Constants.as_public() in object.data["to"] or
Pleroma.Constants.as_public() in object.data["cc"]) and
object.data["content"] not in ["", "."] do
data = object.data
content_str =
case data["content"] do
[nil | rest] -> to_string(rest)
str -> str
end
content =
with {:ok, scrubbed} <-
FastSanitize.Sanitizer.scrub(content_str, Pleroma.HTML.Scrubber.SearchIndexing),
trimmed <- String.trim(scrubbed) do
trimmed
end
# Make sure we have a non-empty string
if content != "" do
{:ok, published, _} = DateTime.from_iso8601(data["published"])
%{
id: object.id,
content: content,
ap: data["id"],
published: published |> DateTime.to_unix()
}
end
end
end
@impl true
def add_to_index(activity) do
maybe_search_data = object_to_search_data(activity.object)
def add_to_index(%Activity{object: %Object{} = object} = activity) do
search_data = Search.object_to_search_data(object)
if activity.data["type"] == "Create" and maybe_search_data do
result =
meili_put(
"/indexes/objects/documents",
[maybe_search_data]
)
result =
meili_put(
"/indexes/objects/documents",
[search_data]
)
with {:ok, %{"status" => "enqueued"}} <- result do
# Added successfully
:ok
else
_ ->
# There was an error, report it
Logger.error("Failed to add activity #{activity.id} to index: #{inspect(result)}")
{:error, result}
end
else
# The post isn't something we can search, that's ok
with {:ok, %{"status" => "enqueued"}} <- result do
# Added successfully
:ok
else
_ ->
# There was an error, report it
Logger.error("Failed to add activity #{activity.id} to index: #{inspect(result)}")
{:error, result}
end
end

287
lib/pleroma/search/parade_db.ex Normal file
View file

@ -0,0 +1,287 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Search.ParadeDB do
@behaviour Pleroma.Search.SearchBackend
require Logger
require Pleroma.Constants
import Ecto.Query
import Pleroma.Search.DatabaseSearch,
only: [
maybe_fetch: 3,
maybe_restrict_author: 2,
maybe_restrict_blocked: 2,
maybe_restrict_local: 2
]
alias Pleroma.Activity
alias Pleroma.Config.Getting, as: Config
alias Pleroma.Search
@max_limit 40
@default_table "pleroma_search_documents"
defp client_impl do
Config.get([Pleroma.Search.ParadeDB, :client_impl]) ||
Pleroma.Search.ParadeDB.Client.Postgres
end
defp table do
Config.get([Pleroma.Search.ParadeDB, :table], @default_table)
end
defp fuzzy_distance do
case Config.get([Pleroma.Search.ParadeDB, :fuzzy_distance], 0) do
value when is_integer(value) and value in 0..2 ->
value
value when is_binary(value) ->
case Integer.parse(value) do
{parsed, ""} when parsed in 0..2 -> parsed
_ -> 0
end
_ ->
0
end
end
@impl true
def create_index do
with {:ok, _} <- client_impl().query("CREATE EXTENSION IF NOT EXISTS pg_search", []),
{:ok, _} <- client_impl().query(create_table_sql(table()), []),
{:ok, _} <- client_impl().query(create_object_id_index_sql(table()), []),
{:ok, _} <- client_impl().query(create_bm25_index_sql(table()), []) do
:ok
else
{:error, error} ->
Logger.error("ParadeDB create_index failed: #{inspect(error)}")
{:error, error}
end
end
@impl true
def drop_index do
case client_impl().query("DROP TABLE IF EXISTS #{table()}", []) do
{:ok, _} ->
:ok
{:error, error} ->
Logger.error("ParadeDB drop_index failed: #{inspect(error)}")
{:error, error}
end
end
@impl true
def healthcheck_endpoints, do: nil
@impl true
def add_to_index(activity) do
with true <- Search.indexable?(activity),
maybe_search_data when not is_nil(maybe_search_data) <-
Search.object_to_search_data(activity.object) do
dumped_activity_id = dump_activity_id(activity.id)
actor_ap_id = activity.data["actor"]
published_at = DateTime.from_unix!(maybe_search_data.published)
if is_nil(dumped_activity_id) do
Logger.error("ParadeDB add_to_index failed: invalid activity id #{inspect(activity.id)}")
:ok
else
sql =
"""
INSERT INTO #{table()} (id, object_id, object_ap_id, actor_ap_id, content, published_at)
VALUES ($1, $2, $3, $4, $5, $6)
ON CONFLICT (id) DO UPDATE SET
object_id = EXCLUDED.object_id,
object_ap_id = EXCLUDED.object_ap_id,
actor_ap_id = EXCLUDED.actor_ap_id,
content = EXCLUDED.content,
published_at = EXCLUDED.published_at
"""
params = [
dumped_activity_id,
maybe_search_data.id,
maybe_search_data.ap,
actor_ap_id,
maybe_search_data.content,
published_at
]
case client_impl().query(sql, params) do
{:ok, _} ->
:ok
{:error, error} ->
Logger.error("ParadeDB add_to_index failed: #{inspect(error)}")
{:error, error}
end
end
else
_ -> :ok
end
end
@impl true
def remove_from_index(object) do
sql = "DELETE FROM #{table()} WHERE object_id = $1"
case client_impl().query(sql, [object.id]) do
{:ok, _} ->
:ok
{:error, error} ->
Logger.error("ParadeDB remove_from_index failed: #{inspect(error)}")
{:error, error}
end
end
@impl true
def search(user, query, options \\ []) do
limit = options |> Keyword.get(:limit, @max_limit) |> min(@max_limit)
offset = Keyword.get(options, :offset, 0)
author = Keyword.get(options, :author)
case search_ids(query, limit, offset, author) do
{:ok, []} ->
maybe_fetch([], user, query)
{:ok, ids} ->
ids
|> fetch_activities(user, query, author)
{:error, error} ->
Logger.error("ParadeDB search failed: #{inspect(error)}")
maybe_fetch([], user, query)
end
end
defp fetch_activities(ids, user, query, author) do
dumped_ids =
ids
|> Enum.map(&dump_activity_id/1)
|> Enum.reject(&is_nil/1)
from(a in Activity, where: a.id in ^dumped_ids)
|> Activity.with_preloaded_object()
|> Activity.restrict_deactivated_users()
|> maybe_restrict_public(user)
|> maybe_restrict_local(user)
|> maybe_restrict_author(author)
|> maybe_restrict_blocked(user)
|> order_by(
[a],
fragment(
"array_position(?, ?)",
^dumped_ids,
a.id
)
)
|> Pleroma.Repo.all()
|> maybe_fetch(user, query)
end
defp maybe_restrict_public(query, %Pleroma.User{}) do
intended_recipients = [
Pleroma.Constants.as_public(),
Pleroma.Web.ActivityPub.Utils.as_local_public()
]
from(a in query,
where: fragment("?->>'type' = 'Create'", a.data),
where: fragment("? && ?", ^intended_recipients, a.recipients)
)
end
defp maybe_restrict_public(query, _user) do
from(a in query,
where: fragment("?->>'type' = 'Create'", a.data),
where: ^Pleroma.Constants.as_public() in a.recipients
)
end
defp search_ids(query, limit, offset, author) do
base_sql =
"SELECT id FROM #{table()} WHERE content &&& #{search_query_param_sql(fuzzy_distance())}"
params = [query]
{base_sql, params} =
case author do
%Pleroma.User{ap_id: ap_id} when is_binary(ap_id) ->
{base_sql <> " AND actor_ap_id = $2", params ++ [ap_id]}
_ ->
{base_sql, params}
end
limit_arg = length(params) + 1
offset_arg = length(params) + 2
sql =
base_sql <>
" ORDER BY published_at DESC, id DESC LIMIT $#{limit_arg} OFFSET $#{offset_arg}"
params = params ++ [limit, offset]
with {:ok, %{rows: rows}} <- client_impl().query(sql, params) do
{:ok, Enum.map(rows, fn [id] -> id end)}
end
end
defp search_query_param_sql(0), do: "$1"
defp search_query_param_sql(distance), do: "($1::text)::pdb.fuzzy(#{distance})"
defp create_table_sql(table) do
"""
CREATE TABLE IF NOT EXISTS #{table} (
id uuid PRIMARY KEY,
object_id bigint NOT NULL,
object_ap_id text NOT NULL,
actor_ap_id text,
content text NOT NULL,
published_at timestamptz
)
"""
end
defp create_object_id_index_sql(table) do
"CREATE INDEX IF NOT EXISTS #{table}_object_id_idx ON #{table} (object_id)"
end
defp create_bm25_index_sql(table) do
"""
CREATE INDEX IF NOT EXISTS #{table}_bm25_idx ON #{table}
USING bm25 (id, content, (actor_ap_id::pdb.literal), published_at)
WITH (key_field = 'id')
"""
end
defp dump_activity_id(<<_::binary-size(16)>> = uuid), do: uuid
defp dump_activity_id(id) when is_binary(id) do
case Ecto.UUID.dump(id) do
{:ok, <<_::binary-size(16)>> = uuid} ->
uuid
:error ->
dump_flake_activity_id(id)
end
end
defp dump_activity_id(_), do: nil
defp dump_flake_activity_id(id) do
case FlakeId.Ecto.CompatType.dump(id) do
{:ok, <<_::binary-size(16)>> = uuid} -> uuid
_ -> nil
end
rescue
_ -> nil
end
end

24
lib/pleroma/search/parade_db/client.ex Normal file
View file

@ -0,0 +1,24 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Search.ParadeDB.Client do
@moduledoc false
@type sql :: String.t()
@type params :: list()
@type result :: %{optional(:rows) => list(list())}
@callback query(sql(), params()) :: {:ok, result()} | {:error, any()}
end
defmodule Pleroma.Search.ParadeDB.Client.Postgres do
@moduledoc false
@behaviour Pleroma.Search.ParadeDB.Client
@impl true
def query(sql, params \\ []) do
Ecto.Adapters.SQL.query(Pleroma.Search.ParadeDB.Repo, sql, params)
end
end

24
lib/pleroma/search/parade_db/repo.ex Normal file
View file

@ -0,0 +1,24 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Search.ParadeDB.Repo do
use Ecto.Repo,
otp_app: :pleroma,
adapter: Ecto.Adapters.Postgres,
migration_timestamps: [type: :naive_datetime_usec]
@doc """
Loads the ParadeDB database URL from `PARADEDB_DATABASE_URL`.
If the env var is not set, falls back to `config :pleroma, Pleroma.Search.ParadeDB, url: ...`.
"""
def init(_, opts) do
config = Application.get_env(:pleroma, Pleroma.Search.ParadeDB, [])
url = System.get_env("PARADEDB_DATABASE_URL") || config[:url]
url = if is_nil(url), do: nil, else: to_string(url)
{:ok, Keyword.put(opts, :url, url)}
end
end

83
lib/pleroma/search/qdrant_search.ex
View file

@ -4,11 +4,12 @@ defmodule Pleroma.Search.QdrantSearch do
alias Pleroma.Activity
alias Pleroma.Config.Getting, as: Config
alias Pleroma.Object
alias Pleroma.Search
alias __MODULE__.OpenAIClient
alias __MODULE__.QdrantClient
import Pleroma.Search.Meilisearch, only: [object_to_search_data: 1]
import Pleroma.Search.DatabaseSearch, only: [maybe_fetch: 3]
@impl true
@ -82,23 +83,18 @@ defmodule Pleroma.Search.QdrantSearch do
end
@impl true
def add_to_index(activity) do
# This will only index public or unlisted notes
maybe_search_data = object_to_search_data(activity.object)
def add_to_index(%Activity{object: %Object{} = object} = activity) do
search_data = Search.object_to_search_data(object)
if activity.data["type"] == "Create" and maybe_search_data do
with {:ok, embedding} <- get_embedding(maybe_search_data.content),
{:ok, %{status: 200}} <-
QdrantClient.put(
"/collections/posts/points",
build_index_payload(activity, embedding)
) do
:ok
else
e -> {:error, e}
end
else
with {:ok, embedding} <- get_embedding(search_data.content),
{:ok, %{status: 200}} <-
QdrantClient.put(
"/collections/posts/points",
build_index_payload(activity, embedding)
) do
:ok
else
e -> {:error, e}
end
end
@ -157,26 +153,55 @@ defmodule Pleroma.Search.QdrantSearch do
end
defmodule Pleroma.Search.QdrantSearch.OpenAIClient do
use Tesla
alias Pleroma.Config.Getting, as: Config
plug(Tesla.Middleware.BaseUrl, Config.get([Pleroma.Search.QdrantSearch, :openai_url]))
plug(Tesla.Middleware.JSON)
def post(path, body) do
Tesla.post(client(), path, body)
end
plug(Tesla.Middleware.Headers, [
{"Authorization",
"Bearer #{Pleroma.Config.get([Pleroma.Search.QdrantSearch, :openai_api_key])}"}
])
defp client do
Tesla.client(middleware())
end
defp middleware do
[
{Tesla.Middleware.BaseUrl, Config.get([Pleroma.Search.QdrantSearch, :openai_url])},
Tesla.Middleware.JSON,
{Tesla.Middleware.Headers,
[
{"Authorization", "Bearer #{Config.get([Pleroma.Search.QdrantSearch, :openai_api_key])}"}
]}
]
end
end
defmodule Pleroma.Search.QdrantSearch.QdrantClient do
use Tesla
alias Pleroma.Config.Getting, as: Config
plug(Tesla.Middleware.BaseUrl, Config.get([Pleroma.Search.QdrantSearch, :qdrant_url]))
plug(Tesla.Middleware.JSON)
def delete(path) do
Tesla.delete(client(), path)
end
plug(Tesla.Middleware.Headers, [
{"api-key", Pleroma.Config.get([Pleroma.Search.QdrantSearch, :qdrant_api_key])}
])
def post(path, body) do
Tesla.post(client(), path, body)
end
def put(path, body) do
Tesla.put(client(), path, body)
end
defp client do
Tesla.client(middleware())
end
defp middleware do
[
{Tesla.Middleware.BaseUrl, Config.get([Pleroma.Search.QdrantSearch, :qdrant_url])},
Tesla.Middleware.JSON,
{Tesla.Middleware.Headers,
[
{"api-key", Pleroma.Config.get([Pleroma.Search.QdrantSearch, :qdrant_api_key])}
]}
]
end
end

5
lib/pleroma/signature.ex
View file

@ -3,6 +3,7 @@
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Signature do
@behaviour Pleroma.Signature.API
@behaviour HTTPSignatures.Adapter
alias Pleroma.EctoType.ActivityPub.ObjectValidators
@ -53,7 +54,7 @@ defmodule Pleroma.Signature do
def fetch_public_key(conn) do
with {:ok, actor_id} <- get_actor_id(conn),
{:ok, public_key} <- User.get_public_key_for_ap_id(actor_id) do
{:ok, public_key} <- User.get_or_fetch_public_key_for_ap_id(actor_id) do
{:ok, public_key}
else
e ->
@ -103,7 +104,7 @@ defmodule Pleroma.Signature do
|> put_req_header("(request-target)", request_target)
|> put_req_header("@request-target", request_target)
@http_signatures_impl.validate_conn(conn)
@http_signatures_impl.validate_conn(conn) == true
end
@spec validate_signature(Plug.Conn.t()) :: boolean()

14
lib/pleroma/signature/api.ex Normal file
View file

@ -0,0 +1,14 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Signature.API do
@moduledoc """
Behaviour for signing requests and producing HTTP Date headers.
This is used to allow tests to replace the signing implementation with Mox.
"""
@callback sign(user :: Pleroma.User.t(), headers :: map()) :: String.t()
@callback signed_date() :: String.t()
end

2
lib/pleroma/telemetry/logger.ex
View file

@ -55,8 +55,8 @@ defmodule Pleroma.Telemetry.Logger do
def handle_event(
[:pleroma, :connection_pool, :provision_failure],
%{opts: [key | _]},
_,
%{key: key},
_
) do
Logger.debug(fn ->

60
lib/pleroma/tesla/middleware/connection_pool.ex
View file

@ -12,25 +12,38 @@ defmodule Pleroma.Tesla.Middleware.ConnectionPool do
alias Pleroma.Gun.ConnectionPool
@impl Tesla.Middleware
def call(%Tesla.Env{url: url, opts: opts} = env, next, _) do
def call(%Tesla.Env{opts: opts} = env, next, middleware_opts) do
if opts[:adapter][:body_as] == :stream do
{:error, :pooled_streaming_requires_chunks}
else
do_call(env, next, middleware_opts)
end
end
defp do_call(%Tesla.Env{url: url, opts: opts} = env, next, _) do
uri = URI.parse(url)
# Avoid leaking connections when the middleware is called twice
# with body_as: :chunks. We assume only the middleware can set
# opts[:adapter][:conn]
if opts[:adapter][:conn] do
ConnectionPool.release_conn(opts[:adapter][:conn])
end
cleanup_previous_request(env)
case ConnectionPool.get_conn(uri, opts[:adapter]) do
{:ok, conn_pid} ->
adapter_opts = Keyword.merge(opts[:adapter], conn: conn_pid, close_conn: false)
tunnel = ConnectionPool.tunnel_ref(conn_pid)
adapter_opts =
opts[:adapter]
|> Keyword.drop([:conn, :stream, :tunnel])
|> Keyword.merge(conn: conn_pid, close_conn: false, tunnel: tunnel)
opts = Keyword.put(opts, :adapter, adapter_opts)
env = %{env | opts: opts}
next = use_pool_adapter(next)
case Tesla.run(env, next) do
{:ok, env} ->
unless opts[:adapter][:body_as] == :chunks do
unless opts[:adapter][:body_as] == :chunks and chunk_client?(env.body) do
ConnectionPool.release_conn(conn_pid)
{_, res} = pop_in(env.opts[:adapter][:conn])
{:ok, res}
@ -38,8 +51,11 @@ defmodule Pleroma.Tesla.Middleware.ConnectionPool do
{:ok, env}
end
{:error, {:gun_stream_error, reason}} ->
{:error, reason}
err ->
ConnectionPool.release_conn(conn_pid)
ConnectionPool.discard_conn(conn_pid)
err
end
@ -47,4 +63,34 @@ defmodule Pleroma.Tesla.Middleware.ConnectionPool do
err
end
end
defp cleanup_previous_request(%Tesla.Env{
body: %{pid: pid, stream: stream},
opts: opts
}) do
if opts[:adapter][:conn], do: ConnectionPool.cancel_stream(pid, stream)
end
defp cleanup_previous_request(%Tesla.Env{opts: opts}) do
conn = opts[:adapter][:conn]
stream = opts[:adapter][:stream]
cond do
conn && stream -> ConnectionPool.cancel_stream(conn, stream)
conn -> ConnectionPool.release_conn(conn)
true -> :ok
end
end
defp chunk_client?(%{pid: pid, stream: stream}) when is_pid(pid) and not is_nil(stream),
do: true
defp chunk_client?(_body), do: false
defp use_pool_adapter(next) do
List.update_at(next, -1, fn
{Tesla.Adapter.Gun, function, args} -> {Pleroma.HTTP.Adapter.Gun, function, args}
adapter -> adapter
end)
end
end

29
lib/pleroma/tesla/middleware/encode_url.ex Normal file
View file

@ -0,0 +1,29 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2025 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Tesla.Middleware.EncodeUrl do
@moduledoc """
Middleware to encode URLs properly
We must decode and then re-encode to ensure correct encoding.
If we only encode it will re-encode each % as %25 causing a space
already encoded as %20 to be %2520.
Similar problem for query parameters which need spaces to be the + character
"""
@behaviour Tesla.Middleware
@impl Tesla.Middleware
def call(%Tesla.Env{url: url} = env, next, _) do
url = Pleroma.Utils.URIEncoding.encode_url(url)
env = %{env | url: url}
case Tesla.run(env, next) do
{:ok, env} -> {:ok, env}
err -> err
end
end
end

14
lib/pleroma/upload.ex
View file

@ -35,6 +35,7 @@ defmodule Pleroma.Upload do
"""
alias Ecto.UUID
alias Pleroma.Maps
alias Pleroma.Utils.URIEncoding
alias Pleroma.Web.ActivityPub.Utils
require Logger
@ -92,7 +93,7 @@ defmodule Pleroma.Upload do
def store(upload, opts \\ []) do
opts = get_opts(opts)
with {:ok, upload} <- prepare_upload(upload, opts),
with {:ok, %__MODULE__{} = upload} <- prepare_upload(upload, opts),
upload = %__MODULE__{upload | path: upload.path || "#{upload.id}/#{upload.name}"},
{:ok, upload} <- Pleroma.Upload.Filter.filter(opts.filters, upload),
description = get_description(upload),
@ -230,11 +231,18 @@ defmodule Pleroma.Upload do
tmp_path
end
# Encoding the whole path here is fine since the path is in a
# UUID/<file name> form.
# The file at this point isn't %-encoded, so the path shouldn't
# be decoded first like Pleroma.Utils.URIEncoding.encode_url/1 does.
defp url_from_spec(%__MODULE__{name: name}, base_url, {:file, path}) do
encode_opts = [bypass_decode: true, bypass_parse: true]
path =
URI.encode(path, &char_unescaped?/1) <>
URIEncoding.encode_url(path, encode_opts) <>
if Pleroma.Config.get([__MODULE__, :link_name], false) do
"?name=#{URI.encode(name, &char_unescaped?/1)}"
enum = %{name: name}
"?#{URI.encode_query(enum)}"
else
""
end

10
lib/pleroma/upload/filter/analyze_metadata.ex
View file

@ -90,9 +90,13 @@ defmodule Pleroma.Upload.Filter.AnalyzeMetadata do
{:ok, rgb} =
if Image.has_alpha?(resized_image) do
# remove alpha channel
resized_image
|> Operation.extract_band!(0, n: 3)
|> Image.write_to_binary()
case Operation.extract_band(resized_image, 0, n: 3) do
{:ok, data} ->
Image.write_to_binary(data)
_ ->
Image.write_to_binary(resized_image)
end
else
Image.write_to_binary(resized_image)
end

4
lib/pleroma/upload/filter/anonymize_filename.ex
View file

@ -10,7 +10,7 @@ defmodule Pleroma.Upload.Filter.AnonymizeFilename do
"""
@behaviour Pleroma.Upload.Filter
alias Pleroma.Config
@config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config)
alias Pleroma.Upload
def filter(%Upload{name: name} = upload) do
@ -23,7 +23,7 @@ defmodule Pleroma.Upload.Filter.AnonymizeFilename do
@spec predefined_name(String.t()) :: String.t() | nil
defp predefined_name(extension) do
with name when not is_nil(name) <- Config.get([__MODULE__, :text]),
with name when not is_nil(name) <- @config_impl.get([__MODULE__, :text]),
do: String.replace(name, "{extension}", extension)
end

32
lib/pleroma/upload/filter/exiftool/read_description.ex
View file

@ -29,22 +29,26 @@ defmodule Pleroma.Upload.Filter.Exiftool.ReadDescription do
do: current_description
defp read_when_empty(_, file, tag) do
try do
{tag_content, 0} =
System.cmd("exiftool", ["-b", "-s3", tag, file],
stderr_to_stdout: false,
parallelism: true
)
if File.exists?(file) do
try do
{tag_content, 0} =
System.cmd("exiftool", ["-m", "-b", "-s3", tag, file],
stderr_to_stdout: false,
parallelism: true
)
tag_content = String.trim(tag_content)
tag_content = String.trim(tag_content)
if tag_content != "" and
String.length(tag_content) <=
Pleroma.Config.get([:instance, :description_limit]),
do: tag_content,
else: nil
rescue
_ in ErlangError -> nil
if tag_content != "" and
String.length(tag_content) <=
Pleroma.Config.get([:instance, :description_limit]),
do: tag_content,
else: nil
rescue
_ in ErlangError -> nil
end
else
nil
end
end
end

5
lib/pleroma/upload/filter/exiftool/strip_location.ex
View file

@ -16,11 +16,12 @@ defmodule Pleroma.Upload.Filter.Exiftool.StripLocation do
def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do
try do
case System.cmd("exiftool", ["-overwrite_original", "-gps:all=", "-png:all=", file],
case System.cmd("exiftool", ["-m", "-overwrite_original", "-gps:all=", "-png:all=", file],
stderr_to_stdout: true,
parallelism: true
) do
{_response, 0} -> {:ok, :filtered}
{error, 1} -> {:error, error}
{error, _} -> {:error, error}
end
rescue
e in ErlangError ->

17
lib/pleroma/upload/filter/mogrify.ex
View file

@ -8,9 +8,16 @@ defmodule Pleroma.Upload.Filter.Mogrify do
@type conversion :: action :: String.t() | {action :: String.t(), opts :: String.t()}
@type conversions :: conversion() | [conversion()]
@config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config)
@mogrify_impl Application.compile_env(
:pleroma,
[__MODULE__, :mogrify_impl],
Pleroma.MogrifyWrapper
)
def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do
try do
do_filter(file, Pleroma.Config.get!([__MODULE__, :args]))
do_filter(file, @config_impl.get!([__MODULE__, :args]))
{:ok, :filtered}
rescue
e in ErlangError ->
@ -22,9 +29,9 @@ defmodule Pleroma.Upload.Filter.Mogrify do
def do_filter(file, filters) do
file
|> Mogrify.open()
|> @mogrify_impl.open()
|> mogrify_filter(filters)
|> Mogrify.save(in_place: true)
|> @mogrify_impl.save(in_place: true)
end
defp mogrify_filter(mogrify, nil), do: mogrify
@ -38,10 +45,10 @@ defmodule Pleroma.Upload.Filter.Mogrify do
defp mogrify_filter(mogrify, []), do: mogrify
defp mogrify_filter(mogrify, {action, options}) do
Mogrify.custom(mogrify, action, options)
@mogrify_impl.custom(mogrify, action, options)
end
defp mogrify_filter(mogrify, action) when is_binary(action) do
Mogrify.custom(mogrify, action)
@mogrify_impl.custom(mogrify, action)
end
end

2
lib/pleroma/uploaders/local.ex
View file

@ -19,7 +19,7 @@ defmodule Pleroma.Uploaders.Local do
[file | folders] ->
path = Path.join([upload_path()] ++ Enum.reverse(folders))
File.mkdir_p!(path)
Pleroma.Backports.mkdir_p!(path)
{path, file}
end

204
lib/pleroma/user.ex
View file

@ -19,6 +19,7 @@ defmodule Pleroma.User do
alias Pleroma.Emoji
alias Pleroma.FollowingRelationship
alias Pleroma.Formatter
alias Pleroma.Hashtag
alias Pleroma.HTML
alias Pleroma.Keys
alias Pleroma.MFA
@ -27,6 +28,7 @@ defmodule Pleroma.User do
alias Pleroma.Registration
alias Pleroma.Repo
alias Pleroma.User
alias Pleroma.User.HashtagFollow
alias Pleroma.UserRelationship
alias Pleroma.Web.ActivityPub.ActivityPub
alias Pleroma.Web.ActivityPub.Builder
@ -37,6 +39,7 @@ defmodule Pleroma.User do
alias Pleroma.Web.Endpoint
alias Pleroma.Web.OAuth
alias Pleroma.Web.RelMe
alias Pleroma.Webhook.Notify
alias Pleroma.Workers.BackgroundWorker
alias Pleroma.Workers.DeleteWorker
alias Pleroma.Workers.UserRefreshWorker
@ -148,7 +151,7 @@ defmodule Pleroma.User do
field(:allow_following_move, :boolean, default: true)
field(:skip_thread_containment, :boolean, default: false)
field(:actor_type, :string, default: "Person")
field(:also_known_as, {:array, ObjectValidators.ObjectID}, default: [])
field(:also_known_as, {:array, ObjectValidators.BareUri}, default: [])
field(:inbox, :string)
field(:shared_inbox, :string)
field(:accepts_chat_messages, :boolean, default: nil)
@ -174,6 +177,12 @@ defmodule Pleroma.User do
has_many(:outgoing_relationships, UserRelationship, foreign_key: :source_id)
has_many(:incoming_relationships, UserRelationship, foreign_key: :target_id)
many_to_many(:followed_hashtags, Hashtag,
on_replace: :delete,
on_delete: :delete_all,
join_through: HashtagFollow
)
for {relationship_type,
[
{outgoing_relation, outgoing_relation_target},
@ -225,8 +234,8 @@ defmodule Pleroma.User do
for {_relationship_type, [{_outgoing_relation, outgoing_relation_target}, _]} <-
@user_relationships_config do
# `def blocked_users_relation/2`, `def muted_users_relation/2`,
# `def reblog_muted_users_relation/2`, `def notification_muted_users/2`,
# `def subscriber_users/2`, `def endorsed_users_relation/2`
# `def reblog_muted_users_relation/2`, `def notification_muted_users_relation/2`,
# `def subscriber_users_relation/2`, `def endorsed_users_relation/2`
def unquote(:"#{outgoing_relation_target}_relation")(user, restrict_deactivated? \\ false) do
target_users_query = assoc(user, unquote(outgoing_relation_target))
@ -264,13 +273,13 @@ defmodule Pleroma.User do
end
def cached_blocked_users_ap_ids(user) do
@cachex.fetch!(:user_cache, "blocked_users_ap_ids:#{user.ap_id}", fn _ ->
@cachex.fetch!(:user_cache, cache_key(:blocked_users_ap_ids, user), fn _ ->
blocked_users_ap_ids(user)
end)
end
def cached_muted_users_ap_ids(user) do
@cachex.fetch!(:user_cache, "muted_users_ap_ids:#{user.ap_id}", fn _ ->
@cachex.fetch!(:user_cache, cache_key(:muted_users_ap_ids, user), fn _ ->
muted_users_ap_ids(user)
end)
end
@ -279,7 +288,14 @@ defmodule Pleroma.User do
defdelegate following(user), to: FollowingRelationship
defdelegate following?(follower, followed), to: FollowingRelationship
defdelegate following_ap_ids(user), to: FollowingRelationship
defdelegate get_follow_requests(user), to: FollowingRelationship
defdelegate get_follow_requests_query(user), to: FollowingRelationship
defdelegate get_outgoing_follow_requests_query(user), to: FollowingRelationship
def get_follow_requests(user) do
get_follow_requests_query(user)
|> Repo.all()
end
defdelegate search(query, opts \\ []), to: User.Search
@doc """
@ -300,7 +316,7 @@ defmodule Pleroma.User do
def binary_id(%User{} = user), do: binary_id(user.id)
@doc "Returns status account"
@doc "Returns account status"
@spec account_status(User.t()) :: account_status()
def account_status(%User{is_active: false}), do: :deactivated
def account_status(%User{password_reset_pending: true}), do: :password_reset_pending
@ -793,13 +809,6 @@ defmodule Pleroma.User do
when is_nil(password) do
params = Map.put_new(params, :accepts_chat_messages, true)
params =
if Map.has_key?(params, :email) do
Map.put_new(params, :email, params[:email])
else
params
end
struct
|> cast(params, [
:name,
@ -887,7 +896,7 @@ defmodule Pleroma.User do
end)
end
def validate_email_not_in_blacklisted_domain(changeset, field) do
defp validate_email_not_in_blacklisted_domain(changeset, field) do
validate_change(changeset, field, fn _, value ->
valid? =
Config.get([User, :email_blacklist])
@ -904,9 +913,9 @@ defmodule Pleroma.User do
end)
end
def maybe_validate_required_email(changeset, true), do: changeset
defp maybe_validate_required_email(changeset, true), do: changeset
def maybe_validate_required_email(changeset, _) do
defp maybe_validate_required_email(changeset, _) do
if Config.get([:instance, :account_activation_required]) do
validate_required(changeset, [:email])
else
@ -979,6 +988,7 @@ defmodule Pleroma.User do
@doc "Inserts provided changeset, performs post-registration actions (confirmation email sending etc.)"
def register(%Ecto.Changeset{} = changeset) do
with {:ok, user} <- Repo.insert(changeset) do
Notify.trigger_webhooks(user, :"account.created")
post_register_action(user)
end
end
@ -1101,15 +1111,15 @@ defmodule Pleroma.User do
defp maybe_send_registration_email(_), do: {:ok, :noop}
def needs_update?(%User{local: true}), do: false
defp needs_update?(%User{local: true}), do: false
def needs_update?(%User{local: false, last_refreshed_at: nil}), do: true
defp needs_update?(%User{local: false, last_refreshed_at: nil}), do: true
def needs_update?(%User{local: false} = user) do
defp needs_update?(%User{local: false} = user) do
NaiveDateTime.diff(NaiveDateTime.utc_now(), user.last_refreshed_at) >= 86_400
end
def needs_update?(_), do: true
defp needs_update?(_), do: true
@spec maybe_direct_follow(User.t(), User.t()) ::
{:ok, User.t(), User.t()} | {:error, String.t()}
@ -1245,13 +1255,18 @@ defmodule Pleroma.User do
get_cached_by_nickname(nickname)
end
defp cache_key(:id, %User{id: id}), do: cache_key(:id, id)
defp cache_key(:nickname, %User{nickname: nickname}), do: cache_key(:nickname, nickname)
defp cache_key(namespace, %User{ap_id: ap_id}), do: cache_key(namespace, ap_id)
defp cache_key(namespace, value), do: "#{namespace}:#{value}"
def set_cache({:ok, user}), do: set_cache(user)
def set_cache({:error, err}), do: {:error, err}
def set_cache(%User{} = user) do
@cachex.put(:user_cache, "ap_id:#{user.ap_id}", user)
@cachex.put(:user_cache, "nickname:#{user.nickname}", user)
@cachex.put(:user_cache, "friends_ap_ids:#{user.nickname}", get_user_friends_ap_ids(user))
@cachex.put(:user_cache, cache_key(:ap_id, user), user)
@cachex.put(:user_cache, cache_key(:nickname, user), user)
@cachex.put(:user_cache, cache_key(:friends_ap_ids, user), get_user_friends_ap_ids(user))
{:ok, user}
end
@ -1279,22 +1294,23 @@ defmodule Pleroma.User do
@spec get_cached_user_friends_ap_ids(User.t()) :: [String.t()]
def get_cached_user_friends_ap_ids(user) do
@cachex.fetch!(:user_cache, "friends_ap_ids:#{user.ap_id}", fn _ ->
@cachex.fetch!(:user_cache, cache_key(:friends_ap_ids, user), fn _ ->
get_user_friends_ap_ids(user)
end)
end
def invalidate_cache(user) do
@cachex.del(:user_cache, "ap_id:#{user.ap_id}")
@cachex.del(:user_cache, "nickname:#{user.nickname}")
@cachex.del(:user_cache, "friends_ap_ids:#{user.ap_id}")
@cachex.del(:user_cache, "blocked_users_ap_ids:#{user.ap_id}")
@cachex.del(:user_cache, "muted_users_ap_ids:#{user.ap_id}")
@cachex.del(:user_cache, cache_key(:ap_id, user))
@cachex.del(:user_cache, cache_key(:nickname, user))
@cachex.del(:user_cache, cache_key(:id, user))
@cachex.del(:user_cache, cache_key(:friends_ap_ids, user))
@cachex.del(:user_cache, cache_key(:blocked_users_ap_ids, user))
@cachex.del(:user_cache, cache_key(:muted_users_ap_ids, user))
end
@spec get_cached_by_ap_id(String.t()) :: User.t() | nil
def get_cached_by_ap_id(ap_id) do
key = "ap_id:#{ap_id}"
key = cache_key(:ap_id, ap_id)
with {:ok, nil} <- @cachex.get(:user_cache, key),
user when not is_nil(user) <- get_by_ap_id(ap_id),
@ -1307,14 +1323,14 @@ defmodule Pleroma.User do
end
def get_cached_by_id(id) do
key = "id:#{id}"
key = cache_key(:id, id)
ap_id =
@cachex.fetch!(:user_cache, key, fn _ ->
user = get_by_id(id)
if user do
@cachex.put(:user_cache, "ap_id:#{user.ap_id}", user)
@cachex.put(:user_cache, cache_key(:ap_id, user), user)
{:commit, user.ap_id}
else
{:ignore, ""}
@ -1325,7 +1341,7 @@ defmodule Pleroma.User do
end
def get_cached_by_nickname(nickname) do
key = "nickname:#{nickname}"
key = cache_key(:nickname, nickname)
@cachex.fetch!(:user_cache, key, fn _ ->
case get_or_fetch_by_nickname(nickname) do
@ -1356,7 +1372,7 @@ defmodule Pleroma.User do
@spec get_by_nickname(String.t()) :: User.t() | nil
def get_by_nickname(nickname) do
Repo.get_by(User, nickname: nickname) ||
if Regex.match?(~r(@#{Pleroma.Web.Endpoint.host()})i, nickname) do
if Regex.match?(~r(@#{Pleroma.Web.Endpoint.host()}$)i, nickname) do
Repo.get_by(User, nickname: local_nickname(nickname))
end
end
@ -1643,7 +1659,7 @@ defmodule Pleroma.User do
|> Oban.insert()
end
@cachex.del(:user_cache, "muted_users_ap_ids:#{muter.ap_id}")
@cachex.del(:user_cache, cache_key(:muted_users_ap_ids, muter))
{:ok, Enum.filter([user_mute, user_notification_mute], & &1)}
end
@ -1653,7 +1669,7 @@ defmodule Pleroma.User do
with {:ok, user_mute} <- UserRelationship.delete_mute(muter, mutee),
{:ok, user_notification_mute} <-
UserRelationship.delete_notification_mute(muter, mutee) do
@cachex.del(:user_cache, "muted_users_ap_ids:#{muter.ap_id}")
@cachex.del(:user_cache, cache_key(:muted_users_ap_ids, muter))
{:ok, [user_mute, user_notification_mute]}
end
end
@ -1700,7 +1716,9 @@ defmodule Pleroma.User do
end
end
def block(%User{} = blocker, %User{} = blocked) do
def block(blocker, blocked, params \\ %{})
def block(%User{} = blocker, %User{} = blocked, params) do
# sever any follow relationships to prevent leaks per activitypub (Pleroma issue #213)
blocker =
if following?(blocker, blocked) do
@ -1730,12 +1748,33 @@ defmodule Pleroma.User do
{:ok, blocker} = update_follower_count(blocker)
{:ok, blocker, _} = Participation.mark_all_as_read(blocker, blocked)
add_to_block(blocker, blocked)
duration = Map.get(params, :duration, 0)
expires_at =
if duration > 0 do
DateTime.utc_now()
|> DateTime.add(duration)
else
nil
end
user_block = add_to_block(blocker, blocked, expires_at)
if duration > 0 do
Pleroma.Workers.MuteExpireWorker.new(
%{"op" => "unblock_user", "blocker_id" => blocker.id, "blocked_id" => blocked.id},
scheduled_at: expires_at
)
|> Oban.insert()
end
user_block
end
# helper to handle the block given only an actor's AP id
def block(%User{} = blocker, %{ap_id: ap_id}) do
block(blocker, get_cached_by_ap_id(ap_id))
def block(%User{} = blocker, %{ap_id: ap_id}, params) do
block(blocker, get_cached_by_ap_id(ap_id), params)
end
def unblock(%User{} = blocker, %User{} = blocked) do
@ -1933,6 +1972,12 @@ defmodule Pleroma.User do
def approve(%User{} = user), do: {:ok, user}
def reject(%User{is_approved: false} = user) do
delete(user)
end
def reject(%User{} = _user), do: {:error, "User is approved"}
def confirm(users) when is_list(users) do
Repo.transaction(fn ->
Enum.map(users, fn user ->
@ -1976,7 +2021,7 @@ defmodule Pleroma.User do
end
@spec purge_user_changeset(User.t()) :: Ecto.Changeset.t()
def purge_user_changeset(user) do
defp purge_user_changeset(user) do
# "Right to be forgotten"
# https://gdpr.eu/right-to-be-forgotten/
change(user, %{
@ -2148,7 +2193,7 @@ defmodule Pleroma.User do
Repo.all(query)
end
def delete_notifications_from_user_activities(%User{ap_id: ap_id}) do
defp delete_notifications_from_user_activities(%User{ap_id: ap_id}) do
Notification
|> join(:inner, [n], activity in assoc(n, :activity))
|> where([n, a], fragment("? = ?", a.actor, ^ap_id))
@ -2283,6 +2328,15 @@ defmodule Pleroma.User do
def public_key(_), do: {:error, "key not found"}
def get_or_fetch_public_key_for_ap_id(ap_id) do
with {:ok, %User{} = user} <- get_or_fetch_by_ap_id(ap_id),
{:ok, public_key} <- public_key(user) do
{:ok, public_key}
else
_ -> :error
end
end
def get_public_key_for_ap_id(ap_id) do
with %User{} = user <- get_cached_by_ap_id(ap_id),
{:ok, public_key} <- public_key(user) do
@ -2607,7 +2661,7 @@ defmodule Pleroma.User do
end
end
# Internal function; public one is `deactivate/2`
# Internal function; public one is `set_activation/2`
defp set_activation_status(user, status) do
user
|> cast(%{is_active: status}, [:is_active])
@ -2626,7 +2680,7 @@ defmodule Pleroma.User do
|> update_and_set_cache()
end
def validate_fields(changeset, remote? \\ false) do
defp validate_fields(changeset, remote?) do
limit_name = if remote?, do: :max_remote_account_fields, else: :max_account_fields
limit = Config.get([:instance, limit_name], 0)
@ -2771,11 +2825,11 @@ defmodule Pleroma.User do
set_domain_blocks(user, List.delete(user.domain_blocks, domain_blocked))
end
@spec add_to_block(User.t(), User.t()) ::
@spec add_to_block(User.t(), User.t(), integer() | nil) ::
{:ok, UserRelationship.t()} | {:error, Ecto.Changeset.t()}
defp add_to_block(%User{} = user, %User{} = blocked) do
with {:ok, relationship} <- UserRelationship.create_block(user, blocked) do
@cachex.del(:user_cache, "blocked_users_ap_ids:#{user.ap_id}")
defp add_to_block(%User{} = user, %User{} = blocked, expires_at) do
with {:ok, relationship} <- UserRelationship.create_block(user, blocked, expires_at) do
@cachex.del(:user_cache, cache_key(:blocked_users_ap_ids, user))
{:ok, relationship}
end
end
@ -2784,7 +2838,7 @@ defmodule Pleroma.User do
{:ok, UserRelationship.t()} | {:ok, nil} | {:error, Ecto.Changeset.t()}
defp remove_from_block(%User{} = user, %User{} = blocked) do
with {:ok, relationship} <- UserRelationship.delete_block(user, blocked) do
@cachex.del(:user_cache, "blocked_users_ap_ids:#{user.ap_id}")
@cachex.del(:user_cache, cache_key(:blocked_users_ap_ids, user))
{:ok, relationship}
end
end
@ -2861,4 +2915,54 @@ defmodule Pleroma.User do
birthday_month: month
})
end
defp maybe_load_followed_hashtags(%User{followed_hashtags: follows} = user)
when is_list(follows),
do: user
defp maybe_load_followed_hashtags(%User{} = user) do
followed_hashtags = HashtagFollow.get_by_user(user)
%{user | followed_hashtags: followed_hashtags}
end
def followed_hashtags(%User{followed_hashtags: follows})
when is_list(follows),
do: follows
def followed_hashtags(%User{} = user) do
{:ok, user} =
user
|> maybe_load_followed_hashtags()
|> set_cache()
user.followed_hashtags
end
def follow_hashtag(%User{} = user, %Hashtag{} = hashtag) do
Logger.debug("Follow hashtag #{hashtag.name} for user #{user.nickname}")
user = maybe_load_followed_hashtags(user)
with {:ok, _} <- HashtagFollow.new(user, hashtag),
follows <- HashtagFollow.get_by_user(user),
%User{} = user <- user |> Map.put(:followed_hashtags, follows) do
user
|> set_cache()
end
end
def unfollow_hashtag(%User{} = user, %Hashtag{} = hashtag) do
Logger.debug("Unfollow hashtag #{hashtag.name} for user #{user.nickname}")
user = maybe_load_followed_hashtags(user)
with {:ok, _} <- HashtagFollow.delete(user, hashtag),
follows <- HashtagFollow.get_by_user(user),
%User{} = user <- user |> Map.put(:followed_hashtags, follows) do
user
|> set_cache()
end
end
def following_hashtag?(%User{} = user, %Hashtag{} = hashtag) do
not is_nil(HashtagFollow.get(user, hashtag))
end
end

19
lib/pleroma/user/backup.ex
View file

@ -16,6 +16,7 @@ defmodule Pleroma.User.Backup do
alias Pleroma.Bookmark
alias Pleroma.Config
alias Pleroma.Repo
alias Pleroma.SafeZip
alias Pleroma.Uploaders.Uploader
alias Pleroma.User
alias Pleroma.Web.ActivityPub.ActivityPub
@ -179,12 +180,12 @@ defmodule Pleroma.User.Backup do
end
@files [
~c"actor.json",
~c"outbox.json",
~c"likes.json",
~c"bookmarks.json",
~c"followers.json",
~c"following.json"
"actor.json",
"outbox.json",
"likes.json",
"bookmarks.json",
"followers.json",
"following.json"
]
@spec run(t()) :: {:ok, t()} | {:error, :failed}
@ -192,7 +193,7 @@ defmodule Pleroma.User.Backup do
backup = Repo.preload(backup, :user)
tempfile = Path.join([backup.tempdir, backup.file_name])
with {_, :ok} <- {:mkdir, File.mkdir_p(backup.tempdir)},
with {_, :ok} <- {:mkdir, Pleroma.Backports.mkdir_p(backup.tempdir)},
{_, :ok} <- {:actor, actor(backup.tempdir, backup.user)},
{_, :ok} <- {:statuses, statuses(backup.tempdir, backup.user)},
{_, :ok} <- {:likes, likes(backup.tempdir, backup.user)},
@ -200,7 +201,7 @@ defmodule Pleroma.User.Backup do
{_, :ok} <- {:followers, followers(backup.tempdir, backup.user)},
{_, :ok} <- {:following, following(backup.tempdir, backup.user)},
{_, {:ok, _zip_path}} <-
{:zip, :zip.create(to_charlist(tempfile), @files, cwd: to_charlist(backup.tempdir))},
{:zip, SafeZip.zip(tempfile, @files, backup.tempdir)},
{_, {:ok, %File.Stat{size: zip_size}}} <- {:filestat, File.stat(tempfile)},
{:ok, updated_backup} <- update_record(backup, %{file_size: zip_size}) do
{:ok, updated_backup}
@ -341,7 +342,7 @@ defmodule Pleroma.User.Backup do
dir,
"outbox",
fn a ->
with {:ok, activity} <- Transmogrifier.prepare_outgoing(a.data) do
with {:ok, activity} <- Transmogrifier.prepare_activity(a.data) do
{:ok, Map.delete(activity, "@context")}
end
end

Some files were not shown because too many files have changed in this diff Show more