hj/pleroma
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge remote-tracking branch 'origin/develop' into shigusegubu

Browse source
This commit is contained in:
Henry Jameson 2025-08-11 09:14:40 +03:00
commit d380bdd2f9
76 changed files with 1961 additions and 577 deletions
Download patch file Download diff file Expand all files Collapse all files

6
lib/mix/pleroma.ex
View file

@ -26,7 +26,11 @@ defmodule Mix.Pleroma do
Application.put_env(:phoenix, :serve_endpoints, false, persistent: true)
unless System.get_env("DEBUG") do
Logger.remove_backend(:console)
try do
Logger.remove_backend(:console)
catch
:exit, _ -> :ok
end
end
adapter = Application.get_env(:tesla, :adapter)

91
lib/pleroma/emoji/pack.ex
View file

@ -225,6 +225,97 @@ defmodule Pleroma.Emoji.Pack do
end
end
def download_zip(name, opts \\ %{}) do
with :ok <- validate_not_empty([name]),
:ok <- validate_new_pack(name),
{:ok, archive_data} <- fetch_archive_data(opts),
pack_path <- path_join_name_safe(emoji_path(), name),
:ok <- create_pack_dir(pack_path),
:ok <- safe_unzip(archive_data, pack_path) do
ensure_pack_json(pack_path, archive_data, opts)
else
{:error, :empty_values} -> {:error, "Pack name cannot be empty"}
{:error, reason} when is_binary(reason) -> {:error, reason}
_ -> {:error, "Could not process pack"}
end
end
defp create_pack_dir(pack_path) do
case File.mkdir_p(pack_path) do
:ok -> :ok
{:error, _} -> {:error, "Could not create the pack directory"}
end
end
defp safe_unzip(archive_data, pack_path) do
case SafeZip.unzip_data(archive_data, pack_path) do
{:ok, _} -> :ok
{:error, reason} when is_binary(reason) -> {:error, reason}
_ -> {:error, "Could not unzip pack"}
end
end
defp validate_new_pack(name) do
pack_path = path_join_name_safe(emoji_path(), name)
if File.exists?(pack_path) do
{:error, "Pack already exists, refusing to import #{name}"}
else
:ok
end
end
defp fetch_archive_data(%{url: url}) do
case Pleroma.HTTP.get(url) do
{:ok, %{status: 200, body: data}} -> {:ok, data}
_ -> {:error, "Could not download pack"}
end
end
defp fetch_archive_data(%{file: %Plug.Upload{path: path}}) do
case File.read(path) do
{:ok, data} -> {:ok, data}
_ -> {:error, "Could not read the uploaded pack file"}
end
end
defp fetch_archive_data(_) do
{:error, "Neither file nor URL was present in the request"}
end
defp ensure_pack_json(pack_path, archive_data, opts) do
pack_json_path = Path.join(pack_path, "pack.json")
if not File.exists?(pack_json_path) do
create_pack_json(pack_path, pack_json_path, archive_data, opts)
end
:ok
end
defp create_pack_json(pack_path, pack_json_path, archive_data, opts) do
emoji_map =
Pleroma.Emoji.Loader.make_shortcode_to_file_map(
pack_path,
Map.get(opts, :exts, [".png", ".gif", ".jpg"])
)
archive_sha = :crypto.hash(:sha256, archive_data) |> Base.encode16()
pack_json = %{
pack: %{
license: Map.get(opts, :license, ""),
homepage: Map.get(opts, :homepage, ""),
description: Map.get(opts, :description, ""),
src: Map.get(opts, :url),
src_sha256: archive_sha
},
files: emoji_map
}
File.write!(pack_json_path, Jason.encode!(pack_json, pretty: true))
end
@spec download(String.t(), String.t(), String.t()) :: {:ok, t()} | {:error, atom()}
def download(name, url, as) do
uri = url |> String.trim() |> URI.parse()

28
lib/pleroma/gopher/server.ex
View file

@ -22,14 +22,18 @@ defmodule Pleroma.Gopher.Server do
def init([ip, port]) do
Logger.info("Starting gopher server on #{port}")
:ranch.start_listener(
:gopher,
100,
:ranch_tcp,
[ip: ip, port: port],
__MODULE__.ProtocolHandler,
[]
)
{:ok, _pid} =
:ranch.start_listener(
:gopher,
:ranch_tcp,
%{
num_acceptors: 100,
max_connections: 100,
socket_opts: [ip: ip, port: port]
},
__MODULE__.ProtocolHandler,
[]
)
{:ok, %{ip: ip, port: port}}
end
@ -43,13 +47,13 @@ defmodule Pleroma.Gopher.Server.ProtocolHandler do
alias Pleroma.Web.ActivityPub.ActivityPub
alias Pleroma.Web.ActivityPub.Visibility
def start_link(ref, socket, transport, opts) do
pid = spawn_link(__MODULE__, :init, [ref, socket, transport, opts])
def start_link(ref, transport, opts) do
pid = spawn_link(__MODULE__, :init, [ref, transport, opts])
{:ok, pid}
end
def init(ref, socket, transport, [] = _Opts) do
:ok = :ranch.accept_ack(ref)
def init(ref, transport, opts \\ []) do
{:ok, socket} = :ranch.handshake(ref, opts)
loop(socket, transport)
end

62
lib/pleroma/hashtag.ex
View file

@ -130,4 +130,66 @@ defmodule Pleroma.Hashtag do
end
def get_recipients_for_activity(_activity), do: []
def search(query, options \\ []) do
limit = Keyword.get(options, :limit, 20)
offset = Keyword.get(options, :offset, 0)
search_terms =
query
|> String.downcase()
|> String.trim()
|> String.split(~r/\s+/)
|> Enum.filter(&(&1 != ""))
|> Enum.map(&String.trim_leading(&1, "#"))
|> Enum.filter(&(&1 != ""))
if Enum.empty?(search_terms) do
[]
else
# Use PostgreSQL's ANY operator with array for efficient multi-term search
# This is much more efficient than multiple OR clauses
search_patterns = Enum.map(search_terms, &"%#{&1}%")
# Create ranking query that prioritizes exact matches and closer matches
# Use a subquery to properly handle computed columns in ORDER BY
base_query =
from(ht in Hashtag,
where: fragment("LOWER(?) LIKE ANY(?)", ht.name, ^search_patterns),
select: %{
name: ht.name,
# Ranking: exact matches get highest priority (0)
# then prefix matches (1), then contains (2)
match_rank:
fragment(
"""
CASE
WHEN LOWER(?) = ANY(?) THEN 0
WHEN LOWER(?) LIKE ANY(?) THEN 1
ELSE 2
END
""",
ht.name,
^search_terms,
ht.name,
^Enum.map(search_terms, &"#{&1}%")
),
# Secondary sort by name length (shorter names first)
name_length: fragment("LENGTH(?)", ht.name)
}
)
from(result in subquery(base_query),
order_by: [
asc: result.match_rank,
asc: result.name_length,
asc: result.name
],
limit: ^limit,
offset: ^offset
)
|> Repo.all()
|> Enum.map(& &1.name)
end
end
end

57
lib/pleroma/http.ex
View file

@ -105,20 +105,57 @@ defmodule Pleroma.HTTP do
end
defp adapter_middlewares(Tesla.Adapter.Gun, extra_middleware) do
[Tesla.Middleware.FollowRedirects, Pleroma.Tesla.Middleware.ConnectionPool] ++
default_middleware() ++
[Pleroma.Tesla.Middleware.ConnectionPool] ++
extra_middleware
end
defp adapter_middlewares({Tesla.Adapter.Finch, _}, extra_middleware) do
[Tesla.Middleware.FollowRedirects] ++ extra_middleware
end
defp adapter_middlewares(_, extra_middleware) do
if Pleroma.Config.get(:env) == :test do
# Emulate redirects in test env, which are handled by adapters in other environments
[Tesla.Middleware.FollowRedirects]
else
extra_middleware
# A lot of tests are written expecting unencoded URLs
# and the burden of fixing that is high. Also it makes
# them hard to read. Tests will opt-in when we want to validate
# the encoding is being done correctly.
cond do
Pleroma.Config.get(:env) == :test and Pleroma.Config.get(:test_url_encoding) ->
default_middleware()
Pleroma.Config.get(:env) == :test ->
# Emulate redirects in test env, which are handled by adapters in other environments
[Tesla.Middleware.FollowRedirects]
# Hackney and Finch
true ->
default_middleware() ++ extra_middleware
end
end
defp default_middleware,
do: [Tesla.Middleware.FollowRedirects, Pleroma.Tesla.Middleware.EncodeUrl]
def encode_url(url) when is_binary(url) do
URI.parse(url)
|> then(fn parsed ->
path = encode_path(parsed.path)
query = encode_query(parsed.query)
%{parsed | path: path, query: query}
end)
|> URI.to_string()
end
defp encode_path(nil), do: nil
defp encode_path(path) when is_binary(path) do
path
|> URI.decode()
|> URI.encode()
end
defp encode_query(nil), do: nil
defp encode_query(query) when is_binary(query) do
query
|> URI.decode_query()
|> URI.encode_query()
end
end

37
lib/pleroma/instances.ex
View file

@ -15,25 +15,7 @@ defmodule Pleroma.Instances do
defdelegate set_unreachable(url_or_host, unreachable_since \\ nil), to: Instance
defdelegate get_consistently_unreachable, to: Instance
def set_consistently_unreachable(url_or_host),
do: set_unreachable(url_or_host, reachability_datetime_threshold())
def reachability_datetime_threshold do
federation_reachability_timeout_days =
Pleroma.Config.get([:instance, :federation_reachability_timeout_days], 0)
if federation_reachability_timeout_days > 0 do
NaiveDateTime.add(
NaiveDateTime.utc_now(),
-federation_reachability_timeout_days * 24 * 3600,
:second
)
else
~N[0000-01-01 00:00:00]
end
end
defdelegate get_unreachable, to: Instance
def host(url_or_host) when is_binary(url_or_host) do
if url_or_host =~ ~r/^http/i do
@ -42,4 +24,21 @@ defmodule Pleroma.Instances do
url_or_host
end
end
@doc "Schedules reachability checks for all unreachable instances"
def check_all_unreachable do
get_unreachable()
|> Enum.each(fn {domain, _} ->
Pleroma.Workers.ReachabilityWorker.new(%{"domain" => domain})
|> Oban.insert()
end)
end
@doc "Deletes all users and activities for unreachable instances"
def delete_all_unreachable do
get_unreachable()
|> Enum.each(fn {domain, _} ->
Instance.delete(domain)
end)
end
end

39
lib/pleroma/instances/instance.ex
View file

@ -50,7 +50,7 @@ defmodule Pleroma.Instances.Instance do
|> cast(params, [:software_name, :software_version, :software_repository])
end
def filter_reachable([]), do: %{}
def filter_reachable([]), do: []
def filter_reachable(urls_or_hosts) when is_list(urls_or_hosts) do
hosts =
@ -67,19 +67,15 @@ defmodule Pleroma.Instances.Instance do
)
|> Map.new(& &1)
reachability_datetime_threshold = Instances.reachability_datetime_threshold()
for entry <- Enum.filter(urls_or_hosts, &is_binary/1) do
host = host(entry)
unreachable_since = unreachable_since_by_host[host]
if !unreachable_since ||
NaiveDateTime.compare(unreachable_since, reachability_datetime_threshold) == :gt do
{entry, unreachable_since}
if is_nil(unreachable_since) do
entry
end
end
|> Enum.filter(& &1)
|> Map.new(& &1)
end
def reachable?(url_or_host) when is_binary(url_or_host) do
@ -87,7 +83,7 @@ defmodule Pleroma.Instances.Instance do
from(i in Instance,
where:
i.host == ^host(url_or_host) and
i.unreachable_since <= ^Instances.reachability_datetime_threshold(),
not is_nil(i.unreachable_since),
select: true
)
)
@ -96,9 +92,16 @@ defmodule Pleroma.Instances.Instance do
def reachable?(url_or_host) when is_binary(url_or_host), do: true
def set_reachable(url_or_host) when is_binary(url_or_host) do
%Instance{host: host(url_or_host)}
|> changeset(%{unreachable_since: nil})
|> Repo.insert(on_conflict: {:replace, [:unreachable_since]}, conflict_target: :host)
host = host(url_or_host)
result =
%Instance{host: host}
|> changeset(%{unreachable_since: nil})
|> Repo.insert(on_conflict: {:replace, [:unreachable_since]}, conflict_target: :host)
Pleroma.Workers.ReachabilityWorker.delete_jobs_for_host(host)
result
end
def set_reachable(_), do: {:error, nil}
@ -131,11 +134,9 @@ defmodule Pleroma.Instances.Instance do
def set_unreachable(_, _), do: {:error, nil}
def get_consistently_unreachable do
reachability_datetime_threshold = Instances.reachability_datetime_threshold()
def get_unreachable do
from(i in Instance,
where: ^reachability_datetime_threshold > i.unreachable_since,
where: not is_nil(i.unreachable_since),
order_by: i.unreachable_since,
select: {i.host, i.unreachable_since}
)
@ -295,8 +296,14 @@ defmodule Pleroma.Instances.Instance do
Deletes all users from an instance in a background task, thus also deleting
all of those users' activities and notifications.
"""
def delete_users_and_activities(host) when is_binary(host) do
def delete(host) when is_binary(host) do
DeleteWorker.new(%{"op" => "delete_instance", "host" => host})
|> Oban.insert()
end
@doc "Schedules reachability check for instance"
def check_unreachable(domain) when is_binary(domain) do
Pleroma.Workers.ReachabilityWorker.new(%{"domain" => domain})
|> Oban.insert()
end
end

2
lib/pleroma/language/translation/provider.ex
View file

@ -25,7 +25,7 @@ defmodule Pleroma.Language.Translation.Provider do
@callback supported_languages(type :: :string | :target) ::
{:ok, [String.t()]} | {:error, atom()}
@callback languages_matrix() :: {:ok, Map.t()} | {:error, atom()}
@callback languages_matrix() :: {:ok, map()} | {:error, atom()}
@callback name() :: String.t()

16
lib/pleroma/object/fetcher.ex
View file

@ -4,7 +4,6 @@
defmodule Pleroma.Object.Fetcher do
alias Pleroma.HTTP
alias Pleroma.Instances
alias Pleroma.Maps
alias Pleroma.Object
alias Pleroma.Object.Containment
@ -19,8 +18,6 @@ defmodule Pleroma.Object.Fetcher do
require Logger
require Pleroma.Constants
@mix_env Mix.env()
@spec reinject_object(struct(), map()) :: {:ok, Object.t()} | {:error, any()}
defp reinject_object(%Object{data: %{}} = object, new_data) do
Logger.debug("Reinjecting object #{new_data["id"]}")
@ -152,10 +149,6 @@ defmodule Pleroma.Object.Fetcher do
{:ok, body} <- get_object(id),
{:ok, data} <- safe_json_decode(body),
:ok <- Containment.contain_origin_from_id(id, data) do
if not Instances.reachable?(id) do
Instances.set_reachable(id)
end
{:ok, data}
else
{:scheme, _} ->
@ -178,13 +171,8 @@ defmodule Pleroma.Object.Fetcher do
def fetch_and_contain_remote_object_from_id(_id),
do: {:error, "id must be a string"}
defp check_crossdomain_redirect(final_host, original_url)
# Handle the common case in tests where responses don't include URLs
if @mix_env == :test do
defp check_crossdomain_redirect(nil, _) do
{:cross_domain_redirect, false}
end
defp check_crossdomain_redirect(final_host, _original_url) when is_nil(final_host) do
{:cross_domain_redirect, false}
end
defp check_crossdomain_redirect(final_host, original_url) do

16
lib/pleroma/reverse_proxy.ex
View file

@ -158,6 +158,8 @@ defmodule Pleroma.ReverseProxy do
Logger.debug("#{__MODULE__} #{method} #{url} #{inspect(headers)}")
method = method |> String.downcase() |> String.to_existing_atom()
url = maybe_encode_url(url)
case client().request(method, url, headers, "", opts) do
{:ok, code, headers, client} when code in @valid_resp_codes ->
{:ok, code, downcase_headers(headers), client}
@ -449,4 +451,18 @@ defmodule Pleroma.ReverseProxy do
_ -> delete_resp_header(conn, "content-length")
end
end
# Only when Tesla adapter is Hackney or Finch does the URL
# need encoding before Reverse Proxying as both end up
# using the raw Hackney client and cannot leverage our
# EncodeUrl Tesla middleware
# Also do it for test environment
defp maybe_encode_url(url) do
case Application.get_env(:tesla, :adapter) do
Tesla.Adapter.Hackney -> Pleroma.HTTP.encode_url(url)
{Tesla.Adapter.Finch, _} -> Pleroma.HTTP.encode_url(url)
Tesla.Mock -> Pleroma.HTTP.encode_url(url)
_ -> url
end
end
end

4
lib/pleroma/safe_zip.ex
View file

@ -56,10 +56,6 @@ defmodule Pleroma.SafeZip do
{_, true} <- {:safe_path, safe_path?(path)} do
{:cont, {:ok, maybe_add_file(type, path, fl)}}
else
{:get_type, e} ->
{:halt,
{:error, "Couldn't determine file type of ZIP entry at #{path} (#{inspect(e)})"}}
{:type, _} ->
{:halt, {:error, "Potentially unsafe file type in ZIP at: #{path}"}}

55
lib/pleroma/search/qdrant_search.ex
View file

@ -157,26 +157,55 @@ defmodule Pleroma.Search.QdrantSearch do
end
defmodule Pleroma.Search.QdrantSearch.OpenAIClient do
use Tesla
alias Pleroma.Config.Getting, as: Config
plug(Tesla.Middleware.BaseUrl, Config.get([Pleroma.Search.QdrantSearch, :openai_url]))
plug(Tesla.Middleware.JSON)
def post(path, body) do
Tesla.post(client(), path, body)
end
plug(Tesla.Middleware.Headers, [
{"Authorization",
"Bearer #{Pleroma.Config.get([Pleroma.Search.QdrantSearch, :openai_api_key])}"}
])
defp client do
Tesla.client(middleware())
end
defp middleware do
[
{Tesla.Middleware.BaseUrl, Config.get([Pleroma.Search.QdrantSearch, :openai_url])},
Tesla.Middleware.JSON,
{Tesla.Middleware.Headers,
[
{"Authorization", "Bearer #{Config.get([Pleroma.Search.QdrantSearch, :openai_api_key])}"}
]}
]
end
end
defmodule Pleroma.Search.QdrantSearch.QdrantClient do
use Tesla
alias Pleroma.Config.Getting, as: Config
plug(Tesla.Middleware.BaseUrl, Config.get([Pleroma.Search.QdrantSearch, :qdrant_url]))
plug(Tesla.Middleware.JSON)
def delete(path) do
Tesla.delete(client(), path)
end
plug(Tesla.Middleware.Headers, [
{"api-key", Pleroma.Config.get([Pleroma.Search.QdrantSearch, :qdrant_api_key])}
])
def post(path, body) do
Tesla.post(client(), path, body)
end
def put(path, body) do
Tesla.put(client(), path, body)
end
defp client do
Tesla.client(middleware())
end
defp middleware do
[
{Tesla.Middleware.BaseUrl, Config.get([Pleroma.Search.QdrantSearch, :qdrant_url])},
Tesla.Middleware.JSON,
{Tesla.Middleware.Headers,
[
{"api-key", Pleroma.Config.get([Pleroma.Search.QdrantSearch, :qdrant_api_key])}
]}
]
end
end

29
lib/pleroma/tesla/middleware/encode_url.ex Normal file
View file

@ -0,0 +1,29 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2025 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Tesla.Middleware.EncodeUrl do
@moduledoc """
Middleware to encode URLs properly
We must decode and then re-encode to ensure correct encoding.
If we only encode it will re-encode each % as %25 causing a space
already encoded as %20 to be %2520.
Similar problem for query parameters which need spaces to be the + character
"""
@behaviour Tesla.Middleware
@impl Tesla.Middleware
def call(%Tesla.Env{url: url} = env, next, _) do
url = Pleroma.HTTP.encode_url(url)
env = %{env | url: url}
case Tesla.run(env, next) do
{:ok, env} -> {:ok, env}
err -> err
end
end
end

38
lib/pleroma/web/activity_pub/activity_pub_controller.ex
View file

@ -53,7 +53,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
)
plug(:log_inbox_metadata when action in [:inbox])
plug(:set_requester_reachable when action in [:inbox])
plug(:relay_active? when action in [:relay])
defp relay_active?(conn, _) do
@ -274,13 +273,37 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
end
def inbox(%{assigns: %{valid_signature: true}} = conn, %{"nickname" => nickname} = params) do
with %User{is_active: true} = recipient <- User.get_cached_by_nickname(nickname),
{:ok, %User{is_active: true} = actor} <- User.get_or_fetch_by_ap_id(params["actor"]),
with {:recipient_exists, %User{} = recipient} <-
{:recipient_exists, User.get_cached_by_nickname(nickname)},
{:sender_exists, {:ok, %User{} = actor}} <-
{:sender_exists, User.get_or_fetch_by_ap_id(params["actor"])},
{:recipient_active, true} <- {:recipient_active, recipient.is_active},
{:sender_active, true} <- {:sender_active, actor.is_active},
true <- Utils.recipient_in_message(recipient, actor, params),
params <- Utils.maybe_splice_recipient(recipient.ap_id, params) do
Federator.incoming_ap_doc(params)
json(conn, "ok")
else
{:recipient_exists, _} ->
conn
|> put_status(:not_found)
|> json("User does not exist")
{:sender_exists, _} ->
conn
|> put_status(:not_found)
|> json("Sender does not exist")
{:recipient_active, _} ->
conn
|> put_status(:not_found)
|> json("User deactivated")
{:sender_active, _} ->
conn
|> put_status(:not_found)
|> json("Sender deactivated")
_ ->
conn
|> put_status(:bad_request)
@ -520,15 +543,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
|> json(dgettext("errors", "error"))
end
defp set_requester_reachable(%Plug.Conn{} = conn, _) do
with actor <- conn.params["actor"],
true <- is_binary(actor) do
Pleroma.Instances.set_reachable(actor)
end
conn
end
defp log_inbox_metadata(%{params: %{"actor" => actor, "type" => type}} = conn, _) do
Logger.metadata(actor: actor, type: type)
conn

1
lib/pleroma/web/activity_pub/mrf/remote_report_policy.ex
View file

@ -15,7 +15,6 @@ defmodule Pleroma.Web.ActivityPub.MRF.RemoteReportPolicy do
else
{:local, true} -> {:ok, object}
{:reject, message} -> {:reject, message}
error -> {:reject, error}
end
end

26
lib/pleroma/web/activity_pub/publisher.ex
View file

@ -161,17 +161,9 @@ defmodule Pleroma.Web.ActivityPub.Publisher do
{"digest", p.digest}
]
) do
if not is_nil(p.unreachable_since) do
Instances.set_reachable(p.inbox)
end
result
else
{_post_result, %{status: code} = response} = e ->
if is_nil(p.unreachable_since) do
Instances.set_unreachable(p.inbox)
end
Logger.metadata(activity: p.activity_id, inbox: p.inbox, status: code)
Logger.error("Publisher failed to inbox #{p.inbox} with status #{code}")
@ -192,10 +184,6 @@ defmodule Pleroma.Web.ActivityPub.Publisher do
connection_pool_snooze()
e ->
if is_nil(p.unreachable_since) do
Instances.set_unreachable(p.inbox)
end
Logger.metadata(activity: p.activity_id, inbox: p.inbox)
Logger.error("Publisher failed to inbox #{p.inbox} #{inspect(e)}")
{:error, e}
@ -307,7 +295,7 @@ defmodule Pleroma.Web.ActivityPub.Publisher do
[priority_recipients, recipients] = recipients(actor, activity)
inboxes =
[priority_inboxes, other_inboxes] =
[priority_recipients, recipients]
|> Enum.map(fn recipients ->
recipients
@ -320,8 +308,8 @@ defmodule Pleroma.Web.ActivityPub.Publisher do
end)
Repo.checkout(fn ->
Enum.each(inboxes, fn inboxes ->
Enum.each(inboxes, fn {inbox, unreachable_since} ->
Enum.each([priority_inboxes, other_inboxes], fn inboxes ->
Enum.each(inboxes, fn inbox ->
%User{ap_id: ap_id} = Enum.find(recipients, fn actor -> actor.inbox == inbox end)
# Get all the recipients on the same host and add them to cc. Otherwise, a remote
@ -331,8 +319,7 @@ defmodule Pleroma.Web.ActivityPub.Publisher do
__MODULE__.enqueue_one(%{
inbox: inbox,
cc: cc,
activity_id: activity.id,
unreachable_since: unreachable_since
activity_id: activity.id
})
end)
end)
@ -365,12 +352,11 @@ defmodule Pleroma.Web.ActivityPub.Publisher do
|> Enum.each(fn {inboxes, priority} ->
inboxes
|> Instances.filter_reachable()
|> Enum.each(fn {inbox, unreachable_since} ->
|> Enum.each(fn inbox ->
__MODULE__.enqueue_one(
%{
inbox: inbox,
activity_id: activity.id,
unreachable_since: unreachable_since
activity_id: activity.id
},
priority: priority
)

2
lib/pleroma/web/admin_api/controllers/instance_controller.ex
View file

@ -49,7 +49,7 @@ defmodule Pleroma.Web.AdminAPI.InstanceController do
end
def delete(conn, %{"instance" => instance}) do
with {:ok, _job} <- Instance.delete_users_and_activities(instance) do
with {:ok, _job} <- Instance.delete(instance) do
json(conn, instance)
end
end

2
lib/pleroma/web/api_spec.ex
View file

@ -158,6 +158,6 @@ defmodule Pleroma.Web.ApiSpec do
}
}
# discover request/response schemas from path specs
|> OpenApiSpex.resolve_schema_modules()
|> then(&OpenApiSpex.resolve_schema_modules/1)
end
end

33
lib/pleroma/web/api_spec/operations/pleroma_emoji_pack_operation.ex
View file

@ -127,6 +127,20 @@ defmodule Pleroma.Web.ApiSpec.PleromaEmojiPackOperation do
}
end
def download_zip_operation do
%Operation{
tags: ["Emoji pack administration"],
summary: "Download a pack from a URL or an uploaded file",
operationId: "PleromaAPI.EmojiPackController.download_zip",
security: [%{"oAuth" => ["admin:write"]}],
requestBody: request_body("Parameters", download_zip_request(), required: true),
responses: %{
200 => ok_response(),
400 => Operation.response("Bad Request", "application/json", ApiError)
}
}
end
defp download_request do
%Schema{
type: :object,
@ -143,6 +157,25 @@ defmodule Pleroma.Web.ApiSpec.PleromaEmojiPackOperation do
}
end
defp download_zip_request do
%Schema{
type: :object,
required: [:name],
properties: %{
url: %Schema{
type: :string,
format: :uri,
description: "URL of the file"
},
file: %Schema{
description: "The uploaded ZIP file",
type: :object
},
name: %Schema{type: :string, format: :uri, description: "Pack Name"}
}
}
end
def create_operation do
%Operation{
tags: ["Emoji pack administration"],

6
lib/pleroma/web/api_spec/scopes/compiler.ex
View file

@ -26,7 +26,11 @@ defmodule Pleroma.Web.ApiSpec.Scopes.Compiler do
end
def extract_all_scopes do
extract_all_scopes_from(Pleroma.Web.ApiSpec.spec())
try do
extract_all_scopes_from(Pleroma.Web.ApiSpec.spec())
catch
_, _ -> []
end
end
def extract_all_scopes_from(specs) do

60
lib/pleroma/web/mastodon_api/controllers/search_controller.ex
View file

@ -5,6 +5,7 @@
defmodule Pleroma.Web.MastodonAPI.SearchController do
use Pleroma.Web, :controller
alias Pleroma.Hashtag
alias Pleroma.Repo
alias Pleroma.User
alias Pleroma.Web.ControllerHelper
@ -120,69 +121,14 @@ defmodule Pleroma.Web.MastodonAPI.SearchController do
defp resource_search(:v2, "hashtags", query, options) do
tags_path = Endpoint.url() <> "/tag/"
query
|> prepare_tags(options)
Hashtag.search(query, options)
|> Enum.map(fn tag ->
%{name: tag, url: tags_path <> tag}
end)
end
defp resource_search(:v1, "hashtags", query, options) do
prepare_tags(query, options)
end
defp prepare_tags(query, options) do
tags =
query
|> preprocess_uri_query()
|> String.split(~r/[^#\w]+/u, trim: true)
|> Enum.uniq_by(&String.downcase/1)
explicit_tags = Enum.filter(tags, fn tag -> String.starts_with?(tag, "#") end)
tags =
if Enum.any?(explicit_tags) do
explicit_tags
else
tags
end
tags = Enum.map(tags, fn tag -> String.trim_leading(tag, "#") end)
tags =
if Enum.empty?(explicit_tags) && !options[:skip_joined_tag] do
add_joined_tag(tags)
else
tags
end
Pleroma.Pagination.paginate_list(tags, options)
end
defp add_joined_tag(tags) do
tags
|> Kernel.++([joined_tag(tags)])
|> Enum.uniq_by(&String.downcase/1)
end
# If `query` is a URI, returns last component of its path, otherwise returns `query`
defp preprocess_uri_query(query) do
if query =~ ~r/https?:\/\// do
query
|> String.trim_trailing("/")
|> URI.parse()
|> Map.get(:path)
|> String.split("/")
|> Enum.at(-1)
else
query
end
end
defp joined_tag(tags) do
tags
|> Enum.map(fn tag -> String.capitalize(tag) end)
|> Enum.join()
Hashtag.search(query, options)
end
defp with_fallback(f, fallback \\ []) do

3
lib/pleroma/web/mastodon_api/controllers/status_controller.ex
View file

@ -584,6 +584,9 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do
{:error, error} when error in [:unexpected_response, :quota_exceeded, :too_many_requests] ->
render_error(conn, :service_unavailable, "Translation service not available")
_ ->
render_error(conn, :internal_server_error, "Translation failed")
end
end

22
lib/pleroma/web/pleroma_api/controllers/emoji_pack_controller.ex
View file

@ -16,6 +16,7 @@ defmodule Pleroma.Web.PleromaAPI.EmojiPackController do
:import_from_filesystem,
:remote,
:download,
:download_zip,
:create,
:update,
:delete
@ -113,6 +114,27 @@ defmodule Pleroma.Web.PleromaAPI.EmojiPackController do
end
end
def download_zip(
%{private: %{open_api_spex: %{body_params: params}}} = conn,
_
) do
name = Map.get(params, :name)
with :ok <- Pack.download_zip(name, params) do
json(conn, "ok")
else
{:error, error} when is_binary(error) ->
conn
|> put_status(:bad_request)
|> json(%{error: error})
{:error, _} ->
conn
|> put_status(:bad_request)
|> json(%{error: "Could not process pack"})
end
end
def download(
%{private: %{open_api_spex: %{body_params: %{url: url, name: name} = params}}} = conn,
_

2
lib/pleroma/web/pleroma_api/controllers/instances_controller.ex
View file

@ -13,7 +13,7 @@ defmodule Pleroma.Web.PleromaAPI.InstancesController do
def show(conn, _params) do
unreachable =
Instances.get_consistently_unreachable()
Instances.get_unreachable()
|> Map.new(fn {host, date} -> {host, to_string(date)} end)
json(conn, %{"unreachable" => unreachable})

1
lib/pleroma/web/router.ex
View file

@ -466,6 +466,7 @@ defmodule Pleroma.Web.Router do
get("/import", EmojiPackController, :import_from_filesystem)
get("/remote", EmojiPackController, :remote)
post("/download", EmojiPackController, :download)
post("/download_zip", EmojiPackController, :download_zip)
post("/files", EmojiFileController, :create)
patch("/files", EmojiFileController, :update)

12
lib/pleroma/workers/delete_worker.ex
View file

@ -14,6 +14,7 @@ defmodule Pleroma.Workers.DeleteWorker do
end
def perform(%Job{args: %{"op" => "delete_instance", "host" => host}}) do
# Schedule the per-user deletion jobs
Pleroma.Repo.transaction(fn ->
User.Query.build(%{nickname: "@#{host}"})
|> Pleroma.Repo.all()
@ -22,6 +23,17 @@ defmodule Pleroma.Workers.DeleteWorker do
|> __MODULE__.new()
|> Oban.insert()
end)
# Delete the instance from the Instances table
case Pleroma.Repo.get_by(Pleroma.Instances.Instance, host: host) do
nil -> :ok
instance -> Pleroma.Repo.delete(instance)
end
# Delete any pending ReachabilityWorker jobs for this domain
Pleroma.Workers.ReachabilityWorker.delete_jobs_for_host(host)
:ok
end)
end

28
lib/pleroma/workers/publisher_worker.ex
View file

@ -4,9 +4,10 @@
defmodule Pleroma.Workers.PublisherWorker do
alias Pleroma.Activity
alias Pleroma.Instances
alias Pleroma.Web.Federator
use Oban.Worker, queue: :federator_outgoing, max_attempts: 5
use Oban.Worker, queue: :federator_outgoing, max_attempts: 13
@impl true
def perform(%Job{args: %{"op" => "publish", "activity_id" => activity_id}}) do
@ -14,9 +15,30 @@ defmodule Pleroma.Workers.PublisherWorker do
Federator.perform(:publish, activity)
end
def perform(%Job{args: %{"op" => "publish_one", "params" => params}}) do
def perform(%Job{args: %{"op" => "publish_one", "params" => params}} = job) do
params = Map.new(params, fn {k, v} -> {String.to_atom(k), v} end)
Federator.perform(:publish_one, params)
# Cancel / skip the job if this server believed to be unreachable now
if not Instances.reachable?(params.inbox) do
{:cancel, :unreachable}
else
case Federator.perform(:publish_one, params) do
{:ok, _} ->
:ok
{:error, _} = error ->
# Only mark as unreachable on final failure
if job.attempt == job.max_attempts do
Instances.set_unreachable(params.inbox)
end
error
error ->
# Unexpected error, may have been client side
error
end
end
end
@impl true

116
lib/pleroma/workers/reachability_worker.ex Normal file
View file

@ -0,0 +1,116 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Workers.ReachabilityWorker do
use Oban.Worker,
queue: :background,
max_attempts: 1,
unique: [period: :infinity, states: [:available, :scheduled], keys: [:domain]]
alias Pleroma.HTTP
alias Pleroma.Instances
import Ecto.Query
@impl true
def perform(%Oban.Job{args: %{"domain" => domain, "phase" => phase, "attempt" => attempt}}) do
case check_reachability(domain) do
:ok ->
Instances.set_reachable("https://#{domain}")
:ok
{:error, _} = error ->
handle_failed_attempt(domain, phase, attempt)
error
end
end
# New jobs enter here and are immediately re-scheduled for the first phase
@impl true
def perform(%Oban.Job{args: %{"domain" => domain}}) do
scheduled_at = DateTime.add(DateTime.utc_now(), 60, :second)
%{
"domain" => domain,
"phase" => "phase_1min",
"attempt" => 1
}
|> new(scheduled_at: scheduled_at, replace: true)
|> Oban.insert()
:ok
end
@impl true
def timeout(_job), do: :timer.seconds(5)
@doc "Deletes scheduled jobs to check reachability for specified instance"
def delete_jobs_for_host(host) do
Oban.Job
|> where(worker: "Pleroma.Workers.ReachabilityWorker")
|> where([j], j.args["domain"] == ^host)
|> Oban.delete_all_jobs()
end
defp check_reachability(domain) do
case HTTP.get("https://#{domain}/") do
{:ok, %{status: status}} when status in 200..299 ->
:ok
{:ok, %{status: _status}} ->
{:error, :unreachable}
{:error, _} = error ->
error
end
end
defp handle_failed_attempt(_domain, "final", _attempt), do: :ok
defp handle_failed_attempt(domain, phase, attempt) do
{interval_minutes, max_attempts, next_phase} = get_phase_config(phase)
if attempt >= max_attempts do
# Move to next phase
schedule_next_phase(domain, next_phase)
else
# Retry same phase with incremented attempt
schedule_retry(domain, phase, attempt + 1, interval_minutes)
end
end
defp get_phase_config("phase_1min"), do: {1, 4, "phase_15min"}
defp get_phase_config("phase_15min"), do: {15, 4, "phase_1hour"}
defp get_phase_config("phase_1hour"), do: {60, 4, "phase_8hour"}
defp get_phase_config("phase_8hour"), do: {480, 4, "phase_24hour"}
defp get_phase_config("phase_24hour"), do: {1440, 4, "final"}
defp get_phase_config("final"), do: {nil, 0, nil}
defp schedule_next_phase(_domain, "final"), do: :ok
defp schedule_next_phase(domain, next_phase) do
{interval_minutes, _max_attempts, _next_phase} = get_phase_config(next_phase)
scheduled_at = DateTime.add(DateTime.utc_now(), interval_minutes * 60, :second)
%{
"domain" => domain,
"phase" => next_phase,
"attempt" => 1
}
|> new(scheduled_at: scheduled_at, replace: true)
|> Oban.insert()
end
def schedule_retry(domain, phase, attempt, interval_minutes) do
scheduled_at = DateTime.add(DateTime.utc_now(), interval_minutes * 60, :second)
%{
"domain" => domain,
"phase" => phase,
"attempt" => attempt
}
|> new(scheduled_at: scheduled_at, replace: true)
|> Oban.insert()
end
end

11
lib/pleroma/workers/receiver_worker.ex
View file

@ -3,6 +3,7 @@
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Workers.ReceiverWorker do
alias Pleroma.Instances
alias Pleroma.Signature
alias Pleroma.User
alias Pleroma.Web.Federator
@ -37,6 +38,11 @@ defmodule Pleroma.Workers.ReceiverWorker do
{:ok, _public_key} <- Signature.refetch_public_key(conn_data),
{:signature, true} <- {:signature, Signature.validate_signature(conn_data)},
{:ok, res} <- Federator.perform(:incoming_ap_doc, params) do
unless Instances.reachable?(params["actor"]) do
domain = URI.parse(params["actor"]).host
Oban.insert(Pleroma.Workers.ReachabilityWorker.new(%{"domain" => domain}))
end
{:ok, res}
else
e -> process_errors(e)
@ -45,6 +51,11 @@ defmodule Pleroma.Workers.ReceiverWorker do
def perform(%Job{args: %{"op" => "incoming_ap_doc", "params" => params}}) do
with {:ok, res} <- Federator.perform(:incoming_ap_doc, params) do
unless Instances.reachable?(params["actor"]) do
domain = URI.parse(params["actor"]).host
Oban.insert(Pleroma.Workers.ReachabilityWorker.new(%{"domain" => domain}))
end
{:ok, res}
else
e -> process_errors(e)

6
lib/pleroma/workers/remote_fetcher_worker.ex
View file

@ -3,6 +3,7 @@
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Workers.RemoteFetcherWorker do
alias Pleroma.Instances
alias Pleroma.Object.Fetcher
use Oban.Worker, queue: :background, unique: [period: :infinity]
@ -11,6 +12,11 @@ defmodule Pleroma.Workers.RemoteFetcherWorker do
def perform(%Job{args: %{"op" => "fetch_remote", "id" => id} = args}) do
case Fetcher.fetch_object_from_id(id, depth: args["depth"]) do
{:ok, _object} ->
unless Instances.reachable?(id) do
# Mark the server as reachable since we successfully fetched an object
Instances.set_reachable(id)
end
:ok
{:allowed_depth, false} ->