hj/pleroma
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

feat(http): rework Gun connection pooling

Browse source
This commit is contained in:
Lain Soykaf 2026-07-17 20:35:17 +04:00
commit f11ffca7f5
No known key found for this signature in database
22 changed files with 2133 additions and 192 deletions
Download patch file Download diff file Expand all files Collapse all files

9
lib/pleroma/gun.ex
View file

@ -6,9 +6,12 @@ defmodule Pleroma.Gun do
@callback open(charlist(), pos_integer(), map()) :: {:ok, pid()}
@callback info(pid()) :: map()
@callback close(pid()) :: :ok
@callback cancel(pid(), reference() | [reference()]) :: :ok
@callback await_up(pid, pos_integer()) :: {:ok, atom()} | {:error, atom()}
@callback connect(pid(), map()) :: reference()
@callback await(pid(), reference()) :: {:response, :fin, 200, []}
@callback await(pid(), reference()) :: tuple()
@callback await_tunnel_up(pid(), reference() | :undefined, timeout()) ::
{:ok, atom()} | {:error, term()}
@callback set_owner(pid(), pid()) :: :ok
defp api, do: Pleroma.Config.get([Pleroma.Gun], Pleroma.Gun.API)
@ -19,11 +22,15 @@ defmodule Pleroma.Gun do
def close(pid), do: api().close(pid)
def cancel(pid, stream), do: api().cancel(pid, stream)
def await_up(pid, timeout \\ 5_000), do: api().await_up(pid, timeout)
def connect(pid, opts), do: api().connect(pid, opts)
def await(pid, ref), do: api().await(pid, ref)
def await_tunnel_up(pid, ref, timeout), do: api().await_tunnel_up(pid, ref, timeout)
def set_owner(pid, owner), do: api().set_owner(pid, owner)
end

25
lib/pleroma/gun/api.ex
View file

@ -32,6 +32,9 @@ defmodule Pleroma.Gun.API do
@impl Gun
defdelegate close(pid), to: :gun
@impl Gun
defdelegate cancel(pid, stream), to: :gun
@impl Gun
defdelegate await_up(pid, timeout \\ 5_000), to: :gun
@ -41,6 +44,28 @@ defmodule Pleroma.Gun.API do
@impl Gun
defdelegate await(pid, ref), to: :gun
@impl Gun
def await_tunnel_up(pid, stream_ref, timeout) do
monitor_ref = Process.monitor(pid)
result =
receive do
{:gun_tunnel_up, ^pid, ^stream_ref, protocol} ->
{:ok, protocol}
{:gun_down, ^pid, _protocol, reason, _killed_streams} ->
{:error, {:down, reason}}
{:DOWN, ^monitor_ref, :process, ^pid, reason} ->
{:error, {:down, reason}}
after
timeout -> {:error, :timeout}
end
Process.demonitor(monitor_ref, [:flush])
result
end
@impl Gun
defdelegate set_owner(pid, owner), to: :gun
end

124
lib/pleroma/gun/conn.ex
View file

@ -47,13 +47,14 @@ defmodule Pleroma.Gun.Conn do
uri
|> destination_opts()
|> add_http2_opts(uri.scheme, Map.get(opts, :tls_opts, []))
|> add_proxy_auth(opts)
with open_opts <- Map.delete(opts, :tls_opts),
with open_opts <- proxy_open_opts(opts),
{:ok, conn} <- Gun.open(proxy_host, proxy_port, open_opts),
{:ok, protocol} <- Gun.await_up(conn, opts[:connect_timeout]),
stream <- Gun.connect(conn, connect_opts),
{:response, :fin, 200, _} <- Gun.await(conn, stream) do
{:ok, conn, protocol}
{:ok, _proxy_protocol} <- await_up(conn, opts[:connect_timeout]) do
conn
|> Gun.connect(connect_opts)
|> await_proxy_connect(conn, opts[:connect_timeout])
else
error ->
Logger.warning(
@ -64,30 +65,29 @@ defmodule Pleroma.Gun.Conn do
end
end
defp do_open(uri, %{proxy: {proxy_type, proxy_host, proxy_port}} = opts) do
version =
proxy_type
|> to_string()
|> String.last()
|> case do
"4" -> 4
_ -> 5
end
defp do_open(_uri, %{proxy: {:socks4, _proxy_host, _proxy_port}}) do
{:error, :socks4_unsupported}
end
defp do_open(uri, %{proxy: {proxy_type, proxy_host, proxy_port}} = opts)
when proxy_type in [:socks, :socks5] do
socks_opts =
uri
|> destination_opts()
|> add_http2_opts(uri.scheme, Map.get(opts, :tls_opts, []))
|> Map.put(:version, version)
|> Map.put(:version, 5)
|> add_socks_proxy_auth(opts)
opts =
open_opts =
opts
|> proxy_open_opts()
|> Map.put(:protocols, [:socks])
|> Map.put(:socks_opts, socks_opts)
with {:ok, conn} <- Gun.open(proxy_host, proxy_port, opts),
{:ok, protocol} <- Gun.await_up(conn, opts[:connect_timeout]) do
{:ok, conn, protocol}
with {:ok, conn} <- Gun.open(proxy_host, proxy_port, open_opts),
{:ok, :socks} <- await_up(conn, opts[:connect_timeout]),
{:ok, protocol} <- await_tunnel_up(conn, :undefined, opts[:connect_timeout]) do
{:ok, conn, protocol, nil}
else
error ->
Logger.warning(
@ -100,10 +100,11 @@ defmodule Pleroma.Gun.Conn do
defp do_open(%URI{host: host, port: port} = uri, opts) do
host = Pleroma.HTTP.AdapterHelper.parse_host(host)
opts = Map.put(opts, :transport, transport(uri.scheme))
with {:ok, conn} <- Gun.open(host, port, opts),
{:ok, protocol} <- Gun.await_up(conn, opts[:connect_timeout]) do
{:ok, conn, protocol}
{:ok, protocol} <- await_up(conn, opts[:connect_timeout]) do
{:ok, conn, protocol, nil}
else
error ->
Logger.warning(
@ -120,11 +121,90 @@ defmodule Pleroma.Gun.Conn do
end
defp add_http2_opts(opts, "https", tls_opts) do
Map.merge(opts, %{protocols: [:http2], transport: :tls, tls_opts: tls_opts})
Map.merge(opts, %{protocols: [:http2, :http], transport: :tls, tls_opts: tls_opts})
end
defp add_http2_opts(opts, _, _), do: opts
defp add_proxy_auth(connect_opts, %{proxy_auth: {username, password}})
when is_binary(username) and is_binary(password) do
Map.merge(connect_opts, %{username: username, password: password})
end
defp add_proxy_auth(connect_opts, _), do: connect_opts
defp add_socks_proxy_auth(socks_opts, %{proxy_auth: {username, password}})
when is_binary(username) and is_binary(password) do
Map.put(socks_opts, :auth, [{:username_password, username, password}])
end
defp add_socks_proxy_auth(socks_opts, _), do: socks_opts
defp proxy_open_opts(opts) do
proxy_tls_opts = Map.get(opts, :proxy_tls_opts, [])
opts =
opts
|> Map.drop([:proxy, :proxy_auth, :proxy_tls_opts, :tls_opts])
|> Map.put_new(:transport, :tcp)
if opts.transport == :tls do
Map.put(opts, :tls_opts, proxy_tls_opts)
else
opts
end
end
defp await_up(conn, timeout) do
case Gun.await_up(conn, timeout) do
{:ok, protocol} ->
{:ok, protocol}
error ->
Gun.close(conn)
error
end
end
defp await_proxy_connect(stream, conn, timeout) do
case Gun.await(conn, stream) do
{:response, :fin, 200, _headers} ->
case await_tunnel_up(conn, stream, timeout) do
{:ok, protocol} -> {:ok, conn, protocol, stream}
error -> error
end
{:response, _fin, 403, _headers} ->
close_with_error(conn, :unauthorized)
{:response, _fin, 407, _headers} ->
close_with_error(conn, :proxy_auth_failed)
error ->
Gun.close(conn)
error
end
end
defp close_with_error(conn, reason) do
Gun.close(conn)
{:error, reason}
end
defp transport("https"), do: :tls
defp transport(_), do: :tcp
defp await_tunnel_up(conn, stream_ref, timeout) do
case Gun.await_tunnel_up(conn, stream_ref, timeout) do
{:ok, protocol} ->
{:ok, protocol}
error ->
Gun.close(conn)
error
end
end
def compose_uri_log(%URI{scheme: scheme, host: host, path: path}) do
"#{scheme}://#{host}#{path}"
end

225
lib/pleroma/gun/connection_pool.ex
View file

@ -4,53 +4,132 @@
defmodule Pleroma.Gun.ConnectionPool do
@registry __MODULE__
@locks Module.concat(__MODULE__, Locks)
alias Pleroma.Gun.ConnectionPool.WorkerSupervisor
@connection_option_keys [
:http_opts,
:http2_opts,
:protocols,
:proxy,
:proxy_auth,
:proxy_tls_opts,
:retry,
:retry_timeout,
:socks_opts,
:tcp_opts,
:tls_opts,
:transport,
:ws_opts
]
def children do
[
{Registry, keys: :unique, name: @registry},
{Registry, keys: :duplicate, name: @registry},
{Registry, keys: :unique, name: @locks},
Pleroma.Gun.ConnectionPool.WorkerSupervisor
]
end
@spec get_conn(URI.t(), keyword()) :: {:ok, pid()} | {:error, term()}
def get_conn(uri, opts) do
key = "#{uri.scheme}:#{uri.host}:#{uri.port}"
key = connection_key(uri, opts)
case Registry.lookup(@registry, key) do
# The key has already been registered, but connection is not up yet
[{worker_pid, nil}] ->
get_gun_pid_from_worker(worker_pid, true)
[{worker_pid, {gun_pid, _used_by, _crf, _last_reference}}] ->
GenServer.call(worker_pid, :add_client)
case checkout_existing(Registry.lookup(@registry, key)) do
{:ok, gun_pid} ->
{:ok, gun_pid}
[] ->
# :gun.set_owner fails in :connected state for whatevever reason,
# so we open the connection in the process directly and send it's pid back
# We trust gun to handle timeouts by itself
case WorkerSupervisor.start_worker([key, uri, opts, self()]) do
{:ok, worker_pid} ->
get_gun_pid_from_worker(worker_pid, false)
:not_available ->
provision_or_wait(key, uri, opts)
end
end
{:error, {:already_started, worker_pid}} ->
get_gun_pid_from_worker(worker_pid, true)
defp checkout_existing([]), do: :not_available
err ->
err
defp checkout_existing([{_worker_pid, nil} | workers]), do: checkout_existing(workers)
defp checkout_existing([{worker_pid, _value} | workers]) do
case checkout_worker(worker_pid) do
{:ok, gun_pid} -> {:ok, gun_pid}
_ -> checkout_existing(workers)
end
end
defp provision_or_wait(key, uri, opts) do
result = with_route_lock(key, fn -> provision(key, uri, opts) end)
case result do
{:ok, gun_pid} ->
{:ok, gun_pid}
{:new, worker_pid} ->
get_gun_pid_from_worker(worker_pid)
{:wait, worker_pid} ->
case checkout_worker(worker_pid, :infinity) do
{:ok, gun_pid} -> {:ok, gun_pid}
:busy -> get_conn(uri, opts)
error -> error
end
error ->
error
end
end
defp provision(key, uri, opts) do
workers = Registry.lookup(@registry, key)
case checkout_existing(workers) do
{:ok, gun_pid} ->
{:ok, gun_pid}
:not_available ->
case Enum.find(workers, fn {_worker_pid, value} -> is_nil(value) end) do
{worker_pid, nil} -> {:wait, worker_pid}
nil -> start_worker(key, uri, opts)
end
end
end
defp get_gun_pid_from_worker(worker_pid, register) do
defp with_route_lock(key, function) do
case Registry.register(@locks, key, nil) do
{:ok, _} ->
try do
function.()
after
Registry.unregister(@locks, key)
end
{:error, {:already_registered, _owner}} ->
Process.sleep(1)
with_route_lock(key, function)
end
end
defp start_worker(key, uri, opts) do
# :gun.set_owner fails in :connected state, so the worker opens the connection itself.
case WorkerSupervisor.start_worker([key, uri, opts, self()]) do
{:ok, worker_pid} -> {:new, worker_pid}
error -> error
end
end
defp checkout_worker(worker_pid, timeout \\ 5_000) do
try do
GenServer.call(worker_pid, :checkout, timeout)
catch
:exit, reason -> {:error, {:connection_worker_exit, reason}}
end
end
defp get_gun_pid_from_worker(worker_pid) do
# GenServer.call will block the process for timeout length if
# the server crashes on startup (which will happen if gun fails to connect)
# so instead we use cast + monitor
ref = Process.monitor(worker_pid)
if register, do: GenServer.cast(worker_pid, {:add_client, self()})
receive do
{:conn_pid, pid} ->
@ -68,19 +147,115 @@ defmodule Pleroma.Gun.ConnectionPool do
@spec release_conn(pid()) :: :ok
def release_conn(conn_pid) do
# :ets.fun2ms(fn {_, {worker_pid, {gun_pid, _, _, _}}} when gun_pid == conn_pid ->
# :ets.fun2ms(fn {_, {worker_pid, {gun_pid, _tunnel}}} when gun_pid == conn_pid ->
# worker_pid end)
query_result =
Registry.select(@registry, [
{{:_, :"$1", {:"$2", :_, :_, :_}}, [{:==, :"$2", conn_pid}], [:"$1"]}
{{:_, :"$1", {:"$2", :_}}, [{:==, :"$2", conn_pid}], [:"$1"]}
])
case query_result do
[worker_pid] ->
GenServer.call(worker_pid, :remove_client)
try do
GenServer.call(worker_pid, :remove_client)
catch
:exit, _ -> :ok
end
[] ->
:ok
end
end
@spec discard_conn(pid()) :: :ok
def discard_conn(conn_pid) do
case worker_for_conn(conn_pid) do
nil ->
:ok
worker_pid ->
DynamicSupervisor.terminate_child(WorkerSupervisor, worker_pid)
end
end
@spec cancel_stream(pid(), reference() | [reference()]) :: :ok
def cancel_stream(conn_pid, stream) do
case worker_for_conn(conn_pid) do
nil ->
Pleroma.Gun.cancel(conn_pid, stream)
worker_pid ->
try do
GenServer.call(worker_pid, {:cancel_stream, stream})
catch
:exit, _ -> :ok
end
end
end
@spec register_stream(pid(), reference() | [reference()]) :: :ok
def register_stream(conn_pid, stream) do
call_worker(conn_pid, {:register_stream, stream})
end
@spec finish_stream(pid(), reference() | [reference()]) :: :ok
def finish_stream(conn_pid, stream) do
call_worker(conn_pid, {:finish_stream, stream})
end
@spec release_stream(pid(), reference() | [reference()]) :: :ok
def release_stream(conn_pid, stream) do
call_worker(conn_pid, {:release_stream, stream})
end
@spec tunnel_ref(pid()) :: reference() | nil
def tunnel_ref(conn_pid) do
case Registry.select(@registry, [
{{:_, :_, {:"$1", :"$2"}}, [{:==, :"$1", conn_pid}], [:"$2"]}
]) do
[tunnel] -> tunnel
[] -> nil
end
end
defp worker_for_conn(conn_pid) do
case Registry.select(@registry, [
{{:_, :"$1", {:"$2", :_}}, [{:==, :"$2", conn_pid}], [:"$1"]}
]) do
[worker_pid] -> worker_pid
[] -> nil
end
end
defp call_worker(conn_pid, message) do
case worker_for_conn(conn_pid) do
nil ->
:ok
worker_pid ->
try do
GenServer.call(worker_pid, message)
catch
:exit, _ -> :ok
end
end
end
defp connection_key(uri, opts) do
origin = {
String.downcase(uri.scheme),
String.downcase(uri.host),
uri.port
}
profile =
opts
|> Map.new()
|> Map.take(@connection_option_keys)
|> :erlang.term_to_binary()
|> then(&:crypto.hash(:sha256, &1))
fingerprint = Base.encode16(profile, case: :lower)
Enum.join(Tuple.to_list(origin) ++ [fingerprint], ":")
end
end

51
lib/pleroma/gun/connection_pool/reclaimer.ex
View file

@ -9,11 +9,11 @@ defmodule Pleroma.Gun.ConnectionPool.Reclaimer do
def start_monitor do
pid =
case GenServer.start(__MODULE__, [], name: {:via, Registry, {registry(), "reclaimer"}}) do
case GenServer.start(__MODULE__, [], name: __MODULE__) do
{:ok, pid} ->
pid
{:error, {:already_registered, pid}} ->
{:error, {:already_started, pid}} ->
pid
end
@ -41,17 +41,16 @@ defmodule Pleroma.Gun.ConnectionPool.Reclaimer do
reclaim_max: reclaim_max
})
# :ets.fun2ms(
# fn {_, {worker_pid, {_, used_by, crf, last_reference}}} when used_by == [] ->
# {worker_pid, crf, last_reference} end)
unused_conns =
workers =
Registry.select(
registry(),
[
{{:_, :"$1", {:_, :"$2", :"$3", :"$4"}}, [{:==, :"$2", []}], [{{:"$1", :"$3", :"$4"}}]}
{{:_, :"$1", {:_, :_}}, [], [:"$1"]}
]
)
unused_conns = Enum.flat_map(workers, &reclaim_info/1)
case unused_conns do
[] ->
:telemetry.execute(
@ -65,25 +64,43 @@ defmodule Pleroma.Gun.ConnectionPool.Reclaimer do
{:stop, :no_unused_conns, nil}
unused_conns ->
reclaimed =
reclaimed_count =
unused_conns
|> Enum.sort(fn {_pid1, crf1, last_reference1}, {_pid2, crf2, last_reference2} ->
crf1 <= crf2 and last_reference1 <= last_reference2
end)
|> Enum.take(reclaim_max)
|> Enum.sort_by(fn {_pid, crf, last_reference} -> {crf, last_reference} end)
|> Enum.reduce_while(0, fn
_worker, count when count == reclaim_max ->
{:halt, count}
reclaimed
|> Enum.each(fn {pid, _, _} ->
DynamicSupervisor.terminate_child(Pleroma.Gun.ConnectionPool.WorkerSupervisor, pid)
end)
{worker_pid, _crf, _last_reference}, count ->
if reclaim(worker_pid), do: {:cont, count + 1}, else: {:cont, count}
end)
:telemetry.execute(
[:pleroma, :connection_pool, :reclaim, :stop],
%{reclaimed_count: Enum.count(reclaimed)},
%{reclaimed_count: reclaimed_count},
%{max_connections: max_connections}
)
{:stop, :normal, nil}
end
end
defp reclaim_info(worker_pid) do
try do
case GenServer.call(worker_pid, :reclaim_info) do
{:idle, crf, last_reference} -> [{worker_pid, crf, last_reference}]
:in_use -> []
end
catch
:exit, _ -> []
end
end
defp reclaim(worker_pid) do
try do
GenServer.call(worker_pid, :reclaim) == :reclaimed
catch
:exit, _ -> false
end
end
end

268
lib/pleroma/gun/connection_pool/worker.ex
View file

@ -8,25 +8,22 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do
defp registry, do: Pleroma.Gun.ConnectionPool
def start_link([key | _] = opts) do
GenServer.start_link(__MODULE__, opts, name: {:via, Registry, {registry(), key}})
end
def start_link(opts), do: GenServer.start_link(__MODULE__, opts)
@impl true
def init([_key, _uri, _opts, _client_pid] = opts) do
def init([key, _uri, _opts, _client_pid] = opts) do
{:ok, _} = Registry.register(registry(), key, nil)
{:ok, nil, {:continue, {:connect, opts}}}
end
@impl true
def handle_continue({:connect, [key, uri, opts, client_pid]}, _) do
with {:ok, conn_pid, protocol} <- Gun.Conn.open(uri, opts),
with {:ok, conn_pid, protocol, tunnel} <- Gun.Conn.open(uri, opts),
Process.link(conn_pid) do
time = :erlang.monotonic_time(:millisecond)
{_, _} =
Registry.update_value(registry(), key, fn _ ->
{conn_pid, [client_pid], 1, time}
end)
Registry.unregister(registry(), key)
{:ok, _} = Registry.register(registry(), key, {conn_pid, tunnel})
send(client_pid, {:conn_pid, conn_pid})
@ -34,8 +31,14 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do
%{
key: key,
timer: nil,
client_monitors: %{client_pid => Process.monitor(client_pid)},
protocol: protocol
clients: %{
client_pid => %{count: 1, monitor: Process.monitor(client_pid), streams: MapSet.new()}
},
conn_pid: conn_pid,
protocol: protocol,
proxied: Keyword.has_key?(opts, :proxy),
crf: 1,
last_reference: time
}, :hibernate}
else
err ->
@ -44,106 +47,225 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do
end
@impl true
def handle_cast({:add_client, client_pid}, state) do
case handle_call(:add_client, {client_pid, nil}, state) do
{:reply, conn_pid, state, :hibernate} ->
send(client_pid, {:conn_pid, conn_pid})
{:noreply, state, :hibernate}
end
def handle_call(:checkout, _from, %{protocol: protocol, clients: clients} = state)
when protocol != :http2 and map_size(clients) > 0 do
{:reply, :busy, state, :hibernate}
end
@impl true
def handle_cast({:remove_client, client_pid}, state) do
case handle_call(:remove_client, {client_pid, nil}, state) do
{:reply, _, state, :hibernate} ->
{:noreply, state, :hibernate}
end
end
@impl true
def handle_call(:add_client, {client_pid, _}, %{key: key, protocol: protocol} = state) do
def handle_call(:checkout, {client_pid, _}, %{key: key, protocol: protocol} = state) do
time = :erlang.monotonic_time(:millisecond)
{{conn_pid, used_by, _, _}, _} =
Registry.update_value(registry(), key, fn {conn_pid, used_by, crf, last_reference} ->
{conn_pid, [client_pid | used_by], crf(time - last_reference, crf), time}
end)
[{_, {conn_pid, _tunnel}}] =
Registry.lookup(registry(), key) |> Enum.filter(&(elem(&1, 0) == self()))
state =
state
|> cancel_idle_timer()
|> add_client(client_pid)
|> Map.put(:crf, crf(time - state.last_reference, state.crf))
|> Map.put(:last_reference, time)
:telemetry.execute(
[:pleroma, :connection_pool, :client, :add],
%{client_pid: client_pid, clients: used_by},
%{client_pid: client_pid, clients: Map.keys(state.clients)},
%{key: state.key, protocol: protocol}
)
state =
if state.timer != nil do
Process.cancel_timer(state[:timer])
%{state | timer: nil}
else
state
end
ref = Process.monitor(client_pid)
state = put_in(state.client_monitors[client_pid], ref)
{:reply, conn_pid, state, :hibernate}
{:reply, {:ok, conn_pid}, state, :hibernate}
end
@impl true
def handle_call(:remove_client, {client_pid, _}, %{key: key} = state) do
{{_conn_pid, used_by, _crf, _last_reference}, _} =
Registry.update_value(registry(), key, fn {conn_pid, used_by, crf, last_reference} ->
{conn_pid, List.delete(used_by, client_pid), crf, last_reference}
end)
def handle_call(:remove_client, {client_pid, _}, state) do
case active_streams(state, client_pid) do
[] ->
{:reply, :ok, remove_client(state, client_pid, true), :hibernate}
{ref, state} = pop_in(state.client_monitors[client_pid])
streams when state.protocol == :http2 ->
Enum.each(streams, &(:ok = Gun.cancel(state.conn_pid, &1)))
{:reply, :ok, remove_client(state, client_pid, true), :hibernate}
Process.demonitor(ref, [:flush])
_streams ->
{:stop, :normal, :ok, state}
end
end
timer =
if used_by == [] do
max_idle = Pleroma.Config.get([:connections_pool, :max_idle_time], 30_000)
Process.send_after(self(), :idle_close, max_idle)
else
nil
end
def handle_call({:register_stream, stream}, {client_pid, _}, state) do
{:reply, :ok, update_stream(state, client_pid, &MapSet.put(&1, stream)), :hibernate}
end
{:reply, :ok, %{state | timer: timer}, :hibernate}
def handle_call({:finish_stream, stream}, {client_pid, _}, state) do
{:reply, :ok, update_stream(state, client_pid, &MapSet.delete(&1, stream)), :hibernate}
end
def handle_call({:release_stream, stream}, {client_pid, _}, state) do
state = update_stream(state, client_pid, &MapSet.delete(&1, stream))
{:reply, :ok, remove_client(state, client_pid, true), :hibernate}
end
def handle_call({:cancel_stream, stream}, {client_pid, _}, %{protocol: :http2} = state) do
:ok = Gun.cancel(state.conn_pid, stream)
state = update_stream(state, client_pid, &MapSet.delete(&1, stream))
{:reply, :ok, remove_client(state, client_pid, true), :hibernate}
end
def handle_call({:cancel_stream, _stream}, _from, state) do
{:stop, :normal, :ok, state}
end
def handle_call(:reclaim_info, _from, %{clients: clients} = state)
when map_size(clients) == 0 do
{:reply, {:idle, state.crf, state.last_reference}, state, :hibernate}
end
def handle_call(:reclaim_info, _from, state) do
{:reply, :in_use, state, :hibernate}
end
def handle_call(:reclaim, _from, %{clients: clients} = state) when map_size(clients) == 0 do
{:stop, :normal, :reclaimed, state}
end
def handle_call(:reclaim, _from, state) do
{:reply, :in_use, state, :hibernate}
end
@impl true
def handle_info(:idle_close, state) do
# Gun monitors the owner process, and will close the connection automatically
# when it's terminated
def handle_info({:idle_close, token}, %{timer: {_timer_ref, token}, clients: clients} = state)
when map_size(clients) == 0 do
# Gun monitors the owner process, and will close the connection automatically.
{:stop, :normal, state}
end
def handle_info({:idle_close, _token}, state), do: {:noreply, state, :hibernate}
@impl true
def handle_info({:gun_up, _pid, _protocol}, state) do
{:noreply, state, :hibernate}
def handle_info({:gun_up, _pid, protocol}, state) do
{:noreply, %{state | protocol: protocol}, :hibernate}
end
# Gracefully shutdown if the connection got closed without any streams left
@impl true
def handle_info({:gun_error, conn_pid, stream, _reason}, %{conn_pid: conn_pid} = state) do
{:noreply, delete_stream(state, stream), :hibernate}
end
@impl true
def handle_info({:gun_down, _pid, _protocol, _reason, _streams}, %{proxied: true} = state) do
{:stop, :normal, state}
end
# Gracefully shutdown if the connection got closed without any streams left.
@impl true
def handle_info({:gun_down, _pid, _protocol, _reason, []}, state) do
{:stop, :normal, state}
end
# Otherwise, wait for retry
# Otherwise, wait for retry.
@impl true
def handle_info({:gun_down, _pid, _protocol, _reason, _killed_streams}, state) do
{:noreply, state, :hibernate}
end
@impl true
def handle_info({:DOWN, _ref, :process, pid, reason}, state) do
:telemetry.execute(
[:pleroma, :connection_pool, :client, :dead],
%{client_pid: pid, reason: reason},
%{key: state.key}
)
def handle_info({:DOWN, ref, :process, pid, reason}, state) do
case state.clients[pid] do
%{monitor: ^ref} ->
:telemetry.execute(
[:pleroma, :connection_pool, :client, :dead],
%{client_pid: pid, reason: reason},
%{key: state.key}
)
handle_cast({:remove_client, pid}, state)
handle_dead_client(state, pid)
_ ->
{:noreply, state, :hibernate}
end
end
defp add_client(state, client_pid) do
update_in(state.clients, fn clients ->
case clients[client_pid] do
nil ->
Map.put(clients, client_pid, %{
count: 1,
monitor: Process.monitor(client_pid),
streams: MapSet.new()
})
client ->
Map.put(clients, client_pid, %{client | count: client.count + 1})
end
end)
end
defp handle_dead_client(state, client_pid) do
streams = active_streams(state, client_pid)
cond do
streams == [] ->
{:noreply, remove_client(state, client_pid, false, true), :hibernate}
state.protocol == :http2 ->
Enum.each(streams, &(:ok = Gun.cancel(state.conn_pid, &1)))
{:noreply, remove_client(state, client_pid, false, true), :hibernate}
true ->
{:stop, :normal, state}
end
end
defp active_streams(state, client_pid) do
case state.clients[client_pid] do
%{streams: streams} -> MapSet.to_list(streams)
nil -> []
end
end
defp update_stream(state, client_pid, update) do
update_in(state.clients, fn clients ->
case clients[client_pid] do
nil -> clients
client -> Map.put(clients, client_pid, %{client | streams: update.(client.streams)})
end
end)
end
defp delete_stream(state, stream) do
update_in(state.clients, fn clients ->
Map.new(clients, fn {pid, client} ->
{pid, %{client | streams: MapSet.delete(client.streams, stream)}}
end)
end)
end
defp remove_client(state, client_pid, demonitor, all \\ false) do
case state.clients[client_pid] do
nil ->
state
%{count: count} = client when count > 1 and not all ->
put_in(state.clients[client_pid], %{client | count: count - 1})
client ->
if demonitor, do: Process.demonitor(client.monitor, [:flush])
state = update_in(state.clients, &Map.delete(&1, client_pid))
if map_size(state.clients) == 0, do: schedule_idle_close(state), else: state
end
end
defp cancel_idle_timer(%{timer: nil} = state), do: state
defp cancel_idle_timer(%{timer: {timer_ref, _token}} = state) do
Process.cancel_timer(timer_ref)
%{state | timer: nil}
end
defp schedule_idle_close(state) do
max_idle = Pleroma.Config.get([:connections_pool, :max_idle_time], 30_000)
token = make_ref()
timer_ref = Process.send_after(self(), {:idle_close, token}, max_idle)
%{state | timer: {timer_ref, token}}
end
# LRFU policy: https://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.55.1478

9
lib/pleroma/gun/connection_pool/worker_supervisor.ex
View file

@ -26,7 +26,14 @@ defmodule Pleroma.Gun.ConnectionPool.WorkerSupervisor do
def start_worker(opts, true) do
case DynamicSupervisor.start_child(__MODULE__, {Worker, opts}) do
{:error, :max_children} ->
:telemetry.execute([:pleroma, :connection_pool, :provision_failure], %{opts: opts})
[key | _] = opts
:telemetry.execute(
[:pleroma, :connection_pool, :provision_failure],
%{},
%{key: key}
)
{:error, :pool_full}
res ->

207
lib/pleroma/http/adapter/gun.ex Normal file
View file

@ -0,0 +1,207 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.HTTP.Adapter.Gun do
@behaviour Tesla.Adapter
alias Pleroma.Gun.ConnectionPool
alias Tesla.Multipart
@default_timeout 1_000
@impl Tesla.Adapter
def call(env, opts) do
adapter_opts =
Tesla.Adapter.opts(
[close_conn: true, body_as: :plain, send_body: :at_once, receive: true],
env,
opts
)
|> Map.new()
with {:ok, status, headers, body} <- request(env, adapter_opts) do
env = %{env | status: status, headers: format_headers(headers), body: body}
{:ok, put_stream_ref(env, body)}
end
end
defp put_stream_ref(env, %{stream: stream}) do
put_in(env.opts[:adapter][:stream], stream)
end
defp put_stream_ref(env, _body), do: env
defp request(env, %{conn: conn, tunnel: tunnel} = opts) do
uri = URI.parse(Tesla.build_url(env))
path = Tesla.Adapter.Shared.prepare_path(uri.path, uri.query)
method = Tesla.Adapter.Shared.format_method(env.method)
{headers, body, send_body} = prepare_body(env.headers, env.body, opts[:send_body])
request_opts = request_opts(opts[:reply_to], tunnel)
stream = open_stream(conn, method, path, headers, body, request_opts, send_body)
:ok = ConnectionPool.register_stream(conn, stream)
:ok = send_stream_body(conn, stream, body, send_body)
response = read_response(conn, stream, opts)
case response do
{:error, reason} ->
ConnectionPool.cancel_stream(conn, stream)
{:error, {:gun_stream_error, reason}}
response ->
unless opts[:body_as] in [:chunks, :stream] and stream_body?(response) do
:ok = ConnectionPool.finish_stream(conn, stream)
end
if opts[:close_conn] and opts[:body_as] not in [:stream, :chunks] do
:gun.close(conn)
end
response
end
end
defp stream_body?({:ok, _status, _headers, %{stream: _stream}}), do: true
defp stream_body?({:ok, _status, _headers, %Stream{}}), do: true
defp stream_body?(_response), do: false
defp request_opts(reply_to, nil), do: %{reply_to: reply_to || self()}
defp request_opts(reply_to, tunnel) do
%{reply_to: reply_to || self(), tunnel: tunnel}
end
defp prepare_body(headers, %Multipart{} = multipart, _send_body) do
{format_headers(headers ++ Multipart.headers(multipart)), Multipart.body(multipart), :stream}
end
defp prepare_body(headers, body, _send_body)
when is_struct(body, Stream) or is_function(body) do
{format_headers(headers), body, :stream}
end
defp prepare_body(headers, body, send_body) do
{format_headers(headers), body || "", send_body}
end
defp open_stream(conn, method, path, headers, _body, request_opts, :stream) do
:gun.headers(conn, method, path, headers, request_opts)
end
defp open_stream(conn, method, path, headers, body, request_opts, :at_once) do
:gun.request(conn, method, path, headers, body, request_opts)
end
defp send_stream_body(conn, stream, body, :stream) do
try do
for data <- body, do: :ok = :gun.data(conn, stream, :nofin, data)
:gun.data(conn, stream, :fin, "")
catch
kind, reason ->
ConnectionPool.cancel_stream(conn, stream)
:erlang.raise(kind, reason, __STACKTRACE__)
end
end
defp send_stream_body(_conn, _stream, _body, :at_once), do: :ok
defp read_response(conn, stream, opts) do
receive? = opts[:receive]
receive do
{:gun_response, ^conn, ^stream, :fin, status, headers} ->
{:ok, status, headers, ""}
{:gun_response, ^conn, ^stream, :nofin, status, headers} ->
format_response(conn, stream, opts, status, headers, opts[:body_as])
{:gun_up, ^conn, _protocol} when receive? ->
read_response(conn, stream, opts)
{:gun_error, ^conn, reason} ->
{:error, reason}
{:gun_error, ^conn, ^stream, reason} ->
{:error, reason}
{:gun_down, ^conn, _protocol, _reason, _killed_streams} when receive? ->
read_response(conn, stream, opts)
{:DOWN, _ref, :process, ^conn, reason} ->
{:error, reason}
after
opts[:timeout] || @default_timeout -> {:error, :recv_response_timeout}
end
end
defp format_response(conn, stream, opts, status, headers, :plain) do
case read_body(conn, stream, opts) do
{:ok, body} ->
{:ok, status, headers, body}
{:error, error} ->
:ok = :gun.flush(stream)
{:error, error}
end
end
defp format_response(conn, stream, opts, status, headers, :stream) do
body =
Stream.resource(
fn -> :reading end,
fn
:reading ->
case Tesla.Adapter.Gun.read_chunk(conn, stream, opts) do
{:nofin, part} -> {[part], :reading}
{:fin, part} -> {[part], :done}
{:error, reason} -> raise "Gun stream failed: #{inspect(reason)}"
end
:done ->
{:halt, :done}
end,
fn _state ->
if opts[:close_conn], do: :gun.close(conn)
end
)
{:ok, status, headers, body}
end
defp format_response(conn, stream, opts, status, headers, :chunks) do
{:ok, status, headers, %{pid: conn, stream: stream, opts: Map.to_list(opts)}}
end
defp read_body(conn, stream, opts, acc \\ "") do
receive do
{:gun_data, ^conn, ^stream, :fin, body} ->
append_body(acc, body, opts[:max_body])
{:gun_data, ^conn, ^stream, :nofin, body} ->
with {:ok, acc} <- append_body(acc, body, opts[:max_body]) do
read_body(conn, stream, opts, acc)
end
{:gun_error, ^conn, ^stream, reason} ->
{:error, reason}
{:DOWN, _ref, :process, ^conn, reason} ->
{:error, reason}
after
opts[:timeout] || @default_timeout -> {:error, :recv_body_timeout}
end
end
defp append_body(acc, part, nil), do: {:ok, acc <> part}
defp append_body(acc, part, limit) do
body = acc <> part
if byte_size(body) <= limit, do: {:ok, body}, else: {:error, :body_too_large}
end
defp format_headers(headers) do
Enum.map(headers, fn {key, value} ->
{String.downcase(to_string(key)), to_string(value)}
end)
end
end

4
lib/pleroma/http/adapter_helper/gun.ex
View file

@ -48,10 +48,10 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do
Keyword.put(opts, :timeout, recv_timeout)
end
# Gun uses [body_as: :stream]
# Keep the pool lease until the caller consumes or cancels the body.
defp maybe_stream(opts) do
case Keyword.pop(opts, :stream, nil) do
{true, opts} -> Keyword.put(opts, :body_as, :stream)
{true, opts} -> Keyword.put(opts, :body_as, :chunks)
{_, opts} -> opts
end
end

33
lib/pleroma/reverse_proxy.ex
View file

@ -110,12 +110,8 @@ defmodule Pleroma.ReverseProxy do
end
with {:ok, nil} <- @cachex.get(:failed_proxy_url_cache, url),
{:ok, code, headers, client} <- request(method, url, req_headers, client_opts),
:ok <-
header_length_constraint(
headers,
Keyword.get(opts, :max_body_length, @max_body_length)
) do
{:ok, code, headers, client} <-
request_with_constraints(method, url, req_headers, client_opts, opts) do
response(conn, client, url, code, headers, opts)
else
{:ok, true} ->
@ -170,7 +166,8 @@ defmodule Pleroma.ReverseProxy do
{:ok, code, headers} when code in @valid_resp_codes ->
{:ok, code, downcase_headers(headers)}
{:ok, code, _, _} ->
{:ok, code, _, client} ->
client().close(client)
{:error, {:invalid_http_response, code}}
{:ok, code, _} ->
@ -181,6 +178,26 @@ defmodule Pleroma.ReverseProxy do
end
end
defp request_with_constraints(method, url, headers, client_opts, opts) do
case request(method, url, headers, client_opts) do
{:ok, _code, headers, client} = response ->
case header_length_constraint(
headers,
Keyword.get(opts, :max_body_length, @max_body_length)
) do
:ok ->
response
error ->
client().close(client)
error
end
response ->
response
end
end
defp response(conn, client, url, status, headers, opts) do
Logger.debug("#{__MODULE__} #{status} #{url} #{inspect(headers)}")
@ -406,7 +423,7 @@ defmodule Pleroma.ReverseProxy do
defp header_length_constraint(_, _), do: :ok
defp body_size_constraint(size, limit) when is_integer(limit) and limit > 0 and size >= limit do
defp body_size_constraint(size, limit) when is_integer(limit) and limit > 0 and size > limit do
{:error, :body_too_large}
end

8
lib/pleroma/reverse_proxy/client/tesla.ex
View file

@ -45,8 +45,8 @@ defmodule Pleroma.ReverseProxy.Client.Tesla do
@impl true
@spec stream_body(map()) ::
{:ok, binary(), map()} | {:error, atom() | String.t()} | :done | no_return()
def stream_body(%{pid: pid, fin: true}) do
ConnectionPool.release_conn(pid)
def stream_body(%{pid: pid, stream: stream, fin: true}) do
ConnectionPool.release_stream(pid, stream)
:done
end
@ -70,6 +70,10 @@ defmodule Pleroma.ReverseProxy.Client.Tesla do
@impl true
@spec close(map) :: :ok | no_return()
def close(%{pid: pid, stream: stream}) do
ConnectionPool.cancel_stream(pid, stream)
end
def close(%{pid: pid}) do
ConnectionPool.release_conn(pid)
end

2
lib/pleroma/telemetry/logger.ex
View file

@ -55,8 +55,8 @@ defmodule Pleroma.Telemetry.Logger do
def handle_event(
[:pleroma, :connection_pool, :provision_failure],
%{opts: [key | _]},
_,
%{key: key},
_
) do
Logger.debug(fn ->

60
lib/pleroma/tesla/middleware/connection_pool.ex
View file

@ -12,25 +12,38 @@ defmodule Pleroma.Tesla.Middleware.ConnectionPool do
alias Pleroma.Gun.ConnectionPool
@impl Tesla.Middleware
def call(%Tesla.Env{url: url, opts: opts} = env, next, _) do
def call(%Tesla.Env{opts: opts} = env, next, middleware_opts) do
if opts[:adapter][:body_as] == :stream do
{:error, :pooled_streaming_requires_chunks}
else
do_call(env, next, middleware_opts)
end
end
defp do_call(%Tesla.Env{url: url, opts: opts} = env, next, _) do
uri = URI.parse(url)
# Avoid leaking connections when the middleware is called twice
# with body_as: :chunks. We assume only the middleware can set
# opts[:adapter][:conn]
if opts[:adapter][:conn] do
ConnectionPool.release_conn(opts[:adapter][:conn])
end
cleanup_previous_request(env)
case ConnectionPool.get_conn(uri, opts[:adapter]) do
{:ok, conn_pid} ->
adapter_opts = Keyword.merge(opts[:adapter], conn: conn_pid, close_conn: false)
tunnel = ConnectionPool.tunnel_ref(conn_pid)
adapter_opts =
opts[:adapter]
|> Keyword.drop([:conn, :stream, :tunnel])
|> Keyword.merge(conn: conn_pid, close_conn: false, tunnel: tunnel)
opts = Keyword.put(opts, :adapter, adapter_opts)
env = %{env | opts: opts}
next = use_pool_adapter(next)
case Tesla.run(env, next) do
{:ok, env} ->
unless opts[:adapter][:body_as] == :chunks do
unless opts[:adapter][:body_as] == :chunks and chunk_client?(env.body) do
ConnectionPool.release_conn(conn_pid)
{_, res} = pop_in(env.opts[:adapter][:conn])
{:ok, res}
@ -38,8 +51,11 @@ defmodule Pleroma.Tesla.Middleware.ConnectionPool do
{:ok, env}
end
{:error, {:gun_stream_error, reason}} ->
{:error, reason}
err ->
ConnectionPool.release_conn(conn_pid)
ConnectionPool.discard_conn(conn_pid)
err
end
@ -47,4 +63,34 @@ defmodule Pleroma.Tesla.Middleware.ConnectionPool do
err
end
end
defp cleanup_previous_request(%Tesla.Env{
body: %{pid: pid, stream: stream},
opts: opts
}) do
if opts[:adapter][:conn], do: ConnectionPool.cancel_stream(pid, stream)
end
defp cleanup_previous_request(%Tesla.Env{opts: opts}) do
conn = opts[:adapter][:conn]
stream = opts[:adapter][:stream]
cond do
conn && stream -> ConnectionPool.cancel_stream(conn, stream)
conn -> ConnectionPool.release_conn(conn)
true -> :ok
end
end
defp chunk_client?(%{pid: pid, stream: stream}) when is_pid(pid) and not is_nil(stream),
do: true
defp chunk_client?(_body), do: false
defp use_pool_adapter(next) do
List.update_at(next, -1, fn
{Tesla.Adapter.Gun, function, args} -> {Pleroma.HTTP.Adapter.Gun, function, args}
adapter -> adapter
end)
end
end

63
lib/pleroma/web/rich_media/helpers.ex
View file

@ -4,6 +4,8 @@
defmodule Pleroma.Web.RichMedia.Helpers do
alias Pleroma.Config
alias Pleroma.Gun.ConnectionPool
alias Pleroma.ReverseProxy.Client.Tesla, as: TeslaClient
require Logger
@ -19,12 +21,30 @@ defmodule Pleroma.Web.RichMedia.Helpers do
end
defp stream(url) do
with {_, {:ok, %Tesla.Env{status: 200, body: stream_body, headers: headers}}} <-
{:get, Pleroma.HTTP.get(url, req_headers(), http_options())},
{_, :ok} <- {:content_type, check_content_type(headers)},
{_, :ok} <- {:content_length, check_content_length(headers)},
{:read_stream, {:ok, body}} <- {:read_stream, read_stream(stream_body)} do
{:ok, body}
case Pleroma.HTTP.get(url, req_headers(), http_options()) do
{:ok, %Tesla.Env{status: 200, body: stream_body, headers: headers}} ->
result =
try do
with {_, :ok} <- {:content_type, check_content_type(headers)},
{_, :ok} <- {:content_length, check_content_length(headers)},
{:read_stream, {:ok, body}} <- {:read_stream, read_stream(stream_body)} do
{:ok, body}
end
rescue
exception ->
cleanup_stream(stream_body, :error)
reraise exception, __STACKTRACE__
end
cleanup_stream(stream_body, result)
result
{:ok, %Tesla.Env{body: stream_body}} = response ->
cleanup_stream(stream_body, :rejected)
{:get, response}
response ->
{:get, response}
end
end
@ -95,6 +115,10 @@ defmodule Pleroma.Web.RichMedia.Helpers do
end
end
defp read_stream(%{pid: pid, stream: stream, opts: opts}) do
read_chunks(pid, stream, opts, "", 0, Keyword.get(http_options(), :max_body))
end
defp read_stream(stream) do
max_body = Keyword.get(http_options(), :max_body)
@ -117,6 +141,33 @@ defmodule Pleroma.Web.RichMedia.Helpers do
end
end
defp read_chunks(pid, stream, opts, acc, total_bytes, max_body) do
case Tesla.Adapter.Gun.read_chunk(pid, stream, opts) do
{fin, chunk} when fin in [:fin, :nofin] ->
total_bytes = total_bytes + byte_size(chunk)
cond do
total_bytes > max_body ->
:error
fin == :fin ->
{:ok, acc <> chunk}
true ->
read_chunks(pid, stream, opts, acc <> chunk, total_bytes, max_body)
end
{:error, _reason} ->
:error
end
end
defp cleanup_stream(%{pid: pid, stream: stream}, {:ok, _body}),
do: ConnectionPool.release_stream(pid, stream)
defp cleanup_stream(%{pid: _pid} = client, _result), do: TeslaClient.close(client)
defp cleanup_stream(_stream, _result), do: :ok
defp http_options do
[
pool: :rich_media,