Refactor gun pooling and simplify adapter option insertion
This patch refactors gun pooling to use Elixir process registry and simplifies adapter option insertion. Having the pool use process registry instead of a GenServer has a number of advantages: - Simpler code: the initial implementation adds about half the lines of code it deletes - Concurrency: unlike a GenServer, ETS-based registry can handle multiple checkout/checkin requests at the same time - Precise and easy idle connection clousure: current proposal for closing idle connections in the GenServer-based pool needs to filter through all connections once a minute and compare their last active time with closing time. With Elixir process registry this can be done by just using `Process.send_after`/`Process.cancel_timer` in the worker process. - Lower memory footprint: In my tests `gun-memory-leak` branch uses about 290mb on peak load (250 connections) and 235mb on idle (5-10 connections). Registry-based pool uses 210mb on idle and 240mb on peak load
This commit is contained in:
rinpatch
parent
040524c09f
commit
58a4f350a8
16 changed files with 402 additions and 686 deletions
|
|
@ -3,40 +3,11 @@
|
|||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.Gun.Conn do
|
||||
@moduledoc """
|
||||
Struct for gun connection data
|
||||
"""
|
||||
alias Pleroma.Gun
|
||||
alias Pleroma.Pool.Connections
|
||||
|
||||
require Logger
|
||||
|
||||
@type gun_state :: :up | :down
|
||||
@type conn_state :: :active | :idle
|
||||
|
||||
@type t :: %__MODULE__{
|
||||
conn: pid(),
|
||||
gun_state: gun_state(),
|
||||
conn_state: conn_state(),
|
||||
used_by: [pid()],
|
||||
last_reference: pos_integer(),
|
||||
crf: float(),
|
||||
retries: pos_integer()
|
||||
}
|
||||
|
||||
defstruct conn: nil,
|
||||
gun_state: :open,
|
||||
conn_state: :init,
|
||||
used_by: [],
|
||||
last_reference: 0,
|
||||
crf: 1,
|
||||
retries: 0
|
||||
|
||||
@spec open(String.t() | URI.t(), atom(), keyword()) :: :ok | nil
|
||||
def open(url, name, opts \\ [])
|
||||
def open(url, name, opts) when is_binary(url), do: open(URI.parse(url), name, opts)
|
||||
|
||||
def open(%URI{} = uri, name, opts) do
|
||||
def open(%URI{} = uri, opts) do
|
||||
pool_opts = Pleroma.Config.get([:connections_pool], [])
|
||||
|
||||
opts =
|
||||
|
|
@ -45,30 +16,10 @@ defmodule Pleroma.Gun.Conn do
|
|||
|> Map.put_new(:retry, pool_opts[:retry] || 1)
|
||||
|> Map.put_new(:retry_timeout, pool_opts[:retry_timeout] || 1000)
|
||||
|> Map.put_new(:await_up_timeout, pool_opts[:await_up_timeout] || 5_000)
|
||||
|> Map.put_new(:supervise, false)
|
||||
|> maybe_add_tls_opts(uri)
|
||||
|
||||
key = "#{uri.scheme}:#{uri.host}:#{uri.port}"
|
||||
|
||||
max_connections = pool_opts[:max_connections] || 250
|
||||
|
||||
conn_pid =
|
||||
if Connections.count(name) < max_connections do
|
||||
do_open(uri, opts)
|
||||
else
|
||||
close_least_used_and_do_open(name, uri, opts)
|
||||
end
|
||||
|
||||
if is_pid(conn_pid) do
|
||||
conn = %Pleroma.Gun.Conn{
|
||||
conn: conn_pid,
|
||||
gun_state: :up,
|
||||
conn_state: :active,
|
||||
last_reference: :os.system_time(:second)
|
||||
}
|
||||
|
||||
:ok = Gun.set_owner(conn_pid, Process.whereis(name))
|
||||
Connections.add_conn(name, key, conn)
|
||||
end
|
||||
do_open(uri, opts)
|
||||
end
|
||||
|
||||
defp maybe_add_tls_opts(opts, %URI{scheme: "http"}), do: opts
|
||||
|
|
@ -81,7 +32,7 @@ defmodule Pleroma.Gun.Conn do
|
|||
reuse_sessions: false,
|
||||
verify_fun:
|
||||
{&:ssl_verify_hostname.verify_fun/3,
|
||||
[check_hostname: Pleroma.HTTP.Connection.format_host(host)]}
|
||||
[check_hostname: Pleroma.HTTP.AdapterHelper.format_host(host)]}
|
||||
]
|
||||
|
||||
tls_opts =
|
||||
|
|
@ -105,7 +56,7 @@ defmodule Pleroma.Gun.Conn do
|
|||
{:ok, _} <- Gun.await_up(conn, opts[:await_up_timeout]),
|
||||
stream <- Gun.connect(conn, connect_opts),
|
||||
{:response, :fin, 200, _} <- Gun.await(conn, stream) do
|
||||
conn
|
||||
{:ok, conn}
|
||||
else
|
||||
error ->
|
||||
Logger.warn(
|
||||
|
|
@ -141,7 +92,7 @@ defmodule Pleroma.Gun.Conn do
|
|||
|
||||
with {:ok, conn} <- Gun.open(proxy_host, proxy_port, opts),
|
||||
{:ok, _} <- Gun.await_up(conn, opts[:await_up_timeout]) do
|
||||
conn
|
||||
{:ok, conn}
|
||||
else
|
||||
error ->
|
||||
Logger.warn(
|
||||
|
|
@ -155,11 +106,11 @@ defmodule Pleroma.Gun.Conn do
|
|||
end
|
||||
|
||||
defp do_open(%URI{host: host, port: port} = uri, opts) do
|
||||
host = Pleroma.HTTP.Connection.parse_host(host)
|
||||
host = Pleroma.HTTP.AdapterHelper.parse_host(host)
|
||||
|
||||
with {:ok, conn} <- Gun.open(host, port, opts),
|
||||
{:ok, _} <- Gun.await_up(conn, opts[:await_up_timeout]) do
|
||||
conn
|
||||
{:ok, conn}
|
||||
else
|
||||
error ->
|
||||
Logger.warn(
|
||||
|
|
@ -171,7 +122,7 @@ defmodule Pleroma.Gun.Conn do
|
|||
end
|
||||
|
||||
defp destination_opts(%URI{host: host, port: port}) do
|
||||
host = Pleroma.HTTP.Connection.parse_host(host)
|
||||
host = Pleroma.HTTP.AdapterHelper.parse_host(host)
|
||||
%{host: host, port: port}
|
||||
end
|
||||
|
||||
|
|
@ -181,17 +132,6 @@ defmodule Pleroma.Gun.Conn do
|
|||
|
||||
defp add_http2_opts(opts, _, _), do: opts
|
||||
|
||||
defp close_least_used_and_do_open(name, uri, opts) do
|
||||
with [{key, conn} | _conns] <- Connections.get_unused_conns(name),
|
||||
:ok <- Gun.close(conn.conn) do
|
||||
Connections.remove_conn(name, key)
|
||||
|
||||
do_open(uri, opts)
|
||||
else
|
||||
[] -> {:error, :pool_overflowed}
|
||||
end
|
||||
end
|
||||
|
||||
def compose_uri_log(%URI{scheme: scheme, host: host, path: path}) do
|
||||
"#{scheme}://#{host}#{path}"
|
||||
end
|
||||
|
|
|
|||
129
lib/pleroma/gun/connection_pool.ex
Normal file
129
lib/pleroma/gun/connection_pool.ex
Normal file
|
|
@ -0,0 +1,129 @@
|
|||
defmodule Pleroma.Gun.ConnectionPool do
|
||||
@registry __MODULE__
|
||||
|
||||
def get_conn(uri, opts) do
|
||||
case enforce_pool_limits() do
|
||||
:ok ->
|
||||
key = "#{uri.scheme}:#{uri.host}:#{uri.port}"
|
||||
|
||||
case Registry.lookup(@registry, key) do
|
||||
# The key has already been registered, but connection is not up yet
|
||||
[{worker_pid, {nil, _used_by, _crf, _last_reference}}] ->
|
||||
get_gun_pid_from_worker(worker_pid)
|
||||
|
||||
[{worker_pid, {gun_pid, _used_by, _crf, _last_reference}}] ->
|
||||
GenServer.cast(worker_pid, {:add_client, self(), false})
|
||||
{:ok, gun_pid}
|
||||
|
||||
[] ->
|
||||
# :gun.set_owner fails in :connected state for whatevever reason,
|
||||
# so we open the connection in the process directly and send it's pid back
|
||||
# We trust gun to handle timeouts by itself
|
||||
case GenServer.start(Pleroma.Gun.ConnectionPool.Worker, [uri, key, opts, self()],
|
||||
timeout: :infinity
|
||||
) do
|
||||
{:ok, _worker_pid} ->
|
||||
receive do
|
||||
{:conn_pid, pid} -> {:ok, pid}
|
||||
end
|
||||
|
||||
{:error, {:error, {:already_registered, worker_pid}}} ->
|
||||
get_gun_pid_from_worker(worker_pid)
|
||||
|
||||
err ->
|
||||
err
|
||||
end
|
||||
end
|
||||
|
||||
:error ->
|
||||
{:error, :pool_full}
|
||||
end
|
||||
end
|
||||
|
||||
@enforcer_key "enforcer"
|
||||
defp enforce_pool_limits() do
|
||||
max_connections = Pleroma.Config.get([:connections_pool, :max_connections])
|
||||
|
||||
if Registry.count(@registry) >= max_connections do
|
||||
case Registry.lookup(@registry, @enforcer_key) do
|
||||
[] ->
|
||||
pid =
|
||||
spawn(fn ->
|
||||
{:ok, _pid} = Registry.register(@registry, @enforcer_key, nil)
|
||||
|
||||
reclaim_max =
|
||||
[:connections_pool, :reclaim_multiplier]
|
||||
|> Pleroma.Config.get()
|
||||
|> Kernel.*(max_connections)
|
||||
|> round
|
||||
|> max(1)
|
||||
|
||||
unused_conns =
|
||||
Registry.select(
|
||||
@registry,
|
||||
[
|
||||
{{:_, :"$1", {:_, :"$2", :"$3", :"$4"}}, [{:==, :"$2", []}],
|
||||
[{{:"$1", :"$3", :"$4"}}]}
|
||||
]
|
||||
)
|
||||
|
||||
case unused_conns do
|
||||
[] ->
|
||||
exit(:pool_full)
|
||||
|
||||
unused_conns ->
|
||||
unused_conns
|
||||
|> Enum.sort(fn {_pid1, crf1, last_reference1},
|
||||
{_pid2, crf2, last_reference2} ->
|
||||
crf1 <= crf2 and last_reference1 <= last_reference2
|
||||
end)
|
||||
|> Enum.take(reclaim_max)
|
||||
|> Enum.each(fn {pid, _, _} -> GenServer.call(pid, :idle_close) end)
|
||||
end
|
||||
end)
|
||||
|
||||
wait_for_enforcer_finish(pid)
|
||||
|
||||
[{pid, _}] ->
|
||||
wait_for_enforcer_finish(pid)
|
||||
end
|
||||
else
|
||||
:ok
|
||||
end
|
||||
end
|
||||
|
||||
defp wait_for_enforcer_finish(pid) do
|
||||
ref = Process.monitor(pid)
|
||||
|
||||
receive do
|
||||
{:DOWN, ^ref, :process, ^pid, :pool_full} ->
|
||||
:error
|
||||
|
||||
{:DOWN, ^ref, :process, ^pid, :normal} ->
|
||||
:ok
|
||||
end
|
||||
end
|
||||
|
||||
defp get_gun_pid_from_worker(worker_pid) do
|
||||
# GenServer.call will block the process for timeout length if
|
||||
# the server crashes on startup (which will happen if gun fails to connect)
|
||||
# so instead we use cast + monitor
|
||||
|
||||
ref = Process.monitor(worker_pid)
|
||||
GenServer.cast(worker_pid, {:add_client, self(), true})
|
||||
|
||||
receive do
|
||||
{:conn_pid, pid} -> {:ok, pid}
|
||||
{:DOWN, ^ref, :process, ^worker_pid, reason} -> reason
|
||||
end
|
||||
end
|
||||
|
||||
def release_conn(conn_pid) do
|
||||
[worker_pid] =
|
||||
Registry.select(@registry, [
|
||||
{{:_, :"$1", {:"$2", :_, :_, :_}}, [{:==, :"$2", conn_pid}], [:"$1"]}
|
||||
])
|
||||
|
||||
GenServer.cast(worker_pid, {:remove_client, self()})
|
||||
end
|
||||
end
|
||||
95
lib/pleroma/gun/connection_pool/worker.ex
Normal file
95
lib/pleroma/gun/connection_pool/worker.ex
Normal file
|
|
@ -0,0 +1,95 @@
|
|||
defmodule Pleroma.Gun.ConnectionPool.Worker do
|
||||
alias Pleroma.Gun
|
||||
use GenServer
|
||||
|
||||
@registry Pleroma.Gun.ConnectionPool
|
||||
|
||||
@impl true
|
||||
def init([uri, key, opts, client_pid]) do
|
||||
time = :os.system_time(:second)
|
||||
# Register before opening connection to prevent race conditions
|
||||
with {:ok, _owner} <- Registry.register(@registry, key, {nil, [client_pid], 1, time}),
|
||||
{:ok, conn_pid} <- Gun.Conn.open(uri, opts),
|
||||
Process.link(conn_pid) do
|
||||
{_, _} =
|
||||
Registry.update_value(@registry, key, fn {_, used_by, crf, last_reference} ->
|
||||
{conn_pid, used_by, crf, last_reference}
|
||||
end)
|
||||
|
||||
send(client_pid, {:conn_pid, conn_pid})
|
||||
{:ok, %{key: key, timer: nil}, :hibernate}
|
||||
else
|
||||
err -> {:stop, err}
|
||||
end
|
||||
end
|
||||
|
||||
@impl true
|
||||
def handle_cast({:add_client, client_pid, send_pid_back}, %{key: key} = state) do
|
||||
time = :os.system_time(:second)
|
||||
|
||||
{{conn_pid, _, _, _}, _} =
|
||||
Registry.update_value(@registry, key, fn {conn_pid, used_by, crf, last_reference} ->
|
||||
{conn_pid, [client_pid | used_by], crf(time - last_reference, crf), time}
|
||||
end)
|
||||
|
||||
if send_pid_back, do: send(client_pid, {:conn_pid, conn_pid})
|
||||
|
||||
state =
|
||||
if state.timer != nil do
|
||||
Process.cancel_timer(state[:timer])
|
||||
%{state | timer: nil}
|
||||
else
|
||||
state
|
||||
end
|
||||
|
||||
{:noreply, state, :hibernate}
|
||||
end
|
||||
|
||||
@impl true
|
||||
def handle_cast({:remove_client, client_pid}, %{key: key} = state) do
|
||||
{{_conn_pid, used_by, _crf, _last_reference}, _} =
|
||||
Registry.update_value(@registry, key, fn {conn_pid, used_by, crf, last_reference} ->
|
||||
{conn_pid, List.delete(used_by, client_pid), crf, last_reference}
|
||||
end)
|
||||
|
||||
timer =
|
||||
if used_by == [] do
|
||||
max_idle = Pleroma.Config.get([:connections_pool, :max_idle_time], 30_000)
|
||||
Process.send_after(self(), :idle_close, max_idle)
|
||||
else
|
||||
nil
|
||||
end
|
||||
|
||||
{:noreply, %{state | timer: timer}, :hibernate}
|
||||
end
|
||||
|
||||
@impl true
|
||||
def handle_info(:idle_close, state) do
|
||||
# Gun monitors the owner process, and will close the connection automatically
|
||||
# when it's terminated
|
||||
{:stop, :normal, state}
|
||||
end
|
||||
|
||||
# Gracefully shutdown if the connection got closed without any streams left
|
||||
@impl true
|
||||
def handle_info({:gun_down, _pid, _protocol, _reason, []}, state) do
|
||||
{:stop, :normal, state}
|
||||
end
|
||||
|
||||
# Otherwise, shutdown with an error
|
||||
@impl true
|
||||
def handle_info({:gun_down, _pid, _protocol, _reason, _killed_streams} = down_message, state) do
|
||||
{:stop, {:error, down_message}, state}
|
||||
end
|
||||
|
||||
@impl true
|
||||
def handle_call(:idle_close, _, %{key: key} = state) do
|
||||
Registry.unregister(@registry, key)
|
||||
{:stop, :normal, state}
|
||||
end
|
||||
|
||||
# LRFU policy: https://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.55.1478
|
||||
defp crf(time_delta, prev_crf) do
|
||||
1 + :math.pow(0.5, time_delta / 100) * prev_crf
|
||||
end
|
||||
end
|
||||
Loading…
Add table
Add a link
Reference in a new issue