Merge remote-tracking branch 'remotes/origin/develop' into output-of-relationships-in-statuses
# Conflicts: # CHANGELOG.md
This commit is contained in:
Ivan Tashkinov
commit
98b96e45ab
77 changed files with 3343 additions and 511 deletions
|
|
@ -75,4 +75,43 @@ defmodule Mix.Tasks.Pleroma.Benchmark do
|
|||
inputs: inputs
|
||||
)
|
||||
end
|
||||
|
||||
def run(["adapters"]) do
|
||||
start_pleroma()
|
||||
|
||||
:ok =
|
||||
Pleroma.Gun.Conn.open(
|
||||
"https://httpbin.org/stream-bytes/1500",
|
||||
:gun_connections
|
||||
)
|
||||
|
||||
Process.sleep(1_500)
|
||||
|
||||
Benchee.run(
|
||||
%{
|
||||
"Without conn and without pool" => fn ->
|
||||
{:ok, %Tesla.Env{}} =
|
||||
Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [],
|
||||
adapter: [pool: :no_pool, receive_conn: false]
|
||||
)
|
||||
end,
|
||||
"Without conn and with pool" => fn ->
|
||||
{:ok, %Tesla.Env{}} =
|
||||
Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [],
|
||||
adapter: [receive_conn: false]
|
||||
)
|
||||
end,
|
||||
"With reused conn and without pool" => fn ->
|
||||
{:ok, %Tesla.Env{}} =
|
||||
Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [],
|
||||
adapter: [pool: :no_pool]
|
||||
)
|
||||
end,
|
||||
"With reused conn and with pool" => fn ->
|
||||
{:ok, %Tesla.Env{}} = Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500")
|
||||
end
|
||||
},
|
||||
parallel: 10
|
||||
)
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -4,13 +4,13 @@
|
|||
|
||||
defmodule Mix.Tasks.Pleroma.Emoji do
|
||||
use Mix.Task
|
||||
import Mix.Pleroma
|
||||
|
||||
@shortdoc "Manages emoji packs"
|
||||
@moduledoc File.read!("docs/administration/CLI_tasks/emoji.md")
|
||||
|
||||
def run(["ls-packs" | args]) do
|
||||
Mix.Pleroma.start_pleroma()
|
||||
Application.ensure_all_started(:hackney)
|
||||
start_pleroma()
|
||||
|
||||
{options, [], []} = parse_global_opts(args)
|
||||
|
||||
|
|
@ -36,8 +36,7 @@ defmodule Mix.Tasks.Pleroma.Emoji do
|
|||
end
|
||||
|
||||
def run(["get-packs" | args]) do
|
||||
Mix.Pleroma.start_pleroma()
|
||||
Application.ensure_all_started(:hackney)
|
||||
start_pleroma()
|
||||
|
||||
{options, pack_names, []} = parse_global_opts(args)
|
||||
|
||||
|
|
@ -135,7 +134,7 @@ defmodule Mix.Tasks.Pleroma.Emoji do
|
|||
end
|
||||
|
||||
def run(["gen-pack", src]) do
|
||||
Application.ensure_all_started(:hackney)
|
||||
start_pleroma()
|
||||
|
||||
proposed_name = Path.basename(src) |> Path.rootname()
|
||||
name = String.trim(IO.gets("Pack name [#{proposed_name}]: "))
|
||||
|
|
|
|||
|
|
@ -3,8 +3,12 @@
|
|||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.Application do
|
||||
import Cachex.Spec
|
||||
use Application
|
||||
|
||||
import Cachex.Spec
|
||||
|
||||
alias Pleroma.Config
|
||||
|
||||
require Logger
|
||||
|
||||
@name Mix.Project.config()[:name]
|
||||
|
|
@ -18,9 +22,9 @@ defmodule Pleroma.Application do
|
|||
def repository, do: @repository
|
||||
|
||||
def user_agent do
|
||||
case Pleroma.Config.get([:http, :user_agent], :default) do
|
||||
case Config.get([:http, :user_agent], :default) do
|
||||
:default ->
|
||||
info = "#{Pleroma.Web.base_url()} <#{Pleroma.Config.get([:instance, :email], "")}>"
|
||||
info = "#{Pleroma.Web.base_url()} <#{Config.get([:instance, :email], "")}>"
|
||||
named_version() <> "; " <> info
|
||||
|
||||
custom ->
|
||||
|
|
@ -33,27 +37,51 @@ defmodule Pleroma.Application do
|
|||
def start(_type, _args) do
|
||||
Pleroma.Config.Holder.save_default()
|
||||
Pleroma.HTML.compile_scrubbers()
|
||||
Pleroma.Config.DeprecationWarnings.warn()
|
||||
Config.DeprecationWarnings.warn()
|
||||
Pleroma.Plugs.HTTPSecurityPlug.warn_if_disabled()
|
||||
Pleroma.Repo.check_migrations_applied!()
|
||||
setup_instrumenters()
|
||||
load_custom_modules()
|
||||
|
||||
adapter = Application.get_env(:tesla, :adapter)
|
||||
|
||||
if adapter == Tesla.Adapter.Gun do
|
||||
if version = Pleroma.OTPVersion.version() do
|
||||
[major, minor] =
|
||||
version
|
||||
|> String.split(".")
|
||||
|> Enum.map(&String.to_integer/1)
|
||||
|> Enum.take(2)
|
||||
|
||||
if (major == 22 and minor < 2) or major < 22 do
|
||||
raise "
|
||||
!!!OTP VERSION WARNING!!!
|
||||
You are using gun adapter with OTP version #{version}, which doesn't support correct handling of unordered certificates chains.
|
||||
"
|
||||
end
|
||||
else
|
||||
raise "
|
||||
!!!OTP VERSION WARNING!!!
|
||||
To support correct handling of unordered certificates chains - OTP version must be > 22.2.
|
||||
"
|
||||
end
|
||||
end
|
||||
|
||||
# Define workers and child supervisors to be supervised
|
||||
children =
|
||||
[
|
||||
Pleroma.Repo,
|
||||
Pleroma.Config.TransferTask,
|
||||
Config.TransferTask,
|
||||
Pleroma.Emoji,
|
||||
Pleroma.Captcha,
|
||||
Pleroma.Plugs.RateLimiter.Supervisor
|
||||
] ++
|
||||
cachex_children() ++
|
||||
hackney_pool_children() ++
|
||||
http_children(adapter, @env) ++
|
||||
[
|
||||
Pleroma.Stats,
|
||||
Pleroma.JobQueueMonitor,
|
||||
{Oban, Pleroma.Config.get(Oban)}
|
||||
{Oban, Config.get(Oban)}
|
||||
] ++
|
||||
task_children(@env) ++
|
||||
streamer_child(@env) ++
|
||||
|
|
@ -70,7 +98,7 @@ defmodule Pleroma.Application do
|
|||
end
|
||||
|
||||
def load_custom_modules do
|
||||
dir = Pleroma.Config.get([:modules, :runtime_dir])
|
||||
dir = Config.get([:modules, :runtime_dir])
|
||||
|
||||
if dir && File.exists?(dir) do
|
||||
dir
|
||||
|
|
@ -111,20 +139,6 @@ defmodule Pleroma.Application do
|
|||
Pleroma.Web.Endpoint.Instrumenter.setup()
|
||||
end
|
||||
|
||||
def enabled_hackney_pools do
|
||||
[:media] ++
|
||||
if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do
|
||||
[:federation]
|
||||
else
|
||||
[]
|
||||
end ++
|
||||
if Pleroma.Config.get([Pleroma.Upload, :proxy_remote]) do
|
||||
[:upload]
|
||||
else
|
||||
[]
|
||||
end
|
||||
end
|
||||
|
||||
defp cachex_children do
|
||||
[
|
||||
build_cachex("used_captcha", ttl_interval: seconds_valid_interval()),
|
||||
|
|
@ -146,7 +160,7 @@ defmodule Pleroma.Application do
|
|||
do: expiration(default: :timer.seconds(6 * 60 * 60), interval: :timer.seconds(60))
|
||||
|
||||
defp seconds_valid_interval,
|
||||
do: :timer.seconds(Pleroma.Config.get!([Pleroma.Captcha, :seconds_valid]))
|
||||
do: :timer.seconds(Config.get!([Pleroma.Captcha, :seconds_valid]))
|
||||
|
||||
defp build_cachex(type, opts),
|
||||
do: %{
|
||||
|
|
@ -155,7 +169,7 @@ defmodule Pleroma.Application do
|
|||
type: :worker
|
||||
}
|
||||
|
||||
defp chat_enabled?, do: Pleroma.Config.get([:chat, :enabled])
|
||||
defp chat_enabled?, do: Config.get([:chat, :enabled])
|
||||
|
||||
defp streamer_child(env) when env in [:test, :benchmark], do: []
|
||||
|
||||
|
|
@ -169,13 +183,6 @@ defmodule Pleroma.Application do
|
|||
|
||||
defp chat_child(_, _), do: []
|
||||
|
||||
defp hackney_pool_children do
|
||||
for pool <- enabled_hackney_pools() do
|
||||
options = Pleroma.Config.get([:hackney_pools, pool])
|
||||
:hackney_pool.child_spec(pool, options)
|
||||
end
|
||||
end
|
||||
|
||||
defp task_children(:test) do
|
||||
[
|
||||
%{
|
||||
|
|
@ -200,4 +207,31 @@ defmodule Pleroma.Application do
|
|||
}
|
||||
]
|
||||
end
|
||||
|
||||
# start hackney and gun pools in tests
|
||||
defp http_children(_, :test) do
|
||||
hackney_options = Config.get([:hackney_pools, :federation])
|
||||
hackney_pool = :hackney_pool.child_spec(:federation, hackney_options)
|
||||
[hackney_pool, Pleroma.Pool.Supervisor]
|
||||
end
|
||||
|
||||
defp http_children(Tesla.Adapter.Hackney, _) do
|
||||
pools = [:federation, :media]
|
||||
|
||||
pools =
|
||||
if Config.get([Pleroma.Upload, :proxy_remote]) do
|
||||
[:upload | pools]
|
||||
else
|
||||
pools
|
||||
end
|
||||
|
||||
for pool <- pools do
|
||||
options = Config.get([:hackney_pools, pool])
|
||||
:hackney_pool.child_spec(pool, options)
|
||||
end
|
||||
end
|
||||
|
||||
defp http_children(Tesla.Adapter.Gun, _), do: [Pleroma.Pool.Supervisor]
|
||||
|
||||
defp http_children(_, _), do: []
|
||||
end
|
||||
|
|
|
|||
|
|
@ -278,8 +278,6 @@ defmodule Pleroma.ConfigDB do
|
|||
}
|
||||
end
|
||||
|
||||
defp do_convert({:partial_chain, entity}), do: %{"tuple" => [":partial_chain", inspect(entity)]}
|
||||
|
||||
defp do_convert(entity) when is_tuple(entity) do
|
||||
value =
|
||||
entity
|
||||
|
|
@ -323,15 +321,6 @@ defmodule Pleroma.ConfigDB do
|
|||
{:proxy_url, {do_transform_string(type), parse_host(host), port}}
|
||||
end
|
||||
|
||||
defp do_transform(%{"tuple" => [":partial_chain", entity]}) do
|
||||
{partial_chain, []} =
|
||||
entity
|
||||
|> String.replace(~r/[^\w|^{:,[|^,|^[|^\]^}|^\/|^\.|^"]^\s/, "")
|
||||
|> Code.eval_string()
|
||||
|
||||
{:partial_chain, partial_chain}
|
||||
end
|
||||
|
||||
defp do_transform(%{"tuple" => entity}) do
|
||||
Enum.reduce(entity, {}, fn val, acc -> Tuple.append(acc, do_transform(val)) end)
|
||||
end
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@
|
|||
defmodule Pleroma.Config.TransferTask do
|
||||
use Task
|
||||
|
||||
alias Pleroma.Config
|
||||
alias Pleroma.ConfigDB
|
||||
alias Pleroma.Repo
|
||||
|
||||
|
|
@ -18,7 +19,9 @@ defmodule Pleroma.Config.TransferTask do
|
|||
{:pleroma, Oban},
|
||||
{:pleroma, :rate_limit},
|
||||
{:pleroma, :markup},
|
||||
{:plerome, :streamer}
|
||||
{:pleroma, :streamer},
|
||||
{:pleroma, :pools},
|
||||
{:pleroma, :connections_pool}
|
||||
]
|
||||
|
||||
@reboot_time_subkeys [
|
||||
|
|
@ -32,45 +35,33 @@ defmodule Pleroma.Config.TransferTask do
|
|||
{:pleroma, :gopher, [:enabled]}
|
||||
]
|
||||
|
||||
@reject [nil, :prometheus]
|
||||
|
||||
def start_link(_) do
|
||||
load_and_update_env()
|
||||
if Pleroma.Config.get(:env) == :test, do: Ecto.Adapters.SQL.Sandbox.checkin(Repo)
|
||||
if Config.get(:env) == :test, do: Ecto.Adapters.SQL.Sandbox.checkin(Repo)
|
||||
:ignore
|
||||
end
|
||||
|
||||
@spec load_and_update_env([ConfigDB.t()]) :: :ok | false
|
||||
def load_and_update_env(deleted \\ [], restart_pleroma? \\ true) do
|
||||
with {:configurable, true} <-
|
||||
{:configurable, Pleroma.Config.get(:configurable_from_database)},
|
||||
true <- Ecto.Adapters.SQL.table_exists?(Repo, "config"),
|
||||
started_applications <- Application.started_applications() do
|
||||
@spec load_and_update_env([ConfigDB.t()], boolean()) :: :ok
|
||||
def load_and_update_env(deleted_settings \\ [], restart_pleroma? \\ true) do
|
||||
with {_, true} <- {:configurable, Config.get(:configurable_from_database)} do
|
||||
# We need to restart applications for loaded settings take effect
|
||||
|
||||
in_db = Repo.all(ConfigDB)
|
||||
|
||||
with_deleted = in_db ++ deleted
|
||||
|
||||
reject_for_restart = if restart_pleroma?, do: @reject, else: [:pleroma | @reject]
|
||||
|
||||
applications =
|
||||
with_deleted
|
||||
|> Enum.map(&merge_and_update(&1))
|
||||
|> Enum.uniq()
|
||||
# TODO: some problem with prometheus after restart!
|
||||
|> Enum.reject(&(&1 in reject_for_restart))
|
||||
|
||||
# to be ensured that pleroma will be restarted last
|
||||
applications =
|
||||
if :pleroma in applications do
|
||||
List.delete(applications, :pleroma) ++ [:pleroma]
|
||||
# TODO: some problem with prometheus after restart!
|
||||
reject_restart =
|
||||
if restart_pleroma? do
|
||||
[nil, :prometheus]
|
||||
else
|
||||
Restarter.Pleroma.rebooted()
|
||||
applications
|
||||
[:pleroma, nil, :prometheus]
|
||||
end
|
||||
|
||||
Enum.each(applications, &restart(started_applications, &1, Pleroma.Config.get(:env)))
|
||||
started_applications = Application.started_applications()
|
||||
|
||||
(Repo.all(ConfigDB) ++ deleted_settings)
|
||||
|> Enum.map(&merge_and_update/1)
|
||||
|> Enum.uniq()
|
||||
|> Enum.reject(&(&1 in reject_restart))
|
||||
|> maybe_set_pleroma_last()
|
||||
|> Enum.each(&restart(started_applications, &1, Config.get(:env)))
|
||||
|
||||
:ok
|
||||
else
|
||||
|
|
@ -78,42 +69,54 @@ defmodule Pleroma.Config.TransferTask do
|
|||
end
|
||||
end
|
||||
|
||||
defp maybe_set_pleroma_last(apps) do
|
||||
# to be ensured that pleroma will be restarted last
|
||||
if :pleroma in apps do
|
||||
apps
|
||||
|> List.delete(:pleroma)
|
||||
|> List.insert_at(-1, :pleroma)
|
||||
else
|
||||
Restarter.Pleroma.rebooted()
|
||||
apps
|
||||
end
|
||||
end
|
||||
|
||||
defp group_for_restart(:logger, key, _, merged_value) do
|
||||
# change logger configuration in runtime, without restart
|
||||
if Keyword.keyword?(merged_value) and
|
||||
key not in [:compile_time_application, :backends, :compile_time_purge_matching] do
|
||||
Logger.configure_backend(key, merged_value)
|
||||
else
|
||||
Logger.configure([{key, merged_value}])
|
||||
end
|
||||
|
||||
nil
|
||||
end
|
||||
|
||||
defp group_for_restart(group, _, _, _) when group != :pleroma, do: group
|
||||
|
||||
defp group_for_restart(group, key, value, _) do
|
||||
if pleroma_need_restart?(group, key, value), do: group
|
||||
end
|
||||
|
||||
defp merge_and_update(setting) do
|
||||
try do
|
||||
key = ConfigDB.from_string(setting.key)
|
||||
group = ConfigDB.from_string(setting.group)
|
||||
|
||||
default = Pleroma.Config.Holder.default_config(group, key)
|
||||
default = Config.Holder.default_config(group, key)
|
||||
value = ConfigDB.from_binary(setting.value)
|
||||
|
||||
merged_value =
|
||||
if Ecto.get_meta(setting, :state) == :deleted do
|
||||
default
|
||||
else
|
||||
if can_be_merged?(default, value) do
|
||||
ConfigDB.merge_group(group, key, default, value)
|
||||
else
|
||||
value
|
||||
end
|
||||
cond do
|
||||
Ecto.get_meta(setting, :state) == :deleted -> default
|
||||
can_be_merged?(default, value) -> ConfigDB.merge_group(group, key, default, value)
|
||||
true -> value
|
||||
end
|
||||
|
||||
:ok = update_env(group, key, merged_value)
|
||||
|
||||
if group != :logger do
|
||||
if group != :pleroma or pleroma_need_restart?(group, key, value) do
|
||||
group
|
||||
end
|
||||
else
|
||||
# change logger configuration in runtime, without restart
|
||||
if Keyword.keyword?(merged_value) and
|
||||
key not in [:compile_time_application, :backends, :compile_time_purge_matching] do
|
||||
Logger.configure_backend(key, merged_value)
|
||||
else
|
||||
Logger.configure([{key, merged_value}])
|
||||
end
|
||||
|
||||
nil
|
||||
end
|
||||
group_for_restart(group, key, value, merged_value)
|
||||
rescue
|
||||
error ->
|
||||
error_msg =
|
||||
|
|
|
|||
45
lib/pleroma/gun/api.ex
Normal file
45
lib/pleroma/gun/api.ex
Normal file
|
|
@ -0,0 +1,45 @@
|
|||
# Pleroma: A lightweight social networking server
|
||||
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.Gun.API do
|
||||
@behaviour Pleroma.Gun
|
||||
|
||||
alias Pleroma.Gun
|
||||
|
||||
@gun_keys [
|
||||
:connect_timeout,
|
||||
:http_opts,
|
||||
:http2_opts,
|
||||
:protocols,
|
||||
:retry,
|
||||
:retry_timeout,
|
||||
:trace,
|
||||
:transport,
|
||||
:tls_opts,
|
||||
:tcp_opts,
|
||||
:socks_opts,
|
||||
:ws_opts
|
||||
]
|
||||
|
||||
@impl Gun
|
||||
def open(host, port, opts \\ %{}), do: :gun.open(host, port, Map.take(opts, @gun_keys))
|
||||
|
||||
@impl Gun
|
||||
defdelegate info(pid), to: :gun
|
||||
|
||||
@impl Gun
|
||||
defdelegate close(pid), to: :gun
|
||||
|
||||
@impl Gun
|
||||
defdelegate await_up(pid, timeout \\ 5_000), to: :gun
|
||||
|
||||
@impl Gun
|
||||
defdelegate connect(pid, opts), to: :gun
|
||||
|
||||
@impl Gun
|
||||
defdelegate await(pid, ref), to: :gun
|
||||
|
||||
@impl Gun
|
||||
defdelegate set_owner(pid, owner), to: :gun
|
||||
end
|
||||
196
lib/pleroma/gun/conn.ex
Normal file
196
lib/pleroma/gun/conn.ex
Normal file
|
|
@ -0,0 +1,196 @@
|
|||
# Pleroma: A lightweight social networking server
|
||||
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.Gun.Conn do
|
||||
@moduledoc """
|
||||
Struct for gun connection data
|
||||
"""
|
||||
alias Pleroma.Gun
|
||||
alias Pleroma.Pool.Connections
|
||||
|
||||
require Logger
|
||||
|
||||
@type gun_state :: :up | :down
|
||||
@type conn_state :: :active | :idle
|
||||
|
||||
@type t :: %__MODULE__{
|
||||
conn: pid(),
|
||||
gun_state: gun_state(),
|
||||
conn_state: conn_state(),
|
||||
used_by: [pid()],
|
||||
last_reference: pos_integer(),
|
||||
crf: float(),
|
||||
retries: pos_integer()
|
||||
}
|
||||
|
||||
defstruct conn: nil,
|
||||
gun_state: :open,
|
||||
conn_state: :init,
|
||||
used_by: [],
|
||||
last_reference: 0,
|
||||
crf: 1,
|
||||
retries: 0
|
||||
|
||||
@spec open(String.t() | URI.t(), atom(), keyword()) :: :ok | nil
|
||||
def open(url, name, opts \\ [])
|
||||
def open(url, name, opts) when is_binary(url), do: open(URI.parse(url), name, opts)
|
||||
|
||||
def open(%URI{} = uri, name, opts) do
|
||||
pool_opts = Pleroma.Config.get([:connections_pool], [])
|
||||
|
||||
opts =
|
||||
opts
|
||||
|> Enum.into(%{})
|
||||
|> Map.put_new(:retry, pool_opts[:retry] || 1)
|
||||
|> Map.put_new(:retry_timeout, pool_opts[:retry_timeout] || 1000)
|
||||
|> Map.put_new(:await_up_timeout, pool_opts[:await_up_timeout] || 5_000)
|
||||
|> maybe_add_tls_opts(uri)
|
||||
|
||||
key = "#{uri.scheme}:#{uri.host}:#{uri.port}"
|
||||
|
||||
conn_pid =
|
||||
if Connections.count(name) < opts[:max_connection] do
|
||||
do_open(uri, opts)
|
||||
else
|
||||
close_least_used_and_do_open(name, uri, opts)
|
||||
end
|
||||
|
||||
if is_pid(conn_pid) do
|
||||
conn = %Pleroma.Gun.Conn{
|
||||
conn: conn_pid,
|
||||
gun_state: :up,
|
||||
conn_state: :active,
|
||||
last_reference: :os.system_time(:second)
|
||||
}
|
||||
|
||||
:ok = Gun.set_owner(conn_pid, Process.whereis(name))
|
||||
Connections.add_conn(name, key, conn)
|
||||
end
|
||||
end
|
||||
|
||||
defp maybe_add_tls_opts(opts, %URI{scheme: "http"}), do: opts
|
||||
|
||||
defp maybe_add_tls_opts(opts, %URI{scheme: "https", host: host}) do
|
||||
tls_opts = [
|
||||
verify: :verify_peer,
|
||||
cacertfile: CAStore.file_path(),
|
||||
depth: 20,
|
||||
reuse_sessions: false,
|
||||
verify_fun:
|
||||
{&:ssl_verify_hostname.verify_fun/3,
|
||||
[check_hostname: Pleroma.HTTP.Connection.format_host(host)]}
|
||||
]
|
||||
|
||||
tls_opts =
|
||||
if Keyword.keyword?(opts[:tls_opts]) do
|
||||
Keyword.merge(tls_opts, opts[:tls_opts])
|
||||
else
|
||||
tls_opts
|
||||
end
|
||||
|
||||
Map.put(opts, :tls_opts, tls_opts)
|
||||
end
|
||||
|
||||
defp do_open(uri, %{proxy: {proxy_host, proxy_port}} = opts) do
|
||||
connect_opts =
|
||||
uri
|
||||
|> destination_opts()
|
||||
|> add_http2_opts(uri.scheme, Map.get(opts, :tls_opts, []))
|
||||
|
||||
with open_opts <- Map.delete(opts, :tls_opts),
|
||||
{:ok, conn} <- Gun.open(proxy_host, proxy_port, open_opts),
|
||||
{:ok, _} <- Gun.await_up(conn, opts[:await_up_timeout]),
|
||||
stream <- Gun.connect(conn, connect_opts),
|
||||
{:response, :fin, 200, _} <- Gun.await(conn, stream) do
|
||||
conn
|
||||
else
|
||||
error ->
|
||||
Logger.warn(
|
||||
"Opening proxied connection to #{compose_uri_log(uri)} failed with error #{
|
||||
inspect(error)
|
||||
}"
|
||||
)
|
||||
|
||||
error
|
||||
end
|
||||
end
|
||||
|
||||
defp do_open(uri, %{proxy: {proxy_type, proxy_host, proxy_port}} = opts) do
|
||||
version =
|
||||
proxy_type
|
||||
|> to_string()
|
||||
|> String.last()
|
||||
|> case do
|
||||
"4" -> 4
|
||||
_ -> 5
|
||||
end
|
||||
|
||||
socks_opts =
|
||||
uri
|
||||
|> destination_opts()
|
||||
|> add_http2_opts(uri.scheme, Map.get(opts, :tls_opts, []))
|
||||
|> Map.put(:version, version)
|
||||
|
||||
opts =
|
||||
opts
|
||||
|> Map.put(:protocols, [:socks])
|
||||
|> Map.put(:socks_opts, socks_opts)
|
||||
|
||||
with {:ok, conn} <- Gun.open(proxy_host, proxy_port, opts),
|
||||
{:ok, _} <- Gun.await_up(conn, opts[:await_up_timeout]) do
|
||||
conn
|
||||
else
|
||||
error ->
|
||||
Logger.warn(
|
||||
"Opening socks proxied connection to #{compose_uri_log(uri)} failed with error #{
|
||||
inspect(error)
|
||||
}"
|
||||
)
|
||||
|
||||
error
|
||||
end
|
||||
end
|
||||
|
||||
defp do_open(%URI{host: host, port: port} = uri, opts) do
|
||||
host = Pleroma.HTTP.Connection.parse_host(host)
|
||||
|
||||
with {:ok, conn} <- Gun.open(host, port, opts),
|
||||
{:ok, _} <- Gun.await_up(conn, opts[:await_up_timeout]) do
|
||||
conn
|
||||
else
|
||||
error ->
|
||||
Logger.warn(
|
||||
"Opening connection to #{compose_uri_log(uri)} failed with error #{inspect(error)}"
|
||||
)
|
||||
|
||||
error
|
||||
end
|
||||
end
|
||||
|
||||
defp destination_opts(%URI{host: host, port: port}) do
|
||||
host = Pleroma.HTTP.Connection.parse_host(host)
|
||||
%{host: host, port: port}
|
||||
end
|
||||
|
||||
defp add_http2_opts(opts, "https", tls_opts) do
|
||||
Map.merge(opts, %{protocols: [:http2], transport: :tls, tls_opts: tls_opts})
|
||||
end
|
||||
|
||||
defp add_http2_opts(opts, _, _), do: opts
|
||||
|
||||
defp close_least_used_and_do_open(name, uri, opts) do
|
||||
with [{key, conn} | _conns] <- Connections.get_unused_conns(name),
|
||||
:ok <- Gun.close(conn.conn) do
|
||||
Connections.remove_conn(name, key)
|
||||
|
||||
do_open(uri, opts)
|
||||
else
|
||||
[] -> {:error, :pool_overflowed}
|
||||
end
|
||||
end
|
||||
|
||||
def compose_uri_log(%URI{scheme: scheme, host: host, path: path}) do
|
||||
"#{scheme}://#{host}#{path}"
|
||||
end
|
||||
end
|
||||
31
lib/pleroma/gun/gun.ex
Normal file
31
lib/pleroma/gun/gun.ex
Normal file
|
|
@ -0,0 +1,31 @@
|
|||
# Pleroma: A lightweight social networking server
|
||||
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.Gun do
|
||||
@callback open(charlist(), pos_integer(), map()) :: {:ok, pid()}
|
||||
@callback info(pid()) :: map()
|
||||
@callback close(pid()) :: :ok
|
||||
@callback await_up(pid, pos_integer()) :: {:ok, atom()} | {:error, atom()}
|
||||
@callback connect(pid(), map()) :: reference()
|
||||
@callback await(pid(), reference()) :: {:response, :fin, 200, []}
|
||||
@callback set_owner(pid(), pid()) :: :ok
|
||||
|
||||
@api Pleroma.Config.get([Pleroma.Gun], Pleroma.Gun.API)
|
||||
|
||||
defp api, do: @api
|
||||
|
||||
def open(host, port, opts), do: api().open(host, port, opts)
|
||||
|
||||
def info(pid), do: api().info(pid)
|
||||
|
||||
def close(pid), do: api().close(pid)
|
||||
|
||||
def await_up(pid, timeout \\ 5_000), do: api().await_up(pid, timeout)
|
||||
|
||||
def connect(pid, opts), do: api().connect(pid, opts)
|
||||
|
||||
def await(pid, ref), do: api().await(pid, ref)
|
||||
|
||||
def set_owner(pid, owner), do: api().set_owner(pid, owner)
|
||||
end
|
||||
41
lib/pleroma/http/adapter_helper.ex
Normal file
41
lib/pleroma/http/adapter_helper.ex
Normal file
|
|
@ -0,0 +1,41 @@
|
|||
# Pleroma: A lightweight social networking server
|
||||
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.HTTP.AdapterHelper do
|
||||
alias Pleroma.HTTP.Connection
|
||||
|
||||
@type proxy ::
|
||||
{Connection.host(), pos_integer()}
|
||||
| {Connection.proxy_type(), Connection.host(), pos_integer()}
|
||||
|
||||
@callback options(keyword(), URI.t()) :: keyword()
|
||||
@callback after_request(keyword()) :: :ok
|
||||
|
||||
@spec options(keyword(), URI.t()) :: keyword()
|
||||
def options(opts, _uri) do
|
||||
proxy = Pleroma.Config.get([:http, :proxy_url], nil)
|
||||
maybe_add_proxy(opts, format_proxy(proxy))
|
||||
end
|
||||
|
||||
@spec maybe_get_conn(URI.t(), keyword()) :: keyword()
|
||||
def maybe_get_conn(_uri, opts), do: opts
|
||||
|
||||
@spec after_request(keyword()) :: :ok
|
||||
def after_request(_opts), do: :ok
|
||||
|
||||
@spec format_proxy(String.t() | tuple() | nil) :: proxy() | nil
|
||||
def format_proxy(nil), do: nil
|
||||
|
||||
def format_proxy(proxy_url) do
|
||||
case Connection.parse_proxy(proxy_url) do
|
||||
{:ok, host, port} -> {host, port}
|
||||
{:ok, type, host, port} -> {type, host, port}
|
||||
_ -> nil
|
||||
end
|
||||
end
|
||||
|
||||
@spec maybe_add_proxy(keyword(), proxy() | nil) :: keyword()
|
||||
def maybe_add_proxy(opts, nil), do: opts
|
||||
def maybe_add_proxy(opts, proxy), do: Keyword.put_new(opts, :proxy, proxy)
|
||||
end
|
||||
77
lib/pleroma/http/adapter_helper/gun.ex
Normal file
77
lib/pleroma/http/adapter_helper/gun.ex
Normal file
|
|
@ -0,0 +1,77 @@
|
|||
# Pleroma: A lightweight social networking server
|
||||
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.HTTP.AdapterHelper.Gun do
|
||||
@behaviour Pleroma.HTTP.AdapterHelper
|
||||
|
||||
alias Pleroma.HTTP.AdapterHelper
|
||||
alias Pleroma.Pool.Connections
|
||||
|
||||
require Logger
|
||||
|
||||
@defaults [
|
||||
connect_timeout: 5_000,
|
||||
domain_lookup_timeout: 5_000,
|
||||
tls_handshake_timeout: 5_000,
|
||||
retry: 1,
|
||||
retry_timeout: 1000,
|
||||
await_up_timeout: 5_000
|
||||
]
|
||||
|
||||
@spec options(keyword(), URI.t()) :: keyword()
|
||||
def options(incoming_opts \\ [], %URI{} = uri) do
|
||||
proxy =
|
||||
Pleroma.Config.get([:http, :proxy_url])
|
||||
|> AdapterHelper.format_proxy()
|
||||
|
||||
config_opts = Pleroma.Config.get([:http, :adapter], [])
|
||||
|
||||
@defaults
|
||||
|> Keyword.merge(config_opts)
|
||||
|> add_scheme_opts(uri)
|
||||
|> AdapterHelper.maybe_add_proxy(proxy)
|
||||
|> maybe_get_conn(uri, incoming_opts)
|
||||
end
|
||||
|
||||
@spec after_request(keyword()) :: :ok
|
||||
def after_request(opts) do
|
||||
if opts[:conn] && opts[:body_as] != :chunks do
|
||||
Connections.checkout(opts[:conn], self(), :gun_connections)
|
||||
end
|
||||
|
||||
:ok
|
||||
end
|
||||
|
||||
defp add_scheme_opts(opts, %{scheme: "http"}), do: opts
|
||||
|
||||
defp add_scheme_opts(opts, %{scheme: "https"}) do
|
||||
opts
|
||||
|> Keyword.put(:certificates_verification, true)
|
||||
|> Keyword.put(:tls_opts, log_level: :warning)
|
||||
end
|
||||
|
||||
defp maybe_get_conn(adapter_opts, uri, incoming_opts) do
|
||||
{receive_conn?, opts} =
|
||||
adapter_opts
|
||||
|> Keyword.merge(incoming_opts)
|
||||
|> Keyword.pop(:receive_conn, true)
|
||||
|
||||
if Connections.alive?(:gun_connections) and receive_conn? do
|
||||
checkin_conn(uri, opts)
|
||||
else
|
||||
opts
|
||||
end
|
||||
end
|
||||
|
||||
defp checkin_conn(uri, opts) do
|
||||
case Connections.checkin(uri, :gun_connections) do
|
||||
nil ->
|
||||
Task.start(Pleroma.Gun.Conn, :open, [uri, :gun_connections, opts])
|
||||
opts
|
||||
|
||||
conn when is_pid(conn) ->
|
||||
Keyword.merge(opts, conn: conn, close_conn: false)
|
||||
end
|
||||
end
|
||||
end
|
||||
43
lib/pleroma/http/adapter_helper/hackney.ex
Normal file
43
lib/pleroma/http/adapter_helper/hackney.ex
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
defmodule Pleroma.HTTP.AdapterHelper.Hackney do
|
||||
@behaviour Pleroma.HTTP.AdapterHelper
|
||||
|
||||
@defaults [
|
||||
connect_timeout: 10_000,
|
||||
recv_timeout: 20_000,
|
||||
follow_redirect: true,
|
||||
force_redirect: true,
|
||||
pool: :federation
|
||||
]
|
||||
|
||||
@spec options(keyword(), URI.t()) :: keyword()
|
||||
def options(connection_opts \\ [], %URI{} = uri) do
|
||||
proxy = Pleroma.Config.get([:http, :proxy_url])
|
||||
|
||||
config_opts = Pleroma.Config.get([:http, :adapter], [])
|
||||
|
||||
@defaults
|
||||
|> Keyword.merge(config_opts)
|
||||
|> Keyword.merge(connection_opts)
|
||||
|> add_scheme_opts(uri)
|
||||
|> Pleroma.HTTP.AdapterHelper.maybe_add_proxy(proxy)
|
||||
end
|
||||
|
||||
defp add_scheme_opts(opts, %URI{scheme: "http"}), do: opts
|
||||
|
||||
defp add_scheme_opts(opts, %URI{scheme: "https", host: host}) do
|
||||
ssl_opts = [
|
||||
ssl_options: [
|
||||
# Workaround for remote server certificate chain issues
|
||||
partial_chain: &:hackney_connect.partial_chain/1,
|
||||
|
||||
# We don't support TLS v1.3 yet
|
||||
versions: [:tlsv1, :"tlsv1.1", :"tlsv1.2"],
|
||||
server_name_indication: to_charlist(host)
|
||||
]
|
||||
]
|
||||
|
||||
Keyword.merge(opts, ssl_opts)
|
||||
end
|
||||
|
||||
def after_request(_), do: :ok
|
||||
end
|
||||
|
|
@ -4,40 +4,121 @@
|
|||
|
||||
defmodule Pleroma.HTTP.Connection do
|
||||
@moduledoc """
|
||||
Connection for http-requests.
|
||||
Configure Tesla.Client with default and customized adapter options.
|
||||
"""
|
||||
|
||||
@hackney_options [
|
||||
connect_timeout: 10_000,
|
||||
recv_timeout: 20_000,
|
||||
follow_redirect: true,
|
||||
force_redirect: true,
|
||||
pool: :federation
|
||||
]
|
||||
@adapter Application.get_env(:tesla, :adapter)
|
||||
alias Pleroma.Config
|
||||
alias Pleroma.HTTP.AdapterHelper
|
||||
|
||||
require Logger
|
||||
|
||||
@defaults [pool: :federation]
|
||||
|
||||
@type ip_address :: ipv4_address() | ipv6_address()
|
||||
@type ipv4_address :: {0..255, 0..255, 0..255, 0..255}
|
||||
@type ipv6_address ::
|
||||
{0..65_535, 0..65_535, 0..65_535, 0..65_535, 0..65_535, 0..65_535, 0..65_535, 0..65_535}
|
||||
@type proxy_type() :: :socks4 | :socks5
|
||||
@type host() :: charlist() | ip_address()
|
||||
|
||||
@doc """
|
||||
Configure a client connection
|
||||
|
||||
# Returns
|
||||
|
||||
Tesla.Env.client
|
||||
Merge default connection & adapter options with received ones.
|
||||
"""
|
||||
@spec new(Keyword.t()) :: Tesla.Env.client()
|
||||
def new(opts \\ []) do
|
||||
Tesla.client([], {@adapter, hackney_options(opts)})
|
||||
|
||||
@spec options(URI.t(), keyword()) :: keyword()
|
||||
def options(%URI{} = uri, opts \\ []) do
|
||||
@defaults
|
||||
|> pool_timeout()
|
||||
|> Keyword.merge(opts)
|
||||
|> adapter_helper().options(uri)
|
||||
end
|
||||
|
||||
# fetch Hackney options
|
||||
#
|
||||
def hackney_options(opts) do
|
||||
options = Keyword.get(opts, :adapter, [])
|
||||
adapter_options = Pleroma.Config.get([:http, :adapter], [])
|
||||
proxy_url = Pleroma.Config.get([:http, :proxy_url], nil)
|
||||
defp pool_timeout(opts) do
|
||||
{config_key, default} =
|
||||
if adapter() == Tesla.Adapter.Gun do
|
||||
{:pools, Config.get([:pools, :default, :timeout])}
|
||||
else
|
||||
{:hackney_pools, 10_000}
|
||||
end
|
||||
|
||||
@hackney_options
|
||||
|> Keyword.merge(adapter_options)
|
||||
|> Keyword.merge(options)
|
||||
|> Keyword.merge(proxy: proxy_url)
|
||||
timeout = Config.get([config_key, opts[:pool], :timeout], default)
|
||||
|
||||
Keyword.merge(opts, timeout: timeout)
|
||||
end
|
||||
|
||||
@spec after_request(keyword()) :: :ok
|
||||
def after_request(opts), do: adapter_helper().after_request(opts)
|
||||
|
||||
defp adapter, do: Application.get_env(:tesla, :adapter)
|
||||
|
||||
defp adapter_helper do
|
||||
case adapter() do
|
||||
Tesla.Adapter.Gun -> AdapterHelper.Gun
|
||||
Tesla.Adapter.Hackney -> AdapterHelper.Hackney
|
||||
_ -> AdapterHelper
|
||||
end
|
||||
end
|
||||
|
||||
@spec parse_proxy(String.t() | tuple() | nil) ::
|
||||
{:ok, host(), pos_integer()}
|
||||
| {:ok, proxy_type(), host(), pos_integer()}
|
||||
| {:error, atom()}
|
||||
| nil
|
||||
|
||||
def parse_proxy(nil), do: nil
|
||||
|
||||
def parse_proxy(proxy) when is_binary(proxy) do
|
||||
with [host, port] <- String.split(proxy, ":"),
|
||||
{port, ""} <- Integer.parse(port) do
|
||||
{:ok, parse_host(host), port}
|
||||
else
|
||||
{_, _} ->
|
||||
Logger.warn("Parsing port failed #{inspect(proxy)}")
|
||||
{:error, :invalid_proxy_port}
|
||||
|
||||
:error ->
|
||||
Logger.warn("Parsing port failed #{inspect(proxy)}")
|
||||
{:error, :invalid_proxy_port}
|
||||
|
||||
_ ->
|
||||
Logger.warn("Parsing proxy failed #{inspect(proxy)}")
|
||||
{:error, :invalid_proxy}
|
||||
end
|
||||
end
|
||||
|
||||
def parse_proxy(proxy) when is_tuple(proxy) do
|
||||
with {type, host, port} <- proxy do
|
||||
{:ok, type, parse_host(host), port}
|
||||
else
|
||||
_ ->
|
||||
Logger.warn("Parsing proxy failed #{inspect(proxy)}")
|
||||
{:error, :invalid_proxy}
|
||||
end
|
||||
end
|
||||
|
||||
@spec parse_host(String.t() | atom() | charlist()) :: charlist() | ip_address()
|
||||
def parse_host(host) when is_list(host), do: host
|
||||
def parse_host(host) when is_atom(host), do: to_charlist(host)
|
||||
|
||||
def parse_host(host) when is_binary(host) do
|
||||
host = to_charlist(host)
|
||||
|
||||
case :inet.parse_address(host) do
|
||||
{:error, :einval} -> host
|
||||
{:ok, ip} -> ip
|
||||
end
|
||||
end
|
||||
|
||||
@spec format_host(String.t()) :: charlist()
|
||||
def format_host(host) do
|
||||
host_charlist = to_charlist(host)
|
||||
|
||||
case :inet.parse_address(host_charlist) do
|
||||
{:error, :einval} ->
|
||||
:idna.encode(host_charlist)
|
||||
|
||||
{:ok, _ip} ->
|
||||
host_charlist
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -4,21 +4,47 @@
|
|||
|
||||
defmodule Pleroma.HTTP do
|
||||
@moduledoc """
|
||||
|
||||
Wrapper for `Tesla.request/2`.
|
||||
"""
|
||||
|
||||
alias Pleroma.HTTP.Connection
|
||||
alias Pleroma.HTTP.Request
|
||||
alias Pleroma.HTTP.RequestBuilder, as: Builder
|
||||
alias Tesla.Client
|
||||
alias Tesla.Env
|
||||
|
||||
require Logger
|
||||
|
||||
@type t :: __MODULE__
|
||||
|
||||
@doc """
|
||||
Builds and perform http request.
|
||||
Performs GET request.
|
||||
|
||||
See `Pleroma.HTTP.request/5`
|
||||
"""
|
||||
@spec get(Request.url() | nil, Request.headers(), keyword()) ::
|
||||
nil | {:ok, Env.t()} | {:error, any()}
|
||||
def get(url, headers \\ [], options \\ [])
|
||||
def get(nil, _, _), do: nil
|
||||
def get(url, headers, options), do: request(:get, url, "", headers, options)
|
||||
|
||||
@doc """
|
||||
Performs POST request.
|
||||
|
||||
See `Pleroma.HTTP.request/5`
|
||||
"""
|
||||
@spec post(Request.url(), String.t(), Request.headers(), keyword()) ::
|
||||
{:ok, Env.t()} | {:error, any()}
|
||||
def post(url, body, headers \\ [], options \\ []),
|
||||
do: request(:post, url, body, headers, options)
|
||||
|
||||
@doc """
|
||||
Builds and performs http request.
|
||||
|
||||
# Arguments:
|
||||
`method` - :get, :post, :put, :delete
|
||||
`url`
|
||||
`body`
|
||||
`url` - full url
|
||||
`body` - request body
|
||||
`headers` - a keyworld list of headers, e.g. `[{"content-type", "text/plain"}]`
|
||||
`options` - custom, per-request middleware or adapter options
|
||||
|
||||
|
|
@ -26,61 +52,66 @@ defmodule Pleroma.HTTP do
|
|||
`{:ok, %Tesla.Env{}}` or `{:error, error}`
|
||||
|
||||
"""
|
||||
def request(method, url, body \\ "", headers \\ [], options \\ []) do
|
||||
try do
|
||||
options =
|
||||
process_request_options(options)
|
||||
|> process_sni_options(url)
|
||||
|
||||
params = Keyword.get(options, :params, [])
|
||||
|
||||
%{}
|
||||
|> Builder.method(method)
|
||||
|> Builder.headers(headers)
|
||||
|> Builder.opts(options)
|
||||
|> Builder.url(url)
|
||||
|> Builder.add_param(:body, :body, body)
|
||||
|> Builder.add_param(:query, :query, params)
|
||||
|> Enum.into([])
|
||||
|> (&Tesla.request(Connection.new(options), &1)).()
|
||||
rescue
|
||||
e ->
|
||||
{:error, e}
|
||||
catch
|
||||
:exit, e ->
|
||||
{:error, e}
|
||||
end
|
||||
end
|
||||
|
||||
defp process_sni_options(options, nil), do: options
|
||||
|
||||
defp process_sni_options(options, url) do
|
||||
@spec request(atom(), Request.url(), String.t(), Request.headers(), keyword()) ::
|
||||
{:ok, Env.t()} | {:error, any()}
|
||||
def request(method, url, body, headers, options) when is_binary(url) do
|
||||
uri = URI.parse(url)
|
||||
host = uri.host |> to_charlist()
|
||||
adapter_opts = Connection.options(uri, options[:adapter] || [])
|
||||
options = put_in(options[:adapter], adapter_opts)
|
||||
params = options[:params] || []
|
||||
request = build_request(method, headers, options, url, body, params)
|
||||
|
||||
case uri.scheme do
|
||||
"https" -> options ++ [ssl: [server_name_indication: host]]
|
||||
_ -> options
|
||||
end
|
||||
adapter = Application.get_env(:tesla, :adapter)
|
||||
client = Tesla.client([Tesla.Middleware.FollowRedirects], adapter)
|
||||
|
||||
pid = Process.whereis(adapter_opts[:pool])
|
||||
|
||||
pool_alive? =
|
||||
if adapter == Tesla.Adapter.Gun && pid do
|
||||
Process.alive?(pid)
|
||||
else
|
||||
false
|
||||
end
|
||||
|
||||
request_opts =
|
||||
adapter_opts
|
||||
|> Enum.into(%{})
|
||||
|> Map.put(:env, Pleroma.Config.get([:env]))
|
||||
|> Map.put(:pool_alive?, pool_alive?)
|
||||
|
||||
response = request(client, request, request_opts)
|
||||
|
||||
Connection.after_request(adapter_opts)
|
||||
|
||||
response
|
||||
end
|
||||
|
||||
def process_request_options(options) do
|
||||
Keyword.merge(Pleroma.HTTP.Connection.hackney_options([]), options)
|
||||
@spec request(Client.t(), keyword(), map()) :: {:ok, Env.t()} | {:error, any()}
|
||||
def request(%Client{} = client, request, %{env: :test}), do: request(client, request)
|
||||
|
||||
def request(%Client{} = client, request, %{body_as: :chunks}), do: request(client, request)
|
||||
|
||||
def request(%Client{} = client, request, %{pool_alive?: false}), do: request(client, request)
|
||||
|
||||
def request(%Client{} = client, request, %{pool: pool, timeout: timeout}) do
|
||||
:poolboy.transaction(
|
||||
pool,
|
||||
&Pleroma.Pool.Request.execute(&1, client, request, timeout),
|
||||
timeout
|
||||
)
|
||||
end
|
||||
|
||||
@doc """
|
||||
Performs GET request.
|
||||
@spec request(Client.t(), keyword()) :: {:ok, Env.t()} | {:error, any()}
|
||||
def request(client, request), do: Tesla.request(client, request)
|
||||
|
||||
See `Pleroma.HTTP.request/5`
|
||||
"""
|
||||
def get(url, headers \\ [], options \\ []),
|
||||
do: request(:get, url, "", headers, options)
|
||||
|
||||
@doc """
|
||||
Performs POST request.
|
||||
|
||||
See `Pleroma.HTTP.request/5`
|
||||
"""
|
||||
def post(url, body, headers \\ [], options \\ []),
|
||||
do: request(:post, url, body, headers, options)
|
||||
defp build_request(method, headers, options, url, body, params) do
|
||||
Builder.new()
|
||||
|> Builder.method(method)
|
||||
|> Builder.headers(headers)
|
||||
|> Builder.opts(options)
|
||||
|> Builder.url(url)
|
||||
|> Builder.add_param(:body, :body, body)
|
||||
|> Builder.add_param(:query, :query, params)
|
||||
|> Builder.convert_to_keyword()
|
||||
end
|
||||
end
|
||||
|
|
|
|||
23
lib/pleroma/http/request.ex
Normal file
23
lib/pleroma/http/request.ex
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
# Pleroma: A lightweight social networking server
|
||||
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.HTTP.Request do
|
||||
@moduledoc """
|
||||
Request struct.
|
||||
"""
|
||||
defstruct method: :get, url: "", query: [], headers: [], body: "", opts: []
|
||||
|
||||
@type method :: :head | :get | :delete | :trace | :options | :post | :put | :patch
|
||||
@type url :: String.t()
|
||||
@type headers :: [{String.t(), String.t()}]
|
||||
|
||||
@type t :: %__MODULE__{
|
||||
method: method(),
|
||||
url: url(),
|
||||
query: keyword(),
|
||||
headers: headers(),
|
||||
body: String.t(),
|
||||
opts: keyword()
|
||||
}
|
||||
end
|
||||
|
|
@ -7,136 +7,87 @@ defmodule Pleroma.HTTP.RequestBuilder do
|
|||
Helper functions for building Tesla requests
|
||||
"""
|
||||
|
||||
alias Pleroma.HTTP.Request
|
||||
alias Tesla.Multipart
|
||||
|
||||
@doc """
|
||||
Specify the request method when building a request
|
||||
|
||||
## Parameters
|
||||
|
||||
- request (Map) - Collected request options
|
||||
- m (atom) - Request method
|
||||
|
||||
## Returns
|
||||
|
||||
Map
|
||||
Creates new request
|
||||
"""
|
||||
@spec method(map(), atom) :: map()
|
||||
def method(request, m) do
|
||||
Map.put_new(request, :method, m)
|
||||
end
|
||||
@spec new(Request.t()) :: Request.t()
|
||||
def new(%Request{} = request \\ %Request{}), do: request
|
||||
|
||||
@doc """
|
||||
Specify the request method when building a request
|
||||
|
||||
## Parameters
|
||||
|
||||
- request (Map) - Collected request options
|
||||
- u (String) - Request URL
|
||||
|
||||
## Returns
|
||||
|
||||
Map
|
||||
"""
|
||||
@spec url(map(), String.t()) :: map()
|
||||
def url(request, u) do
|
||||
Map.put_new(request, :url, u)
|
||||
end
|
||||
@spec method(Request.t(), Request.method()) :: Request.t()
|
||||
def method(request, m), do: %{request | method: m}
|
||||
|
||||
@doc """
|
||||
Specify the request method when building a request
|
||||
"""
|
||||
@spec url(Request.t(), Request.url()) :: Request.t()
|
||||
def url(request, u), do: %{request | url: u}
|
||||
|
||||
@doc """
|
||||
Add headers to the request
|
||||
"""
|
||||
@spec headers(map(), list(tuple)) :: map()
|
||||
def headers(request, header_list) do
|
||||
header_list =
|
||||
@spec headers(Request.t(), Request.headers()) :: Request.t()
|
||||
def headers(request, headers) do
|
||||
headers_list =
|
||||
if Pleroma.Config.get([:http, :send_user_agent]) do
|
||||
header_list ++ [{"User-Agent", Pleroma.Application.user_agent()}]
|
||||
[{"user-agent", Pleroma.Application.user_agent()} | headers]
|
||||
else
|
||||
header_list
|
||||
headers
|
||||
end
|
||||
|
||||
Map.put_new(request, :headers, header_list)
|
||||
%{request | headers: headers_list}
|
||||
end
|
||||
|
||||
@doc """
|
||||
Add custom, per-request middleware or adapter options to the request
|
||||
"""
|
||||
@spec opts(map(), Keyword.t()) :: map()
|
||||
def opts(request, options) do
|
||||
Map.put_new(request, :opts, options)
|
||||
end
|
||||
@spec opts(Request.t(), keyword()) :: Request.t()
|
||||
def opts(request, options), do: %{request | opts: options}
|
||||
|
||||
@doc """
|
||||
Add optional parameters to the request
|
||||
|
||||
## Parameters
|
||||
|
||||
- request (Map) - Collected request options
|
||||
- definitions (Map) - Map of parameter name to parameter location.
|
||||
- options (KeywordList) - The provided optional parameters
|
||||
|
||||
## Returns
|
||||
|
||||
Map
|
||||
"""
|
||||
@spec add_optional_params(map(), %{optional(atom) => atom}, keyword()) :: map()
|
||||
def add_optional_params(request, _, []), do: request
|
||||
@spec add_param(Request.t(), atom(), atom(), any()) :: Request.t()
|
||||
def add_param(request, :query, :query, values), do: %{request | query: values}
|
||||
|
||||
def add_optional_params(request, definitions, [{key, value} | tail]) do
|
||||
case definitions do
|
||||
%{^key => location} ->
|
||||
request
|
||||
|> add_param(location, key, value)
|
||||
|> add_optional_params(definitions, tail)
|
||||
|
||||
_ ->
|
||||
add_optional_params(request, definitions, tail)
|
||||
end
|
||||
end
|
||||
|
||||
@doc """
|
||||
Add optional parameters to the request
|
||||
|
||||
## Parameters
|
||||
|
||||
- request (Map) - Collected request options
|
||||
- location (atom) - Where to put the parameter
|
||||
- key (atom) - The name of the parameter
|
||||
- value (any) - The value of the parameter
|
||||
|
||||
## Returns
|
||||
|
||||
Map
|
||||
"""
|
||||
@spec add_param(map(), atom, atom, any()) :: map()
|
||||
def add_param(request, :query, :query, values), do: Map.put(request, :query, values)
|
||||
|
||||
def add_param(request, :body, :body, value), do: Map.put(request, :body, value)
|
||||
def add_param(request, :body, :body, value), do: %{request | body: value}
|
||||
|
||||
def add_param(request, :body, key, value) do
|
||||
request
|
||||
|> Map.put_new_lazy(:body, &Tesla.Multipart.new/0)
|
||||
|> Map.put(:body, Multipart.new())
|
||||
|> Map.update!(
|
||||
:body,
|
||||
&Tesla.Multipart.add_field(
|
||||
&Multipart.add_field(
|
||||
&1,
|
||||
key,
|
||||
Jason.encode!(value),
|
||||
headers: [{:"Content-Type", "application/json"}]
|
||||
headers: [{"content-type", "application/json"}]
|
||||
)
|
||||
)
|
||||
end
|
||||
|
||||
def add_param(request, :file, name, path) do
|
||||
request
|
||||
|> Map.put_new_lazy(:body, &Tesla.Multipart.new/0)
|
||||
|> Map.update!(:body, &Tesla.Multipart.add_file(&1, path, name: name))
|
||||
|> Map.put(:body, Multipart.new())
|
||||
|> Map.update!(:body, &Multipart.add_file(&1, path, name: name))
|
||||
end
|
||||
|
||||
def add_param(request, :form, name, value) do
|
||||
request
|
||||
|> Map.update(:body, %{name => value}, &Map.put(&1, name, value))
|
||||
Map.update(request, :body, %{name => value}, &Map.put(&1, name, value))
|
||||
end
|
||||
|
||||
def add_param(request, location, key, value) do
|
||||
Map.update(request, location, [{key, value}], &(&1 ++ [{key, value}]))
|
||||
end
|
||||
|
||||
def convert_to_keyword(request) do
|
||||
request
|
||||
|> Map.from_struct()
|
||||
|> Enum.into([])
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -141,7 +141,7 @@ defmodule Pleroma.Object.Fetcher do
|
|||
date: date
|
||||
})
|
||||
|
||||
[{:Signature, signature}]
|
||||
[{"signature", signature}]
|
||||
end
|
||||
|
||||
defp sign_fetch(headers, id, date) do
|
||||
|
|
@ -154,7 +154,7 @@ defmodule Pleroma.Object.Fetcher do
|
|||
|
||||
defp maybe_date_fetch(headers, date) do
|
||||
if Pleroma.Config.get([:activitypub, :sign_object_fetches]) do
|
||||
headers ++ [{:Date, date}]
|
||||
headers ++ [{"date", date}]
|
||||
else
|
||||
headers
|
||||
end
|
||||
|
|
@ -166,7 +166,7 @@ defmodule Pleroma.Object.Fetcher do
|
|||
date = Pleroma.Signature.signed_date()
|
||||
|
||||
headers =
|
||||
[{:Accept, "application/activity+json"}]
|
||||
[{"accept", "application/activity+json"}]
|
||||
|> maybe_date_fetch(date)
|
||||
|> sign_fetch(id, date)
|
||||
|
||||
|
|
|
|||
28
lib/pleroma/otp_version.ex
Normal file
28
lib/pleroma/otp_version.ex
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
# Pleroma: A lightweight social networking server
|
||||
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.OTPVersion do
|
||||
@spec version() :: String.t() | nil
|
||||
def version do
|
||||
# OTP Version https://erlang.org/doc/system_principles/versions.html#otp-version
|
||||
[
|
||||
Path.join(:code.root_dir(), "OTP_VERSION"),
|
||||
Path.join([:code.root_dir(), "releases", :erlang.system_info(:otp_release), "OTP_VERSION"])
|
||||
]
|
||||
|> get_version_from_files()
|
||||
end
|
||||
|
||||
@spec get_version_from_files([Path.t()]) :: String.t() | nil
|
||||
def get_version_from_files([]), do: nil
|
||||
|
||||
def get_version_from_files([path | paths]) do
|
||||
if File.exists?(path) do
|
||||
path
|
||||
|> File.read!()
|
||||
|> String.replace(~r/\r|\n|\s/, "")
|
||||
else
|
||||
get_version_from_files(paths)
|
||||
end
|
||||
end
|
||||
end
|
||||
283
lib/pleroma/pool/connections.ex
Normal file
283
lib/pleroma/pool/connections.ex
Normal file
|
|
@ -0,0 +1,283 @@
|
|||
# Pleroma: A lightweight social networking server
|
||||
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.Pool.Connections do
|
||||
use GenServer
|
||||
|
||||
alias Pleroma.Config
|
||||
alias Pleroma.Gun
|
||||
|
||||
require Logger
|
||||
|
||||
@type domain :: String.t()
|
||||
@type conn :: Pleroma.Gun.Conn.t()
|
||||
|
||||
@type t :: %__MODULE__{
|
||||
conns: %{domain() => conn()},
|
||||
opts: keyword()
|
||||
}
|
||||
|
||||
defstruct conns: %{}, opts: []
|
||||
|
||||
@spec start_link({atom(), keyword()}) :: {:ok, pid()}
|
||||
def start_link({name, opts}) do
|
||||
GenServer.start_link(__MODULE__, opts, name: name)
|
||||
end
|
||||
|
||||
@impl true
|
||||
def init(opts), do: {:ok, %__MODULE__{conns: %{}, opts: opts}}
|
||||
|
||||
@spec checkin(String.t() | URI.t(), atom()) :: pid() | nil
|
||||
def checkin(url, name)
|
||||
def checkin(url, name) when is_binary(url), do: checkin(URI.parse(url), name)
|
||||
|
||||
def checkin(%URI{} = uri, name) do
|
||||
timeout = Config.get([:connections_pool, :checkin_timeout], 250)
|
||||
|
||||
GenServer.call(name, {:checkin, uri}, timeout)
|
||||
end
|
||||
|
||||
@spec alive?(atom()) :: boolean()
|
||||
def alive?(name) do
|
||||
if pid = Process.whereis(name) do
|
||||
Process.alive?(pid)
|
||||
else
|
||||
false
|
||||
end
|
||||
end
|
||||
|
||||
@spec get_state(atom()) :: t()
|
||||
def get_state(name) do
|
||||
GenServer.call(name, :state)
|
||||
end
|
||||
|
||||
@spec count(atom()) :: pos_integer()
|
||||
def count(name) do
|
||||
GenServer.call(name, :count)
|
||||
end
|
||||
|
||||
@spec get_unused_conns(atom()) :: [{domain(), conn()}]
|
||||
def get_unused_conns(name) do
|
||||
GenServer.call(name, :unused_conns)
|
||||
end
|
||||
|
||||
@spec checkout(pid(), pid(), atom()) :: :ok
|
||||
def checkout(conn, pid, name) do
|
||||
GenServer.cast(name, {:checkout, conn, pid})
|
||||
end
|
||||
|
||||
@spec add_conn(atom(), String.t(), Pleroma.Gun.Conn.t()) :: :ok
|
||||
def add_conn(name, key, conn) do
|
||||
GenServer.cast(name, {:add_conn, key, conn})
|
||||
end
|
||||
|
||||
@spec remove_conn(atom(), String.t()) :: :ok
|
||||
def remove_conn(name, key) do
|
||||
GenServer.cast(name, {:remove_conn, key})
|
||||
end
|
||||
|
||||
@impl true
|
||||
def handle_cast({:add_conn, key, conn}, state) do
|
||||
state = put_in(state.conns[key], conn)
|
||||
|
||||
Process.monitor(conn.conn)
|
||||
{:noreply, state}
|
||||
end
|
||||
|
||||
@impl true
|
||||
def handle_cast({:checkout, conn_pid, pid}, state) do
|
||||
state =
|
||||
with true <- Process.alive?(conn_pid),
|
||||
{key, conn} <- find_conn(state.conns, conn_pid),
|
||||
used_by <- List.keydelete(conn.used_by, pid, 0) do
|
||||
conn_state = if used_by == [], do: :idle, else: conn.conn_state
|
||||
|
||||
put_in(state.conns[key], %{conn | conn_state: conn_state, used_by: used_by})
|
||||
else
|
||||
false ->
|
||||
Logger.debug("checkout for closed conn #{inspect(conn_pid)}")
|
||||
state
|
||||
|
||||
nil ->
|
||||
Logger.debug("checkout for alive conn #{inspect(conn_pid)}, but is not in state")
|
||||
state
|
||||
end
|
||||
|
||||
{:noreply, state}
|
||||
end
|
||||
|
||||
@impl true
|
||||
def handle_cast({:remove_conn, key}, state) do
|
||||
state = put_in(state.conns, Map.delete(state.conns, key))
|
||||
{:noreply, state}
|
||||
end
|
||||
|
||||
@impl true
|
||||
def handle_call({:checkin, uri}, from, state) do
|
||||
key = "#{uri.scheme}:#{uri.host}:#{uri.port}"
|
||||
|
||||
case state.conns[key] do
|
||||
%{conn: pid, gun_state: :up} = conn ->
|
||||
time = :os.system_time(:second)
|
||||
last_reference = time - conn.last_reference
|
||||
crf = crf(last_reference, 100, conn.crf)
|
||||
|
||||
state =
|
||||
put_in(state.conns[key], %{
|
||||
conn
|
||||
| last_reference: time,
|
||||
crf: crf,
|
||||
conn_state: :active,
|
||||
used_by: [from | conn.used_by]
|
||||
})
|
||||
|
||||
{:reply, pid, state}
|
||||
|
||||
%{gun_state: :down} ->
|
||||
{:reply, nil, state}
|
||||
|
||||
nil ->
|
||||
{:reply, nil, state}
|
||||
end
|
||||
end
|
||||
|
||||
@impl true
|
||||
def handle_call(:state, _from, state), do: {:reply, state, state}
|
||||
|
||||
@impl true
|
||||
def handle_call(:count, _from, state) do
|
||||
{:reply, Enum.count(state.conns), state}
|
||||
end
|
||||
|
||||
@impl true
|
||||
def handle_call(:unused_conns, _from, state) do
|
||||
unused_conns =
|
||||
state.conns
|
||||
|> Enum.filter(&filter_conns/1)
|
||||
|> Enum.sort(&sort_conns/2)
|
||||
|
||||
{:reply, unused_conns, state}
|
||||
end
|
||||
|
||||
defp filter_conns({_, %{conn_state: :idle, used_by: []}}), do: true
|
||||
defp filter_conns(_), do: false
|
||||
|
||||
defp sort_conns({_, c1}, {_, c2}) do
|
||||
c1.crf <= c2.crf and c1.last_reference <= c2.last_reference
|
||||
end
|
||||
|
||||
@impl true
|
||||
def handle_info({:gun_up, conn_pid, _protocol}, state) do
|
||||
%{origin_host: host, origin_scheme: scheme, origin_port: port} = Gun.info(conn_pid)
|
||||
|
||||
host =
|
||||
case :inet.ntoa(host) do
|
||||
{:error, :einval} -> host
|
||||
ip -> ip
|
||||
end
|
||||
|
||||
key = "#{scheme}:#{host}:#{port}"
|
||||
|
||||
state =
|
||||
with {key, conn} <- find_conn(state.conns, conn_pid, key),
|
||||
{true, key} <- {Process.alive?(conn_pid), key} do
|
||||
put_in(state.conns[key], %{
|
||||
conn
|
||||
| gun_state: :up,
|
||||
conn_state: :active,
|
||||
retries: 0
|
||||
})
|
||||
else
|
||||
{false, key} ->
|
||||
put_in(
|
||||
state.conns,
|
||||
Map.delete(state.conns, key)
|
||||
)
|
||||
|
||||
nil ->
|
||||
:ok = Gun.close(conn_pid)
|
||||
|
||||
state
|
||||
end
|
||||
|
||||
{:noreply, state}
|
||||
end
|
||||
|
||||
@impl true
|
||||
def handle_info({:gun_down, conn_pid, _protocol, _reason, _killed}, state) do
|
||||
retries = Config.get([:connections_pool, :retry], 1)
|
||||
# we can't get info on this pid, because pid is dead
|
||||
state =
|
||||
with {key, conn} <- find_conn(state.conns, conn_pid),
|
||||
{true, key} <- {Process.alive?(conn_pid), key} do
|
||||
if conn.retries == retries do
|
||||
:ok = Gun.close(conn.conn)
|
||||
|
||||
put_in(
|
||||
state.conns,
|
||||
Map.delete(state.conns, key)
|
||||
)
|
||||
else
|
||||
put_in(state.conns[key], %{
|
||||
conn
|
||||
| gun_state: :down,
|
||||
retries: conn.retries + 1
|
||||
})
|
||||
end
|
||||
else
|
||||
{false, key} ->
|
||||
put_in(
|
||||
state.conns,
|
||||
Map.delete(state.conns, key)
|
||||
)
|
||||
|
||||
nil ->
|
||||
Logger.debug(":gun_down for conn which isn't found in state")
|
||||
|
||||
state
|
||||
end
|
||||
|
||||
{:noreply, state}
|
||||
end
|
||||
|
||||
@impl true
|
||||
def handle_info({:DOWN, _ref, :process, conn_pid, reason}, state) do
|
||||
Logger.debug("received DOWM message for #{inspect(conn_pid)} reason -> #{inspect(reason)}")
|
||||
|
||||
state =
|
||||
with {key, conn} <- find_conn(state.conns, conn_pid) do
|
||||
Enum.each(conn.used_by, fn {pid, _ref} ->
|
||||
Process.exit(pid, reason)
|
||||
end)
|
||||
|
||||
put_in(
|
||||
state.conns,
|
||||
Map.delete(state.conns, key)
|
||||
)
|
||||
else
|
||||
nil ->
|
||||
Logger.debug(":DOWN for conn which isn't found in state")
|
||||
|
||||
state
|
||||
end
|
||||
|
||||
{:noreply, state}
|
||||
end
|
||||
|
||||
defp find_conn(conns, conn_pid) do
|
||||
Enum.find(conns, fn {_key, conn} ->
|
||||
conn.conn == conn_pid
|
||||
end)
|
||||
end
|
||||
|
||||
defp find_conn(conns, conn_pid, conn_key) do
|
||||
Enum.find(conns, fn {key, conn} ->
|
||||
key == conn_key and conn.conn == conn_pid
|
||||
end)
|
||||
end
|
||||
|
||||
def crf(current, steps, crf) do
|
||||
1 + :math.pow(0.5, current / steps) * crf
|
||||
end
|
||||
end
|
||||
22
lib/pleroma/pool/pool.ex
Normal file
22
lib/pleroma/pool/pool.ex
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
# Pleroma: A lightweight social networking server
|
||||
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.Pool do
|
||||
def child_spec(opts) do
|
||||
poolboy_opts =
|
||||
opts
|
||||
|> Keyword.put(:worker_module, Pleroma.Pool.Request)
|
||||
|> Keyword.put(:name, {:local, opts[:name]})
|
||||
|> Keyword.put(:size, opts[:size])
|
||||
|> Keyword.put(:max_overflow, opts[:max_overflow])
|
||||
|
||||
%{
|
||||
id: opts[:id] || {__MODULE__, make_ref()},
|
||||
start: {:poolboy, :start_link, [poolboy_opts, [name: opts[:name]]]},
|
||||
restart: :permanent,
|
||||
shutdown: 5000,
|
||||
type: :worker
|
||||
}
|
||||
end
|
||||
end
|
||||
65
lib/pleroma/pool/request.ex
Normal file
65
lib/pleroma/pool/request.ex
Normal file
|
|
@ -0,0 +1,65 @@
|
|||
# Pleroma: A lightweight social networking server
|
||||
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.Pool.Request do
|
||||
use GenServer
|
||||
|
||||
require Logger
|
||||
|
||||
def start_link(args) do
|
||||
GenServer.start_link(__MODULE__, args)
|
||||
end
|
||||
|
||||
@impl true
|
||||
def init(_), do: {:ok, []}
|
||||
|
||||
@spec execute(pid() | atom(), Tesla.Client.t(), keyword(), pos_integer()) ::
|
||||
{:ok, Tesla.Env.t()} | {:error, any()}
|
||||
def execute(pid, client, request, timeout) do
|
||||
GenServer.call(pid, {:execute, client, request}, timeout)
|
||||
end
|
||||
|
||||
@impl true
|
||||
def handle_call({:execute, client, request}, _from, state) do
|
||||
response = Pleroma.HTTP.request(client, request)
|
||||
|
||||
{:reply, response, state}
|
||||
end
|
||||
|
||||
@impl true
|
||||
def handle_info({:gun_data, _conn, _stream, _, _}, state) do
|
||||
{:noreply, state}
|
||||
end
|
||||
|
||||
@impl true
|
||||
def handle_info({:gun_up, _conn, _protocol}, state) do
|
||||
{:noreply, state}
|
||||
end
|
||||
|
||||
@impl true
|
||||
def handle_info({:gun_down, _conn, _protocol, _reason, _killed}, state) do
|
||||
{:noreply, state}
|
||||
end
|
||||
|
||||
@impl true
|
||||
def handle_info({:gun_error, _conn, _stream, _error}, state) do
|
||||
{:noreply, state}
|
||||
end
|
||||
|
||||
@impl true
|
||||
def handle_info({:gun_push, _conn, _stream, _new_stream, _method, _uri, _headers}, state) do
|
||||
{:noreply, state}
|
||||
end
|
||||
|
||||
@impl true
|
||||
def handle_info({:gun_response, _conn, _stream, _, _status, _headers}, state) do
|
||||
{:noreply, state}
|
||||
end
|
||||
|
||||
@impl true
|
||||
def handle_info(msg, state) do
|
||||
Logger.warn("Received unexpected message #{inspect(__MODULE__)} #{inspect(msg)}")
|
||||
{:noreply, state}
|
||||
end
|
||||
end
|
||||
42
lib/pleroma/pool/supervisor.ex
Normal file
42
lib/pleroma/pool/supervisor.ex
Normal file
|
|
@ -0,0 +1,42 @@
|
|||
# Pleroma: A lightweight social networking server
|
||||
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.Pool.Supervisor do
|
||||
use Supervisor
|
||||
|
||||
alias Pleroma.Config
|
||||
alias Pleroma.Pool
|
||||
|
||||
def start_link(args) do
|
||||
Supervisor.start_link(__MODULE__, args, name: __MODULE__)
|
||||
end
|
||||
|
||||
def init(_) do
|
||||
conns_child = %{
|
||||
id: Pool.Connections,
|
||||
start:
|
||||
{Pool.Connections, :start_link, [{:gun_connections, Config.get([:connections_pool])}]}
|
||||
}
|
||||
|
||||
Supervisor.init([conns_child | pools()], strategy: :one_for_one)
|
||||
end
|
||||
|
||||
defp pools do
|
||||
pools = Config.get(:pools)
|
||||
|
||||
pools =
|
||||
if Config.get([Pleroma.Upload, :proxy_remote]) == false do
|
||||
Keyword.delete(pools, :upload)
|
||||
else
|
||||
pools
|
||||
end
|
||||
|
||||
for {pool_name, pool_opts} <- pools do
|
||||
pool_opts
|
||||
|> Keyword.put(:id, {Pool, pool_name})
|
||||
|> Keyword.put(:name, pool_name)
|
||||
|> Pool.child_spec()
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
@ -3,19 +3,23 @@
|
|||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.ReverseProxy.Client do
|
||||
@callback request(atom(), String.t(), [tuple()], String.t(), list()) ::
|
||||
{:ok, pos_integer(), [tuple()], reference() | map()}
|
||||
| {:ok, pos_integer(), [tuple()]}
|
||||
@type status :: pos_integer()
|
||||
@type header_name :: String.t()
|
||||
@type header_value :: String.t()
|
||||
@type headers :: [{header_name(), header_value()}]
|
||||
|
||||
@callback request(atom(), String.t(), headers(), String.t(), list()) ::
|
||||
{:ok, status(), headers(), reference() | map()}
|
||||
| {:ok, status(), headers()}
|
||||
| {:ok, reference()}
|
||||
| {:error, term()}
|
||||
|
||||
@callback stream_body(reference() | pid() | map()) ::
|
||||
{:ok, binary()} | :done | {:error, String.t()}
|
||||
@callback stream_body(map()) :: {:ok, binary(), map()} | :done | {:error, atom() | String.t()}
|
||||
|
||||
@callback close(reference() | pid() | map()) :: :ok
|
||||
|
||||
def request(method, url, headers, "", opts \\ []) do
|
||||
client().request(method, url, headers, "", opts)
|
||||
def request(method, url, headers, body \\ "", opts \\ []) do
|
||||
client().request(method, url, headers, body, opts)
|
||||
end
|
||||
|
||||
def stream_body(ref), do: client().stream_body(ref)
|
||||
|
|
@ -23,6 +27,12 @@ defmodule Pleroma.ReverseProxy.Client do
|
|||
def close(ref), do: client().close(ref)
|
||||
|
||||
defp client do
|
||||
Pleroma.Config.get([Pleroma.ReverseProxy.Client], :hackney)
|
||||
:tesla
|
||||
|> Application.get_env(:adapter)
|
||||
|> client()
|
||||
end
|
||||
|
||||
defp client(Tesla.Adapter.Hackney), do: Pleroma.ReverseProxy.Client.Hackney
|
||||
defp client(Tesla.Adapter.Gun), do: Pleroma.ReverseProxy.Client.Tesla
|
||||
defp client(_), do: Pleroma.Config.get!(Pleroma.ReverseProxy.Client)
|
||||
end
|
||||
|
|
|
|||
24
lib/pleroma/reverse_proxy/client/hackney.ex
Normal file
24
lib/pleroma/reverse_proxy/client/hackney.ex
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
# Pleroma: A lightweight social networking server
|
||||
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.ReverseProxy.Client.Hackney do
|
||||
@behaviour Pleroma.ReverseProxy.Client
|
||||
|
||||
@impl true
|
||||
def request(method, url, headers, body, opts \\ []) do
|
||||
:hackney.request(method, url, headers, body, opts)
|
||||
end
|
||||
|
||||
@impl true
|
||||
def stream_body(ref) do
|
||||
case :hackney.stream_body(ref) do
|
||||
:done -> :done
|
||||
{:ok, data} -> {:ok, data, ref}
|
||||
{:error, error} -> {:error, error}
|
||||
end
|
||||
end
|
||||
|
||||
@impl true
|
||||
def close(ref), do: :hackney.close(ref)
|
||||
end
|
||||
90
lib/pleroma/reverse_proxy/client/tesla.ex
Normal file
90
lib/pleroma/reverse_proxy/client/tesla.ex
Normal file
|
|
@ -0,0 +1,90 @@
|
|||
# Pleroma: A lightweight social networking server
|
||||
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.ReverseProxy.Client.Tesla do
|
||||
@behaviour Pleroma.ReverseProxy.Client
|
||||
|
||||
@type headers() :: [{String.t(), String.t()}]
|
||||
@type status() :: pos_integer()
|
||||
|
||||
@spec request(atom(), String.t(), headers(), String.t(), keyword()) ::
|
||||
{:ok, status(), headers}
|
||||
| {:ok, status(), headers, map()}
|
||||
| {:error, atom() | String.t()}
|
||||
| no_return()
|
||||
|
||||
@impl true
|
||||
def request(method, url, headers, body, opts \\ []) do
|
||||
check_adapter()
|
||||
|
||||
opts = Keyword.put(opts, :body_as, :chunks)
|
||||
|
||||
with {:ok, response} <-
|
||||
Pleroma.HTTP.request(
|
||||
method,
|
||||
url,
|
||||
body,
|
||||
headers,
|
||||
Keyword.put(opts, :adapter, opts)
|
||||
) do
|
||||
if is_map(response.body) and method != :head do
|
||||
{:ok, response.status, response.headers, response.body}
|
||||
else
|
||||
{:ok, response.status, response.headers}
|
||||
end
|
||||
else
|
||||
{:error, error} -> {:error, error}
|
||||
end
|
||||
end
|
||||
|
||||
@impl true
|
||||
@spec stream_body(map()) ::
|
||||
{:ok, binary(), map()} | {:error, atom() | String.t()} | :done | no_return()
|
||||
def stream_body(%{pid: pid, opts: opts, fin: true}) do
|
||||
# if connection was reused, but in tesla were redirects,
|
||||
# tesla returns new opened connection, which must be closed manually
|
||||
if opts[:old_conn], do: Tesla.Adapter.Gun.close(pid)
|
||||
# if there were redirects we need to checkout old conn
|
||||
conn = opts[:old_conn] || opts[:conn]
|
||||
|
||||
if conn, do: :ok = Pleroma.Pool.Connections.checkout(conn, self(), :gun_connections)
|
||||
|
||||
:done
|
||||
end
|
||||
|
||||
def stream_body(client) do
|
||||
case read_chunk!(client) do
|
||||
{:fin, body} ->
|
||||
{:ok, body, Map.put(client, :fin, true)}
|
||||
|
||||
{:nofin, part} ->
|
||||
{:ok, part, client}
|
||||
|
||||
{:error, error} ->
|
||||
{:error, error}
|
||||
end
|
||||
end
|
||||
|
||||
defp read_chunk!(%{pid: pid, stream: stream, opts: opts}) do
|
||||
adapter = check_adapter()
|
||||
adapter.read_chunk(pid, stream, opts)
|
||||
end
|
||||
|
||||
@impl true
|
||||
@spec close(map) :: :ok | no_return()
|
||||
def close(%{pid: pid}) do
|
||||
adapter = check_adapter()
|
||||
adapter.close(pid)
|
||||
end
|
||||
|
||||
defp check_adapter do
|
||||
adapter = Application.get_env(:tesla, :adapter)
|
||||
|
||||
unless adapter == Tesla.Adapter.Gun do
|
||||
raise "#{adapter} doesn't support reading body in chunks"
|
||||
end
|
||||
|
||||
adapter
|
||||
end
|
||||
end
|
||||
|
|
@ -3,8 +3,6 @@
|
|||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.ReverseProxy do
|
||||
alias Pleroma.HTTP
|
||||
|
||||
@keep_req_headers ~w(accept user-agent accept-encoding cache-control if-modified-since) ++
|
||||
~w(if-unmodified-since if-none-match if-range range)
|
||||
@resp_cache_headers ~w(etag date last-modified)
|
||||
|
|
@ -58,10 +56,10 @@ defmodule Pleroma.ReverseProxy do
|
|||
|
||||
* `req_headers`, `resp_headers` additional headers.
|
||||
|
||||
* `http`: options for [hackney](https://github.com/benoitc/hackney).
|
||||
* `http`: options for [hackney](https://github.com/benoitc/hackney) or [gun](https://github.com/ninenines/gun).
|
||||
|
||||
"""
|
||||
@default_hackney_options [pool: :media]
|
||||
@default_options [pool: :media]
|
||||
|
||||
@inline_content_types [
|
||||
"image/gif",
|
||||
|
|
@ -94,11 +92,7 @@ defmodule Pleroma.ReverseProxy do
|
|||
def call(_conn, _url, _opts \\ [])
|
||||
|
||||
def call(conn = %{method: method}, url, opts) when method in @methods do
|
||||
hackney_opts =
|
||||
Pleroma.HTTP.Connection.hackney_options([])
|
||||
|> Keyword.merge(@default_hackney_options)
|
||||
|> Keyword.merge(Keyword.get(opts, :http, []))
|
||||
|> HTTP.process_request_options()
|
||||
client_opts = Keyword.merge(@default_options, Keyword.get(opts, :http, []))
|
||||
|
||||
req_headers = build_req_headers(conn.req_headers, opts)
|
||||
|
||||
|
|
@ -110,7 +104,7 @@ defmodule Pleroma.ReverseProxy do
|
|||
end
|
||||
|
||||
with {:ok, nil} <- Cachex.get(:failed_proxy_url_cache, url),
|
||||
{:ok, code, headers, client} <- request(method, url, req_headers, hackney_opts),
|
||||
{:ok, code, headers, client} <- request(method, url, req_headers, client_opts),
|
||||
:ok <-
|
||||
header_length_constraint(
|
||||
headers,
|
||||
|
|
@ -156,11 +150,11 @@ defmodule Pleroma.ReverseProxy do
|
|||
|> halt()
|
||||
end
|
||||
|
||||
defp request(method, url, headers, hackney_opts) do
|
||||
defp request(method, url, headers, opts) do
|
||||
Logger.debug("#{__MODULE__} #{method} #{url} #{inspect(headers)}")
|
||||
method = method |> String.downcase() |> String.to_existing_atom()
|
||||
|
||||
case client().request(method, url, headers, "", hackney_opts) do
|
||||
case client().request(method, url, headers, "", opts) do
|
||||
{:ok, code, headers, client} when code in @valid_resp_codes ->
|
||||
{:ok, code, downcase_headers(headers), client}
|
||||
|
||||
|
|
@ -210,7 +204,7 @@ defmodule Pleroma.ReverseProxy do
|
|||
duration,
|
||||
Keyword.get(opts, :max_read_duration, @max_read_duration)
|
||||
),
|
||||
{:ok, data} <- client().stream_body(client),
|
||||
{:ok, data, client} <- client().stream_body(client),
|
||||
{:ok, duration} <- increase_read_duration(duration),
|
||||
sent_so_far = sent_so_far + byte_size(data),
|
||||
:ok <-
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
# Pleroma: A lightweight social networking server
|
||||
# Copyright © 2019 Pleroma Authors <https://pleroma.social/>
|
||||
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.Web.ActivityPub.MRF.AntiFollowbotPolicy do
|
||||
|
|
|
|||
|
|
@ -12,17 +12,23 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do
|
|||
|
||||
require Logger
|
||||
|
||||
@hackney_options [
|
||||
pool: :media,
|
||||
recv_timeout: 10_000
|
||||
@options [
|
||||
pool: :media
|
||||
]
|
||||
|
||||
def perform(:prefetch, url) do
|
||||
Logger.debug("Prefetching #{inspect(url)}")
|
||||
|
||||
opts =
|
||||
if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do
|
||||
Keyword.put(@options, :recv_timeout, 10_000)
|
||||
else
|
||||
@options
|
||||
end
|
||||
|
||||
url
|
||||
|> MediaProxy.url()
|
||||
|> HTTP.get([], adapter: @hackney_options)
|
||||
|> HTTP.get([], adapter: opts)
|
||||
end
|
||||
|
||||
def perform(:preload, %{"object" => %{"attachment" => attachments}} = _message) do
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
# Pleroma: A lightweight social networking server
|
||||
# Copyright © 2019 Pleroma Authors <https://pleroma.social/>
|
||||
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.Web.ActivityPub.MRF.NoPlaceholderTextPolicy do
|
||||
|
|
|
|||
|
|
@ -3,11 +3,9 @@
|
|||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.Web.RelMe do
|
||||
@hackney_options [
|
||||
@options [
|
||||
pool: :media,
|
||||
recv_timeout: 2_000,
|
||||
max_body: 2_000_000,
|
||||
with_body: true
|
||||
max_body: 2_000_000
|
||||
]
|
||||
|
||||
if Pleroma.Config.get(:env) == :test do
|
||||
|
|
@ -25,8 +23,18 @@ defmodule Pleroma.Web.RelMe do
|
|||
def parse(_), do: {:error, "No URL provided"}
|
||||
|
||||
defp parse_url(url) do
|
||||
opts =
|
||||
if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do
|
||||
Keyword.merge(@options,
|
||||
recv_timeout: 2_000,
|
||||
with_body: true
|
||||
)
|
||||
else
|
||||
@options
|
||||
end
|
||||
|
||||
with {:ok, %Tesla.Env{body: html, status: status}} when status in 200..299 <-
|
||||
Pleroma.HTTP.get(url, [], adapter: @hackney_options),
|
||||
Pleroma.HTTP.get(url, [], adapter: opts),
|
||||
{:ok, html_tree} <- Floki.parse_document(html),
|
||||
data <-
|
||||
Floki.attribute(html_tree, "link[rel~=me]", "href") ++
|
||||
|
|
|
|||
|
|
@ -3,11 +3,9 @@
|
|||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.Web.RichMedia.Parser do
|
||||
@hackney_options [
|
||||
@options [
|
||||
pool: :media,
|
||||
recv_timeout: 2_000,
|
||||
max_body: 2_000_000,
|
||||
with_body: true
|
||||
max_body: 2_000_000
|
||||
]
|
||||
|
||||
defp parsers do
|
||||
|
|
@ -77,8 +75,18 @@ defmodule Pleroma.Web.RichMedia.Parser do
|
|||
end
|
||||
|
||||
defp parse_url(url) do
|
||||
opts =
|
||||
if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do
|
||||
Keyword.merge(@options,
|
||||
recv_timeout: 2_000,
|
||||
with_body: true
|
||||
)
|
||||
else
|
||||
@options
|
||||
end
|
||||
|
||||
try do
|
||||
{:ok, %Tesla.Env{body: html}} = Pleroma.HTTP.get(url, [], adapter: @hackney_options)
|
||||
{:ok, %Tesla.Env{body: html}} = Pleroma.HTTP.get(url, [], adapter: opts)
|
||||
|
||||
html
|
||||
|> parse_html()
|
||||
|
|
|
|||
|
|
@ -173,7 +173,8 @@ defmodule Pleroma.Web.WebFinger do
|
|||
get_template_from_xml(body)
|
||||
else
|
||||
_ ->
|
||||
with {:ok, %{body: body}} <- HTTP.get("https://#{domain}/.well-known/host-meta", []) do
|
||||
with {:ok, %{body: body, status: status}} when status in 200..299 <-
|
||||
HTTP.get("https://#{domain}/.well-known/host-meta", []) do
|
||||
get_template_from_xml(body)
|
||||
else
|
||||
e -> {:error, "Can't find LRDD template: #{inspect(e)}"}
|
||||
|
|
@ -205,7 +206,7 @@ defmodule Pleroma.Web.WebFinger do
|
|||
with response <-
|
||||
HTTP.get(
|
||||
address,
|
||||
Accept: "application/xrd+xml,application/jrd+json"
|
||||
[{"accept", "application/xrd+xml,application/jrd+json"}]
|
||||
),
|
||||
{:ok, %{status: status, body: body}} when status in 200..299 <- response do
|
||||
doc = XML.parse_document(body)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue