hj/pleroma
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge branch 'develop' of git.pleroma.social:pleroma/pleroma into unlisted

Browse source
This commit is contained in:
Mark Felder 2020-09-08 09:31:26 -05:00
commit 21efda2edb
34 changed files with 256 additions and 125 deletions
Download patch file Download diff file Expand all files Collapse all files

43
lib/pleroma/config/deprecation_warnings.ex
View file

@ -56,6 +56,7 @@ defmodule Pleroma.Config.DeprecationWarnings do
check_old_mrf_config()
check_media_proxy_whitelist_config()
check_welcome_message_config()
check_gun_pool_options()
end
def check_welcome_message_config do
@ -115,4 +116,46 @@ defmodule Pleroma.Config.DeprecationWarnings do
""")
end
end
def check_gun_pool_options do
pool_config = Config.get(:connections_pool)
if timeout = pool_config[:await_up_timeout] do
Logger.warn("""
!!!DEPRECATION WARNING!!!
Your config is using old setting name `await_up_timeout` instead of `connect_timeout`. Setting should work for now, but you are advised to change format to scheme with port to prevent possible issues later.
""")
Config.put(:connections_pool, Keyword.put_new(pool_config, :connect_timeout, timeout))
end
pools_configs = Config.get(:pools)
warning_preface = """
!!!DEPRECATION WARNING!!!
Your config is using old setting name `timeout` instead of `recv_timeout` in pool settings. Setting should work for now, but you are advised to change format to scheme with port to prevent possible issues later.
"""
updated_config =
Enum.reduce(pools_configs, [], fn {pool_name, config}, acc ->
if timeout = config[:timeout] do
Keyword.put(acc, pool_name, Keyword.put_new(config, :recv_timeout, timeout))
else
acc
end
end)
if updated_config != [] do
pool_warnings =
updated_config
|> Keyword.keys()
|> Enum.map(fn pool_name ->
"\n* `:timeout` options in #{pool_name} pool is now `:recv_timeout`"
end)
Logger.warn(Enum.join([warning_preface | pool_warnings]))
Config.put(:pools, updated_config)
end
end
end

8
lib/pleroma/gun/conn.ex
View file

@ -13,7 +13,7 @@ defmodule Pleroma.Gun.Conn do
opts =
opts
|> Enum.into(%{})
|> Map.put_new(:await_up_timeout, pool_opts[:await_up_timeout] || 5_000)
|> Map.put_new(:connect_timeout, pool_opts[:connect_timeout] || 5_000)
|> Map.put_new(:supervise, false)
|> maybe_add_tls_opts(uri)
@ -50,7 +50,7 @@ defmodule Pleroma.Gun.Conn do
with open_opts <- Map.delete(opts, :tls_opts),
{:ok, conn} <- Gun.open(proxy_host, proxy_port, open_opts),
{:ok, _} <- Gun.await_up(conn, opts[:await_up_timeout]),
{:ok, _} <- Gun.await_up(conn, opts[:connect_timeout]),
stream <- Gun.connect(conn, connect_opts),
{:response, :fin, 200, _} <- Gun.await(conn, stream) do
{:ok, conn}
@ -88,7 +88,7 @@ defmodule Pleroma.Gun.Conn do
|> Map.put(:socks_opts, socks_opts)
with {:ok, conn} <- Gun.open(proxy_host, proxy_port, opts),
{:ok, _} <- Gun.await_up(conn, opts[:await_up_timeout]) do
{:ok, _} <- Gun.await_up(conn, opts[:connect_timeout]) do
{:ok, conn}
else
error ->
@ -106,7 +106,7 @@ defmodule Pleroma.Gun.Conn do
host = Pleroma.HTTP.AdapterHelper.parse_host(host)
with {:ok, conn} <- Gun.open(host, port, opts),
{:ok, _} <- Gun.await_up(conn, opts[:await_up_timeout]) do
{:ok, _} <- Gun.await_up(conn, opts[:connect_timeout]) do
{:ok, conn}
else
error ->

2
lib/pleroma/http/adapter_helper.ex
View file

@ -6,7 +6,7 @@ defmodule Pleroma.HTTP.AdapterHelper do
@moduledoc """
Configure Tesla.Client with default and customized adapter options.
"""
@defaults [pool: :federation]
@defaults [pool: :federation, connect_timeout: 5_000, recv_timeout: 5_000]
@type proxy_type() :: :socks4 | :socks5
@type host() :: charlist() | :inet.ip_address()

14
lib/pleroma/http/adapter_helper/gun.ex
View file

@ -11,12 +11,8 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do
require Logger
@defaults [
connect_timeout: 5_000,
domain_lookup_timeout: 5_000,
tls_handshake_timeout: 5_000,
retry: 1,
retry_timeout: 1000,
await_up_timeout: 5_000
retry_timeout: 1_000
]
@type pool() :: :federation | :upload | :media | :default
@ -45,15 +41,17 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do
end
defp put_timeout(opts) do
{recv_timeout, opts} = Keyword.pop(opts, :recv_timeout, pool_timeout(opts[:pool]))
# this is the timeout to receive a message from Gun
Keyword.put_new(opts, :timeout, pool_timeout(opts[:pool]))
# `:timeout` key is used in Tesla
Keyword.put(opts, :timeout, recv_timeout)
end
@spec pool_timeout(pool()) :: non_neg_integer()
def pool_timeout(pool) do
default = Config.get([:pools, :default, :timeout], 5_000)
default = Config.get([:pools, :default, :recv_timeout], 5_000)
Config.get([:pools, pool, :timeout], default)
Config.get([:pools, pool, :recv_timeout], default)
end
@prefix Pleroma.Gun.ConnectionPool

14
lib/pleroma/http/adapter_helper/hackney.ex
View file

@ -2,11 +2,8 @@ defmodule Pleroma.HTTP.AdapterHelper.Hackney do
@behaviour Pleroma.HTTP.AdapterHelper
@defaults [
connect_timeout: 10_000,
recv_timeout: 20_000,
follow_redirect: true,
force_redirect: true,
pool: :federation
force_redirect: true
]
@spec options(keyword(), URI.t()) :: keyword()
@ -19,6 +16,7 @@ defmodule Pleroma.HTTP.AdapterHelper.Hackney do
|> Keyword.merge(config_opts)
|> Keyword.merge(connection_opts)
|> add_scheme_opts(uri)
|> maybe_add_with_body()
|> Pleroma.HTTP.AdapterHelper.maybe_add_proxy(proxy)
end
@ -27,4 +25,12 @@ defmodule Pleroma.HTTP.AdapterHelper.Hackney do
end
defp add_scheme_opts(opts, _), do: opts
defp maybe_add_with_body(opts) do
if opts[:max_body] do
Keyword.put(opts, :with_body, true)
else
opts
end
end
end

2
lib/pleroma/http/ex_aws.ex
View file

@ -11,7 +11,7 @@ defmodule Pleroma.HTTP.ExAws do
@impl true
def request(method, url, body \\ "", headers \\ [], http_opts \\ []) do
http_opts = Keyword.put_new(http_opts, :adapter, pool: :upload)
http_opts = Keyword.put_new(http_opts, :pool, :upload)
case HTTP.request(method, url, body, headers, http_opts) do
{:ok, env} ->

2
lib/pleroma/http/http.ex
View file

@ -60,7 +60,7 @@ defmodule Pleroma.HTTP do
{:ok, Env.t()} | {:error, any()}
def request(method, url, body, headers, options) when is_binary(url) do
uri = URI.parse(url)
adapter_opts = AdapterHelper.options(uri, options[:adapter] || [])
adapter_opts = AdapterHelper.options(uri, options || [])
options = put_in(options[:adapter], adapter_opts)
params = options[:params] || []

4
lib/pleroma/http/tzdata.ex
View file

@ -11,7 +11,7 @@ defmodule Pleroma.HTTP.Tzdata do
@impl true
def get(url, headers, options) do
options = Keyword.put_new(options, :adapter, pool: :default)
options = Keyword.put_new(options, :pool, :default)
with {:ok, %Tesla.Env{} = env} <- HTTP.get(url, headers, options) do
{:ok, {env.status, env.headers, env.body}}
@ -20,7 +20,7 @@ defmodule Pleroma.HTTP.Tzdata do
@impl true
def head(url, headers, options) do
options = Keyword.put_new(options, :adapter, pool: :default)
options = Keyword.put_new(options, :pool, :default)
with {:ok, %Tesla.Env{} = env} <- HTTP.head(url, headers, options) do
{:ok, {env.status, env.headers}}

13
lib/pleroma/upload/filter.ex
View file

@ -15,7 +15,11 @@ defmodule Pleroma.Upload.Filter do
require Logger
@callback filter(Pleroma.Upload.t()) :: :ok | {:ok, Pleroma.Upload.t()} | {:error, any()}
@callback filter(Pleroma.Upload.t()) ::
{:ok, :filtered}
| {:ok, :noop}
| {:ok, :filtered, Pleroma.Upload.t()}
| {:error, any()}
@spec filter([module()], Pleroma.Upload.t()) :: {:ok, Pleroma.Upload.t()} | {:error, any()}
@ -25,10 +29,13 @@ defmodule Pleroma.Upload.Filter do
def filter([filter | rest], upload) do
case filter.filter(upload) do
:ok ->
{:ok, :filtered} ->
filter(rest, upload)
{:ok, upload} ->
{:ok, :filtered, upload} ->
filter(rest, upload)
{:ok, :noop} ->
filter(rest, upload)
error ->

4
lib/pleroma/upload/filter/anonymize_filename.ex
View file

@ -16,9 +16,11 @@ defmodule Pleroma.Upload.Filter.AnonymizeFilename do
def filter(%Upload{name: name} = upload) do
extension = List.last(String.split(name, "."))
name = predefined_name(extension) || random(extension)
{:ok, %Upload{upload | name: name}}
{:ok, :filtered, %Upload{upload | name: name}}
end
def filter(_), do: {:ok, :noop}
@spec predefined_name(String.t()) :: String.t() | nil
defp predefined_name(extension) do
with name when not is_nil(name) <- Config.get([__MODULE__, :text]),

4
lib/pleroma/upload/filter/dedupe.ex
View file

@ -17,8 +17,8 @@ defmodule Pleroma.Upload.Filter.Dedupe do
|> Base.encode16(case: :lower)
filename = shasum <> "." <> extension
{:ok, %Upload{upload | id: shasum, path: filename}}
{:ok, :filtered, %Upload{upload | id: shasum, path: filename}}
end
def filter(_), do: :ok
def filter(_), do: {:ok, :noop}
end

10
lib/pleroma/upload/filter/exiftool.ex
View file

@ -9,11 +9,15 @@ defmodule Pleroma.Upload.Filter.Exiftool do
"""
@behaviour Pleroma.Upload.Filter
@spec filter(Pleroma.Upload.t()) :: :ok | {:error, String.t()}
@spec filter(Pleroma.Upload.t()) :: {:ok, any()} | {:error, String.t()}
# webp is not compatible with exiftool at this time
def filter(%Pleroma.Upload{content_type: "image/webp"}), do: {:ok, :noop}
def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do
try do
case System.cmd("exiftool", ["-overwrite_original", "-gps:all=", file], parallelism: true) do
{_response, 0} -> :ok
{_response, 0} -> {:ok, :filtered}
{error, 1} -> {:error, error}
end
rescue
@ -22,5 +26,5 @@ defmodule Pleroma.Upload.Filter.Exiftool do
end
end
def filter(_), do: :ok
def filter(_), do: {:ok, :noop}
end

6
lib/pleroma/upload/filter/mogrifun.ex
View file

@ -38,16 +38,16 @@ defmodule Pleroma.Upload.Filter.Mogrifun do
[{"fill", "yellow"}, {"tint", "40"}]
]
@spec filter(Pleroma.Upload.t()) :: :ok | {:error, String.t()}
@spec filter(Pleroma.Upload.t()) :: {:ok, atom()} | {:error, String.t()}
def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do
try do
Filter.Mogrify.do_filter(file, [Enum.random(@filters)])
:ok
{:ok, :filtered}
rescue
_e in ErlangError ->
{:error, "mogrify command not found"}
end
end
def filter(_), do: :ok
def filter(_), do: {:ok, :noop}
end

6
lib/pleroma/upload/filter/mogrify.ex
View file

@ -8,18 +8,18 @@ defmodule Pleroma.Upload.Filter.Mogrify do
@type conversion :: action :: String.t() | {action :: String.t(), opts :: String.t()}
@type conversions :: conversion() | [conversion()]
@spec filter(Pleroma.Upload.t()) :: :ok | {:error, String.t()}
@spec filter(Pleroma.Upload.t()) :: {:ok, :atom} | {:error, String.t()}
def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do
try do
do_filter(file, Pleroma.Config.get!([__MODULE__, :args]))
:ok
{:ok, :filtered}
rescue
_e in ErlangError ->
{:error, "mogrify command not found"}
end
end
def filter(_), do: :ok
def filter(_), do: {:ok, :noop}
def do_filter(file, filters) do
file

12
lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex
View file

@ -13,22 +13,16 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do
require Logger
@options [
pool: :media
pool: :media,
recv_timeout: 10_000
]
def perform(:prefetch, url) do
Logger.debug("Prefetching #{inspect(url)}")
opts =
if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do
Keyword.put(@options, :recv_timeout, 10_000)
else
@options
end
url
|> MediaProxy.url()
|> HTTP.get([], adapter: opts)
|> HTTP.get([], @options)
end
def perform(:preload, %{"object" => %{"attachment" => attachments}} = _message) do

15
lib/pleroma/web/rel_me.ex
View file

@ -5,7 +5,8 @@
defmodule Pleroma.Web.RelMe do
@options [
pool: :media,
max_body: 2_000_000
max_body: 2_000_000,
recv_timeout: 2_000
]
if Pleroma.Config.get(:env) == :test do
@ -23,18 +24,8 @@ defmodule Pleroma.Web.RelMe do
def parse(_), do: {:error, "No URL provided"}
defp parse_url(url) do
opts =
if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do
Keyword.merge(@options,
recv_timeout: 2_000,
with_body: true
)
else
@options
end
with {:ok, %Tesla.Env{body: html, status: status}} when status in 200..299 <-
Pleroma.HTTP.get(url, [], adapter: opts),
Pleroma.HTTP.get(url, [], @options),
{:ok, html_tree} <- Floki.parse_document(html),
data <-
Floki.attribute(html_tree, "link[rel~=me]", "href") ++

19
lib/pleroma/web/rich_media/helpers.ex
View file

@ -9,14 +9,15 @@ defmodule Pleroma.Web.RichMedia.Helpers do
alias Pleroma.Object
alias Pleroma.Web.RichMedia.Parser
@rich_media_options [
@options [
pool: :media,
max_body: 2_000_000
max_body: 2_000_000,
recv_timeout: 2_000
]
@spec validate_page_url(URI.t() | binary()) :: :ok | :error
defp validate_page_url(page_url) when is_binary(page_url) do
validate_tld = Pleroma.Config.get([Pleroma.Formatter, :validate_tld])
validate_tld = Config.get([Pleroma.Formatter, :validate_tld])
page_url
|> Linkify.Parser.url?(validate_tld: validate_tld)
@ -86,16 +87,6 @@ defmodule Pleroma.Web.RichMedia.Helpers do
def rich_media_get(url) do
headers = [{"user-agent", Pleroma.Application.user_agent() <> "; Bot"}]
options =
if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do
Keyword.merge(@rich_media_options,
recv_timeout: 2_000,
with_body: true
)
else
@rich_media_options
end
Pleroma.HTTP.get(url, headers, adapter: options)
Pleroma.HTTP.get(url, headers, @options)
end
end