Merge branch 'develop' into feature/expire-mutes

lib/mix/pleroma.ex

@@ -18,6 +18,7 @@ defmodule Mix.Pleroma do
   @doc "Common functions to be reused in mix tasks"
   def start_pleroma do
     Pleroma.Config.Holder.save_default()
+    Pleroma.Config.Oban.warn()
     Application.put_env(:phoenix, :serve_endpoints, false, persistent: true)
 
     if Pleroma.Config.get(:env) != :test do

lib/mix/tasks/pleroma/benchmark.ex

@@ -91,20 +91,17 @@ defmodule Mix.Tasks.Pleroma.Benchmark do
         "Without conn and without pool" => fn ->
           {:ok, %Tesla.Env{}} =
             Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [],
-              adapter: [pool: :no_pool, receive_conn: false]
+              pool: :no_pool,
+              receive_conn: false
             )
         end,
         "Without conn and with pool" => fn ->
           {:ok, %Tesla.Env{}} =
-            Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [],
-              adapter: [receive_conn: false]
-            )
+            Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [], receive_conn: false)
         end,
         "With reused conn and without pool" => fn ->
           {:ok, %Tesla.Env{}} =
-            Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [],
-              adapter: [pool: :no_pool]
-            )
+            Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [], pool: :no_pool)
         end,
         "With reused conn and with pool" => fn ->
           {:ok, %Tesla.Env{}} = Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500")

lib/mix/tasks/pleroma/frontend.ex

@@ -124,9 +124,7 @@ defmodule Mix.Tasks.Pleroma.Frontend do
     url = String.replace(frontend_info["build_url"], "${ref}", frontend_info["ref"])
 
     with {:ok, %{status: 200, body: zip_body}} <-
-           Pleroma.HTTP.get(url, [],
-             adapter: [pool: :media, timeout: 120_000, recv_timeout: 120_000]
-           ) do
+           Pleroma.HTTP.get(url, [], pool: :media, recv_timeout: 120_000) do
       unzip(zip_body, dest)
     else
       e -> {:error, e}

lib/pleroma/application.ex

@@ -50,6 +50,7 @@ defmodule Pleroma.Application do
     Pleroma.Telemetry.Logger.attach()
     Config.Holder.save_default()
     Pleroma.HTML.compile_scrubbers()
+    Pleroma.Config.Oban.warn()
     Config.DeprecationWarnings.warn()
     Pleroma.Plugs.HTTPSecurityPlug.warn_if_disabled()
     Pleroma.ApplicationRequirements.verify!()

lib/pleroma/config/deprecation_warnings.ex

@@ -56,6 +56,7 @@ defmodule Pleroma.Config.DeprecationWarnings do
     check_old_mrf_config()
     check_media_proxy_whitelist_config()
     check_welcome_message_config()
+    check_gun_pool_options()
   end
 
   def check_welcome_message_config do
@@ -115,4 +116,46 @@ defmodule Pleroma.Config.DeprecationWarnings do
       """)
     end
   end
+
+  def check_gun_pool_options do
+    pool_config = Config.get(:connections_pool)
+
+    if timeout = pool_config[:await_up_timeout] do
+      Logger.warn("""
+      !!!DEPRECATION WARNING!!!
+      Your config is using old setting name `await_up_timeout` instead of `connect_timeout`. Setting should work for now, but you are advised to change format to scheme with port to prevent possible issues later.
+      """)
+
+      Config.put(:connections_pool, Keyword.put_new(pool_config, :connect_timeout, timeout))
+    end
+
+    pools_configs = Config.get(:pools)
+
+    warning_preface = """
+    !!!DEPRECATION WARNING!!!
+    Your config is using old setting name `timeout` instead of `recv_timeout` in pool settings. Setting should work for now, but you are advised to change format to scheme with port to prevent possible issues later.
+    """
+
+    updated_config =
+      Enum.reduce(pools_configs, [], fn {pool_name, config}, acc ->
+        if timeout = config[:timeout] do
+          Keyword.put(acc, pool_name, Keyword.put_new(config, :recv_timeout, timeout))
+        else
+          acc
+        end
+      end)
+
+    if updated_config != [] do
+      pool_warnings =
+        updated_config
+        |> Keyword.keys()
+        |> Enum.map(fn pool_name ->
+          "\n* `:timeout` options in #{pool_name} pool is now `:recv_timeout`"
+        end)
+
+      Logger.warn(Enum.join([warning_preface | pool_warnings]))
+
+      Config.put(:pools, updated_config)
+    end
+  end
 end

lib/pleroma/config/oban.ex

@@ -0,0 +1,30 @@
+defmodule Pleroma.Config.Oban do
+  require Logger
+
+  def warn do
+    oban_config = Pleroma.Config.get(Oban)
+
+    crontab =
+      [Pleroma.Workers.Cron.StatsWorker]
+      |> Enum.reduce(oban_config[:crontab], fn removed_worker, acc ->
+        with acc when is_list(acc) <- acc,
+             setting when is_tuple(setting) <-
+               Enum.find(acc, fn {_, worker} -> worker == removed_worker end) do
+          """
+          !!!OBAN CONFIG WARNING!!!
+          You are using old workers in Oban crontab settings, which were removed.
+          Please, remove setting from crontab in your config file (prod.secret.exs): #{
+            inspect(setting)
+          }
+          """
+          |> Logger.warn()
+
+          List.delete(acc, setting)
+        else
+          _ -> acc
+        end
+      end)
+
+    Pleroma.Config.put(Oban, Keyword.put(oban_config, :crontab, crontab))
+  end
+end

lib/pleroma/gun/conn.ex

@@ -13,7 +13,7 @@ defmodule Pleroma.Gun.Conn do
     opts =
       opts
       |> Enum.into(%{})
-      |> Map.put_new(:await_up_timeout, pool_opts[:await_up_timeout] || 5_000)
+      |> Map.put_new(:connect_timeout, pool_opts[:connect_timeout] || 5_000)
       |> Map.put_new(:supervise, false)
       |> maybe_add_tls_opts(uri)
 
@@ -50,7 +50,7 @@ defmodule Pleroma.Gun.Conn do
 
     with open_opts <- Map.delete(opts, :tls_opts),
          {:ok, conn} <- Gun.open(proxy_host, proxy_port, open_opts),
-         {:ok, _} <- Gun.await_up(conn, opts[:await_up_timeout]),
+         {:ok, _} <- Gun.await_up(conn, opts[:connect_timeout]),
          stream <- Gun.connect(conn, connect_opts),
          {:response, :fin, 200, _} <- Gun.await(conn, stream) do
       {:ok, conn}
@@ -88,7 +88,7 @@ defmodule Pleroma.Gun.Conn do
       |> Map.put(:socks_opts, socks_opts)
 
     with {:ok, conn} <- Gun.open(proxy_host, proxy_port, opts),
-         {:ok, _} <- Gun.await_up(conn, opts[:await_up_timeout]) do
+         {:ok, _} <- Gun.await_up(conn, opts[:connect_timeout]) do
       {:ok, conn}
     else
       error ->
@@ -106,7 +106,7 @@ defmodule Pleroma.Gun.Conn do
     host = Pleroma.HTTP.AdapterHelper.parse_host(host)
 
     with {:ok, conn} <- Gun.open(host, port, opts),
-         {:ok, _} <- Gun.await_up(conn, opts[:await_up_timeout]) do
+         {:ok, _} <- Gun.await_up(conn, opts[:connect_timeout]) do
       {:ok, conn}
     else
       error ->

lib/pleroma/http/adapter_helper.ex

@@ -6,7 +6,7 @@ defmodule Pleroma.HTTP.AdapterHelper do
   @moduledoc """
   Configure Tesla.Client with default and customized adapter options.
   """
-  @defaults [pool: :federation]
+  @defaults [pool: :federation, connect_timeout: 5_000, recv_timeout: 5_000]
 
   @type proxy_type() :: :socks4 | :socks5
   @type host() :: charlist() | :inet.ip_address()

lib/pleroma/http/adapter_helper/gun.ex

@@ -11,12 +11,8 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do
   require Logger
 
   @defaults [
-    connect_timeout: 5_000,
-    domain_lookup_timeout: 5_000,
-    tls_handshake_timeout: 5_000,
     retry: 1,
-    retry_timeout: 1000,
-    await_up_timeout: 5_000
+    retry_timeout: 1_000
   ]
 
   @type pool() :: :federation | :upload | :media | :default
@@ -45,15 +41,17 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do
   end
 
   defp put_timeout(opts) do
+    {recv_timeout, opts} = Keyword.pop(opts, :recv_timeout, pool_timeout(opts[:pool]))
     # this is the timeout to receive a message from Gun
-    Keyword.put_new(opts, :timeout, pool_timeout(opts[:pool]))
+    # `:timeout` key is used in Tesla
+    Keyword.put(opts, :timeout, recv_timeout)
   end
 
   @spec pool_timeout(pool()) :: non_neg_integer()
   def pool_timeout(pool) do
-    default = Config.get([:pools, :default, :timeout], 5_000)
+    default = Config.get([:pools, :default, :recv_timeout], 5_000)
 
-    Config.get([:pools, pool, :timeout], default)
+    Config.get([:pools, pool, :recv_timeout], default)
   end
 
   @prefix Pleroma.Gun.ConnectionPool

lib/pleroma/http/adapter_helper/hackney.ex

@@ -2,11 +2,8 @@ defmodule Pleroma.HTTP.AdapterHelper.Hackney do
   @behaviour Pleroma.HTTP.AdapterHelper
 
   @defaults [
-    connect_timeout: 10_000,
-    recv_timeout: 20_000,
     follow_redirect: true,
-    force_redirect: true,
-    pool: :federation
+    force_redirect: true
   ]
 
   @spec options(keyword(), URI.t()) :: keyword()
@@ -19,8 +16,21 @@ defmodule Pleroma.HTTP.AdapterHelper.Hackney do
     |> Keyword.merge(config_opts)
     |> Keyword.merge(connection_opts)
     |> add_scheme_opts(uri)
+    |> maybe_add_with_body()
     |> Pleroma.HTTP.AdapterHelper.maybe_add_proxy(proxy)
   end
 
+  defp add_scheme_opts(opts, %URI{scheme: "https"}) do
+    Keyword.put(opts, :ssl_options, versions: [:"tlsv1.2", :"tlsv1.1", :tlsv1])
+  end
+
   defp add_scheme_opts(opts, _), do: opts
+
+  defp maybe_add_with_body(opts) do
+    if opts[:max_body] do
+      Keyword.put(opts, :with_body, true)
+    else
+      opts
+    end
+  end
 end

lib/pleroma/http/ex_aws.ex

@@ -11,7 +11,7 @@ defmodule Pleroma.HTTP.ExAws do
 
   @impl true
   def request(method, url, body \\ "", headers \\ [], http_opts \\ []) do
-    http_opts = Keyword.put_new(http_opts, :adapter, pool: :upload)
+    http_opts = Keyword.put_new(http_opts, :pool, :upload)
 
     case HTTP.request(method, url, body, headers, http_opts) do
       {:ok, env} ->

lib/pleroma/http/http.ex

@@ -60,7 +60,7 @@ defmodule Pleroma.HTTP do
           {:ok, Env.t()} | {:error, any()}
   def request(method, url, body, headers, options) when is_binary(url) do
     uri = URI.parse(url)
-    adapter_opts = AdapterHelper.options(uri, options[:adapter] || [])
+    adapter_opts = AdapterHelper.options(uri, options || [])
 
     options = put_in(options[:adapter], adapter_opts)
     params = options[:params] || []

lib/pleroma/http/tzdata.ex

@@ -11,7 +11,7 @@ defmodule Pleroma.HTTP.Tzdata do
 
   @impl true
   def get(url, headers, options) do
-    options = Keyword.put_new(options, :adapter, pool: :default)
+    options = Keyword.put_new(options, :pool, :default)
 
     with {:ok, %Tesla.Env{} = env} <- HTTP.get(url, headers, options) do
       {:ok, {env.status, env.headers, env.body}}
@@ -20,7 +20,7 @@ defmodule Pleroma.HTTP.Tzdata do
 
   @impl true
   def head(url, headers, options) do
-    options = Keyword.put_new(options, :adapter, pool: :default)
+    options = Keyword.put_new(options, :pool, :default)
 
     with {:ok, %Tesla.Env{} = env} <- HTTP.head(url, headers, options) do
       {:ok, {env.status, env.headers}}

lib/pleroma/reverse_proxy/client/hackney.ex

@@ -7,6 +7,7 @@ defmodule Pleroma.ReverseProxy.Client.Hackney do
 
   @impl true
   def request(method, url, headers, body, opts \\ []) do
+    opts = Keyword.put(opts, :ssl_options, versions: [:"tlsv1.2", :"tlsv1.1", :tlsv1])
     :hackney.request(method, url, headers, body, opts)
   end
 

lib/pleroma/stats.ex

@@ -3,12 +3,15 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Stats do
+  use GenServer
+
   import Ecto.Query
+
   alias Pleroma.CounterCache
   alias Pleroma.Repo
   alias Pleroma.User
 
-  use GenServer
+  @interval :timer.seconds(60)
 
   def start_link(_) do
     GenServer.start_link(
@@ -18,6 +21,11 @@ defmodule Pleroma.Stats do
     )
   end
 
+  @impl true
+  def init(_args) do
+    {:ok, nil, {:continue, :calculate_stats}}
+  end
+
   @doc "Performs update stats"
   def force_update do
     GenServer.call(__MODULE__, :force_update)
@@ -29,7 +37,11 @@ defmodule Pleroma.Stats do
   end
 
   @doc "Returns stats data"
-  @spec get_stats() :: %{domain_count: integer(), status_count: integer(), user_count: integer()}
+  @spec get_stats() :: %{
+          domain_count: non_neg_integer(),
+          status_count: non_neg_integer(),
+          user_count: non_neg_integer()
+        }
   def get_stats do
     %{stats: stats} = GenServer.call(__MODULE__, :get_state)
 
@@ -44,25 +56,14 @@ defmodule Pleroma.Stats do
     peers
   end
 
-  def init(_args) do
-    {:ok, calculate_stat_data()}
-  end
-
-  def handle_call(:force_update, _from, _state) do
-    new_stats = calculate_stat_data()
-    {:reply, new_stats, new_stats}
-  end
-
-  def handle_call(:get_state, _from, state) do
-    {:reply, state, state}
-  end
-
-  def handle_cast(:run_update, _state) do
-    new_stats = calculate_stat_data()
-
-    {:noreply, new_stats}
-  end
-
+  @spec calculate_stat_data() :: %{
+          peers: list(),
+          stats: %{
+            domain_count: non_neg_integer(),
+            status_count: non_neg_integer(),
+            user_count: non_neg_integer()
+          }
+        }
   def calculate_stat_data do
     peers =
       from(
@@ -97,6 +98,7 @@ defmodule Pleroma.Stats do
     }
   end
 
+  @spec get_status_visibility_count(String.t() | nil) :: map()
   def get_status_visibility_count(instance \\ nil) do
     if is_nil(instance) do
       CounterCache.get_sum()
@@ -104,4 +106,36 @@ defmodule Pleroma.Stats do
       CounterCache.get_by_instance(instance)
     end
   end
+
+  @impl true
+  def handle_continue(:calculate_stats, _) do
+    stats = calculate_stat_data()
+    Process.send_after(self(), :run_update, @interval)
+    {:noreply, stats}
+  end
+
+  @impl true
+  def handle_call(:force_update, _from, _state) do
+    new_stats = calculate_stat_data()
+    {:reply, new_stats, new_stats}
+  end
+
+  @impl true
+  def handle_call(:get_state, _from, state) do
+    {:reply, state, state}
+  end
+
+  @impl true
+  def handle_cast(:run_update, _state) do
+    new_stats = calculate_stat_data()
+
+    {:noreply, new_stats}
+  end
+
+  @impl true
+  def handle_info(:run_update, _) do
+    new_stats = calculate_stat_data()
+    Process.send_after(self(), :run_update, @interval)
+    {:noreply, new_stats}
+  end
 end

lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex

@@ -13,22 +13,16 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do
   require Logger
 
   @options [
-    pool: :media
+    pool: :media,
+    recv_timeout: 10_000
   ]
 
   def perform(:prefetch, url) do
     Logger.debug("Prefetching #{inspect(url)}")
 
-    opts =
-      if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do
-        Keyword.put(@options, :recv_timeout, 10_000)
-      else
-        @options
-      end
-
     url
     |> MediaProxy.url()
-    |> HTTP.get([], adapter: opts)
+    |> HTTP.get([], @options)
   end
 
   def perform(:preload, %{"object" => %{"attachment" => attachments}} = _message) do

lib/pleroma/web/rel_me.ex

@@ -5,7 +5,8 @@
 defmodule Pleroma.Web.RelMe do
   @options [
     pool: :media,
-    max_body: 2_000_000
+    max_body: 2_000_000,
+    recv_timeout: 2_000
   ]
 
   if Pleroma.Config.get(:env) == :test do
@@ -23,18 +24,8 @@ defmodule Pleroma.Web.RelMe do
   def parse(_), do: {:error, "No URL provided"}
 
   defp parse_url(url) do
-    opts =
-      if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do
-        Keyword.merge(@options,
-          recv_timeout: 2_000,
-          with_body: true
-        )
-      else
-        @options
-      end
-
     with {:ok, %Tesla.Env{body: html, status: status}} when status in 200..299 <-
-           Pleroma.HTTP.get(url, [], adapter: opts),
+           Pleroma.HTTP.get(url, [], @options),
          {:ok, html_tree} <- Floki.parse_document(html),
          data <-
            Floki.attribute(html_tree, "link[rel~=me]", "href") ++

lib/pleroma/web/rich_media/helpers.ex

@@ -9,14 +9,15 @@ defmodule Pleroma.Web.RichMedia.Helpers do
   alias Pleroma.Object
   alias Pleroma.Web.RichMedia.Parser
 
-  @rich_media_options [
+  @options [
     pool: :media,
-    max_body: 2_000_000
+    max_body: 2_000_000,
+    recv_timeout: 2_000
   ]
 
   @spec validate_page_url(URI.t() | binary()) :: :ok | :error
   defp validate_page_url(page_url) when is_binary(page_url) do
-    validate_tld = Pleroma.Config.get([Pleroma.Formatter, :validate_tld])
+    validate_tld = Config.get([Pleroma.Formatter, :validate_tld])
 
     page_url
     |> Linkify.Parser.url?(validate_tld: validate_tld)
@@ -86,16 +87,6 @@ defmodule Pleroma.Web.RichMedia.Helpers do
   def rich_media_get(url) do
     headers = [{"user-agent", Pleroma.Application.user_agent() <> "; Bot"}]
 
-    options =
-      if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do
-        Keyword.merge(@rich_media_options,
-          recv_timeout: 2_000,
-          with_body: true
-        )
-      else
-        @rich_media_options
-      end
-
-    Pleroma.HTTP.get(url, headers, adapter: options)
+    Pleroma.HTTP.get(url, headers, @options)
   end
 end

lib/pleroma/workers/cron/stats_worker.ex

@@ -1,17 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Workers.Cron.StatsWorker do
-  @moduledoc """
-  The worker to update peers statistics.
-  """
-
-  use Oban.Worker, queue: "background"
-
-  @impl Oban.Worker
-  def perform(_job) do
-    Pleroma.Stats.do_collect()
-    :ok
-  end
-end