Merge branch 'develop' into media-preview-proxy

2020-09-09 11:11:36 -05:00 · 2020-09-09 11:11:36 -05:00 · 4ee99dafcc
commit 4ee99dafcc
parent 563718cab9 9d20d29a79
123 changed files with 1809 additions and 308 deletions
--- a/lib/mix/pleroma.ex
+++ b/lib/mix/pleroma.ex
@ -18,6 +18,7 @@ defmodule Mix.Pleroma do
  @doc "Common functions to be reused in mix tasks"
  def start_pleroma do
    Pleroma.Config.Holder.save_default()
+    Pleroma.Config.Oban.warn()
    Application.put_env(:phoenix, :serve_endpoints, false, persistent: true)

    if Pleroma.Config.get(:env) != :test do
--- a/lib/mix/tasks/pleroma/benchmark.ex
+++ b/lib/mix/tasks/pleroma/benchmark.ex
@ -91,20 +91,17 @@ defmodule Mix.Tasks.Pleroma.Benchmark do
        "Without conn and without pool" => fn ->
          {:ok, %Tesla.Env{}} =
            Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [],
-              adapter: [pool: :no_pool, receive_conn: false]
+              pool: :no_pool,
+              receive_conn: false
            )
        end,
        "Without conn and with pool" => fn ->
          {:ok, %Tesla.Env{}} =
-            Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [],
-              adapter: [receive_conn: false]
-            )
+            Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [], receive_conn: false)
        end,
        "With reused conn and without pool" => fn ->
          {:ok, %Tesla.Env{}} =
-            Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [],
-              adapter: [pool: :no_pool]
-            )
+            Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [], pool: :no_pool)
        end,
        "With reused conn and with pool" => fn ->
          {:ok, %Tesla.Env{}} = Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500")
--- a/lib/mix/tasks/pleroma/frontend.ex
+++ b/lib/mix/tasks/pleroma/frontend.ex
@ -124,9 +124,7 @@ defmodule Mix.Tasks.Pleroma.Frontend do
    url = String.replace(frontend_info["build_url"], "${ref}", frontend_info["ref"])

    with {:ok, %{status: 200, body: zip_body}} <-
-           Pleroma.HTTP.get(url, [],
-             adapter: [pool: :media, timeout: 120_000, recv_timeout: 120_000]
-           ) do
+           Pleroma.HTTP.get(url, [], pool: :media, recv_timeout: 120_000) do
      unzip(zip_body, dest)
    else
      e -> {:error, e}
--- a/lib/pleroma/application.ex
+++ b/lib/pleroma/application.ex
@ -50,6 +50,7 @@ defmodule Pleroma.Application do
    Pleroma.Telemetry.Logger.attach()
    Config.Holder.save_default()
    Pleroma.HTML.compile_scrubbers()
+    Pleroma.Config.Oban.warn()
    Config.DeprecationWarnings.warn()
    Pleroma.Plugs.HTTPSecurityPlug.warn_if_disabled()
    Pleroma.ApplicationRequirements.verify!()
--- a/lib/pleroma/config/deprecation_warnings.ex
+++ b/lib/pleroma/config/deprecation_warnings.ex
@ -56,6 +56,7 @@ defmodule Pleroma.Config.DeprecationWarnings do
    check_old_mrf_config()
    check_media_proxy_whitelist_config()
    check_welcome_message_config()
+    check_gun_pool_options()
  end

  def check_welcome_message_config do
@ -115,4 +116,46 @@ defmodule Pleroma.Config.DeprecationWarnings do
      """)
    end
  end
+
+  def check_gun_pool_options do
+    pool_config = Config.get(:connections_pool)
+
+    if timeout = pool_config[:await_up_timeout] do
+      Logger.warn("""
+      !!!DEPRECATION WARNING!!!
+      Your config is using old setting name `await_up_timeout` instead of `connect_timeout`. Setting should work for now, but you are advised to change format to scheme with port to prevent possible issues later.
+      """)
+
+      Config.put(:connections_pool, Keyword.put_new(pool_config, :connect_timeout, timeout))
+    end
+
+    pools_configs = Config.get(:pools)
+
+    warning_preface = """
+    !!!DEPRECATION WARNING!!!
+    Your config is using old setting name `timeout` instead of `recv_timeout` in pool settings. Setting should work for now, but you are advised to change format to scheme with port to prevent possible issues later.
+    """
+
+    updated_config =
+      Enum.reduce(pools_configs, [], fn {pool_name, config}, acc ->
+        if timeout = config[:timeout] do
+          Keyword.put(acc, pool_name, Keyword.put_new(config, :recv_timeout, timeout))
+        else
+          acc
+        end
+      end)
+
+    if updated_config != [] do
+      pool_warnings =
+        updated_config
+        |> Keyword.keys()
+        |> Enum.map(fn pool_name ->
+          "\n* `:timeout` options in #{pool_name} pool is now `:recv_timeout`"
+        end)
+
+      Logger.warn(Enum.join([warning_preface | pool_warnings]))
+
+      Config.put(:pools, updated_config)
+    end
+  end
 end
--- a/lib/pleroma/config/oban.ex
+++ b/lib/pleroma/config/oban.ex
@ -0,0 +1,30 @@
+defmodule Pleroma.Config.Oban do
+  require Logger
+
+  def warn do
+    oban_config = Pleroma.Config.get(Oban)
+
+    crontab =
+      [Pleroma.Workers.Cron.StatsWorker]
+      |> Enum.reduce(oban_config[:crontab], fn removed_worker, acc ->
+        with acc when is_list(acc) <- acc,
+             setting when is_tuple(setting) <-
+               Enum.find(acc, fn {_, worker} -> worker == removed_worker end) do
+          """
+          !!!OBAN CONFIG WARNING!!!
+          You are using old workers in Oban crontab settings, which were removed.
+          Please, remove setting from crontab in your config file (prod.secret.exs): #{
+            inspect(setting)
+          }
+          """
+          |> Logger.warn()
+
+          List.delete(acc, setting)
+        else
+          _ -> acc
+        end
+      end)
+
+    Pleroma.Config.put(Oban, Keyword.put(oban_config, :crontab, crontab))
+  end
+end
--- a/lib/pleroma/gun/conn.ex
+++ b/lib/pleroma/gun/conn.ex
@ -13,7 +13,7 @@ defmodule Pleroma.Gun.Conn do
    opts =
      opts
      |> Enum.into(%{})
-      |> Map.put_new(:await_up_timeout, pool_opts[:await_up_timeout] || 5_000)
+      |> Map.put_new(:connect_timeout, pool_opts[:connect_timeout] || 5_000)
      |> Map.put_new(:supervise, false)
      |> maybe_add_tls_opts(uri)

@ -50,7 +50,7 @@ defmodule Pleroma.Gun.Conn do

    with open_opts <- Map.delete(opts, :tls_opts),
         {:ok, conn} <- Gun.open(proxy_host, proxy_port, open_opts),
-         {:ok, _} <- Gun.await_up(conn, opts[:await_up_timeout]),
+         {:ok, _} <- Gun.await_up(conn, opts[:connect_timeout]),
         stream <- Gun.connect(conn, connect_opts),
         {:response, :fin, 200, _} <- Gun.await(conn, stream) do
      {:ok, conn}
@ -88,7 +88,7 @@ defmodule Pleroma.Gun.Conn do
      |> Map.put(:socks_opts, socks_opts)

    with {:ok, conn} <- Gun.open(proxy_host, proxy_port, opts),
-         {:ok, _} <- Gun.await_up(conn, opts[:await_up_timeout]) do
+         {:ok, _} <- Gun.await_up(conn, opts[:connect_timeout]) do
      {:ok, conn}
    else
      error ->
@ -106,7 +106,7 @@ defmodule Pleroma.Gun.Conn do
    host = Pleroma.HTTP.AdapterHelper.parse_host(host)

    with {:ok, conn} <- Gun.open(host, port, opts),
-         {:ok, _} <- Gun.await_up(conn, opts[:await_up_timeout]) do
+         {:ok, _} <- Gun.await_up(conn, opts[:connect_timeout]) do
      {:ok, conn}
    else
      error ->
--- a/lib/pleroma/html.ex
+++ b/lib/pleroma/html.ex
@ -100,21 +100,27 @@ defmodule Pleroma.HTML do
    end)
  end

-  def extract_first_external_url(_, nil), do: {:error, "No content"}
+  def extract_first_external_url_from_object(%{data: %{"content" => content}} = object)
+      when is_binary(content) do
+    unless object.data["fake"] do
+      key = "URL|#{object.id}"

-  def extract_first_external_url(object, content) do
-    key = "URL|#{object.id}"
+      Cachex.fetch!(:scrubber_cache, key, fn _key ->
+        {:commit, {:ok, extract_first_external_url(content)}}
+      end)
+    else
+      {:ok, extract_first_external_url(content)}
+    end
+  end

-    Cachex.fetch!(:scrubber_cache, key, fn _key ->
-      result =
-        content
-        |> Floki.parse_fragment!()
-        |> Floki.find("a:not(.mention,.hashtag,.attachment,[rel~=\"tag\"])")
-        |> Enum.take(1)
-        |> Floki.attribute("href")
-        |> Enum.at(0)
+  def extract_first_external_url_from_object(_), do: {:error, :no_content}

-      {:commit, {:ok, result}}
-    end)
+  def extract_first_external_url(content) do
+    content
+    |> Floki.parse_fragment!()
+    |> Floki.find("a:not(.mention,.hashtag,.attachment,[rel~=\"tag\"])")
+    |> Enum.take(1)
+    |> Floki.attribute("href")
+    |> Enum.at(0)
  end
 end
--- a/lib/pleroma/http/adapter_helper.ex
+++ b/lib/pleroma/http/adapter_helper.ex
@ -6,7 +6,7 @@ defmodule Pleroma.HTTP.AdapterHelper do
  @moduledoc """
  Configure Tesla.Client with default and customized adapter options.
  """
-  @defaults [pool: :federation]
+  @defaults [pool: :federation, connect_timeout: 5_000, recv_timeout: 5_000]

  @type proxy_type() :: :socks4 | :socks5
  @type host() :: charlist() | :inet.ip_address()
--- a/lib/pleroma/http/adapter_helper/gun.ex
+++ b/lib/pleroma/http/adapter_helper/gun.ex
@ -11,12 +11,8 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do
  require Logger

  @defaults [
-    connect_timeout: 5_000,
-    domain_lookup_timeout: 5_000,
-    tls_handshake_timeout: 5_000,
    retry: 1,
-    retry_timeout: 1000,
-    await_up_timeout: 5_000
+    retry_timeout: 1_000
  ]

  @type pool() :: :federation | :upload | :media | :default
@ -45,15 +41,17 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do
  end

  defp put_timeout(opts) do
+    {recv_timeout, opts} = Keyword.pop(opts, :recv_timeout, pool_timeout(opts[:pool]))
    # this is the timeout to receive a message from Gun
-    Keyword.put_new(opts, :timeout, pool_timeout(opts[:pool]))
+    # `:timeout` key is used in Tesla
+    Keyword.put(opts, :timeout, recv_timeout)
  end

  @spec pool_timeout(pool()) :: non_neg_integer()
  def pool_timeout(pool) do
-    default = Config.get([:pools, :default, :timeout], 5_000)
+    default = Config.get([:pools, :default, :recv_timeout], 5_000)

-    Config.get([:pools, pool, :timeout], default)
+    Config.get([:pools, pool, :recv_timeout], default)
  end

  @prefix Pleroma.Gun.ConnectionPool
--- a/lib/pleroma/http/adapter_helper/hackney.ex
+++ b/lib/pleroma/http/adapter_helper/hackney.ex
@ -2,11 +2,8 @@ defmodule Pleroma.HTTP.AdapterHelper.Hackney do
  @behaviour Pleroma.HTTP.AdapterHelper

  @defaults [
-    connect_timeout: 10_000,
-    recv_timeout: 20_000,
    follow_redirect: true,
-    force_redirect: true,
-    pool: :federation
+    force_redirect: true
  ]

  @spec options(keyword(), URI.t()) :: keyword()
@ -19,8 +16,21 @@ defmodule Pleroma.HTTP.AdapterHelper.Hackney do
    |> Keyword.merge(config_opts)
    |> Keyword.merge(connection_opts)
    |> add_scheme_opts(uri)
+    |> maybe_add_with_body()
    |> Pleroma.HTTP.AdapterHelper.maybe_add_proxy(proxy)
  end

+  defp add_scheme_opts(opts, %URI{scheme: "https"}) do
+    Keyword.put(opts, :ssl_options, versions: [:"tlsv1.2", :"tlsv1.1", :tlsv1])
+  end
+
  defp add_scheme_opts(opts, _), do: opts
+
+  defp maybe_add_with_body(opts) do
+    if opts[:max_body] do
+      Keyword.put(opts, :with_body, true)
+    else
+      opts
+    end
+  end
 end
--- a/lib/pleroma/http/ex_aws.ex
+++ b/lib/pleroma/http/ex_aws.ex
@ -11,7 +11,7 @@ defmodule Pleroma.HTTP.ExAws do

  @impl true
  def request(method, url, body \\ "", headers \\ [], http_opts \\ []) do
-    http_opts = Keyword.put_new(http_opts, :adapter, pool: :upload)
+    http_opts = Keyword.put_new(http_opts, :pool, :upload)

    case HTTP.request(method, url, body, headers, http_opts) do
      {:ok, env} ->
--- a/lib/pleroma/http/http.ex
+++ b/lib/pleroma/http/http.ex
@ -60,7 +60,7 @@ defmodule Pleroma.HTTP do
          {:ok, Env.t()} | {:error, any()}
  def request(method, url, body, headers, options) when is_binary(url) do
    uri = URI.parse(url)
-    adapter_opts = AdapterHelper.options(uri, options[:adapter] || [])
+    adapter_opts = AdapterHelper.options(uri, options || [])

    options = put_in(options[:adapter], adapter_opts)
    params = options[:params] || []
--- a/lib/pleroma/http/tzdata.ex
+++ b/lib/pleroma/http/tzdata.ex
@ -11,7 +11,7 @@ defmodule Pleroma.HTTP.Tzdata do

  @impl true
  def get(url, headers, options) do
-    options = Keyword.put_new(options, :adapter, pool: :default)
+    options = Keyword.put_new(options, :pool, :default)

    with {:ok, %Tesla.Env{} = env} <- HTTP.get(url, headers, options) do
      {:ok, {env.status, env.headers, env.body}}
@ -20,7 +20,7 @@ defmodule Pleroma.HTTP.Tzdata do

  @impl true
  def head(url, headers, options) do
-    options = Keyword.put_new(options, :adapter, pool: :default)
+    options = Keyword.put_new(options, :pool, :default)

    with {:ok, %Tesla.Env{} = env} <- HTTP.head(url, headers, options) do
      {:ok, {env.status, env.headers}}
--- a/lib/pleroma/instances/instance.ex
+++ b/lib/pleroma/instances/instance.ex
@ -14,6 +14,8 @@ defmodule Pleroma.Instances.Instance do
  import Ecto.Query
  import Ecto.Changeset

+  require Logger
+
  schema "instances" do
    field(:host, :string)
    field(:unreachable_since, :naive_datetime_usec)
@ -145,6 +147,10 @@ defmodule Pleroma.Instances.Instance do

      favicon
    end
+  rescue
+    e ->
+      Logger.warn("Instance.get_or_update_favicon(\"#{host}\") error: #{inspect(e)}")
+      nil
  end

  defp scrape_favicon(%URI{} = instance_uri) do
@ -165,7 +171,12 @@ defmodule Pleroma.Instances.Instance do
        _ -> nil
      end
    rescue
-      _ -> nil
+      e ->
+        Logger.warn(
+          "Instance.scrape_favicon(\"#{to_string(instance_uri)}\") error: #{inspect(e)}"
+        )
+
+        nil
    end
  end
 end
--- a/lib/pleroma/reverse_proxy/client/hackney.ex
+++ b/lib/pleroma/reverse_proxy/client/hackney.ex
@ -7,6 +7,7 @@ defmodule Pleroma.ReverseProxy.Client.Hackney do

  @impl true
  def request(method, url, headers, body, opts \\ []) do
+    opts = Keyword.put(opts, :ssl_options, versions: [:"tlsv1.2", :"tlsv1.1", :tlsv1])
    :hackney.request(method, url, headers, body, opts)
  end

--- a/lib/pleroma/stats.ex
+++ b/lib/pleroma/stats.ex
@ -3,12 +3,15 @@
 # SPDX-License-Identifier: AGPL-3.0-only

 defmodule Pleroma.Stats do
+  use GenServer
+
  import Ecto.Query
+
  alias Pleroma.CounterCache
  alias Pleroma.Repo
  alias Pleroma.User

-  use GenServer
+  @interval :timer.seconds(60)

  def start_link(_) do
    GenServer.start_link(
@ -18,6 +21,12 @@ defmodule Pleroma.Stats do
    )
  end

+  @impl true
+  def init(_args) do
+    if Pleroma.Config.get(:env) == :test, do: :ok = Ecto.Adapters.SQL.Sandbox.checkout(Repo)
+    {:ok, nil, {:continue, :calculate_stats}}
+  end
+
  @doc "Performs update stats"
  def force_update do
    GenServer.call(__MODULE__, :force_update)
@ -29,7 +38,11 @@ defmodule Pleroma.Stats do
  end

  @doc "Returns stats data"
-  @spec get_stats() :: %{domain_count: integer(), status_count: integer(), user_count: integer()}
+  @spec get_stats() :: %{
+          domain_count: non_neg_integer(),
+          status_count: non_neg_integer(),
+          user_count: non_neg_integer()
+        }
  def get_stats do
    %{stats: stats} = GenServer.call(__MODULE__, :get_state)

@ -44,25 +57,14 @@ defmodule Pleroma.Stats do
    peers
  end

-  def init(_args) do
-    {:ok, calculate_stat_data()}
-  end
-
-  def handle_call(:force_update, _from, _state) do
-    new_stats = calculate_stat_data()
-    {:reply, new_stats, new_stats}
-  end
-
-  def handle_call(:get_state, _from, state) do
-    {:reply, state, state}
-  end
-
-  def handle_cast(:run_update, _state) do
-    new_stats = calculate_stat_data()
-
-    {:noreply, new_stats}
-  end
-
+  @spec calculate_stat_data() :: %{
+          peers: list(),
+          stats: %{
+            domain_count: non_neg_integer(),
+            status_count: non_neg_integer(),
+            user_count: non_neg_integer()
+          }
+        }
  def calculate_stat_data do
    peers =
      from(
@ -97,6 +99,7 @@ defmodule Pleroma.Stats do
    }
  end

+  @spec get_status_visibility_count(String.t() | nil) :: map()
  def get_status_visibility_count(instance \\ nil) do
    if is_nil(instance) do
      CounterCache.get_sum()
@ -104,4 +107,36 @@ defmodule Pleroma.Stats do
      CounterCache.get_by_instance(instance)
    end
  end
+
+  @impl true
+  def handle_continue(:calculate_stats, _) do
+    stats = calculate_stat_data()
+    Process.send_after(self(), :run_update, @interval)
+    {:noreply, stats}
+  end
+
+  @impl true
+  def handle_call(:force_update, _from, _state) do
+    new_stats = calculate_stat_data()
+    {:reply, new_stats, new_stats}
+  end
+
+  @impl true
+  def handle_call(:get_state, _from, state) do
+    {:reply, state, state}
+  end
+
+  @impl true
+  def handle_cast(:run_update, _state) do
+    new_stats = calculate_stat_data()
+
+    {:noreply, new_stats}
+  end
+
+  @impl true
+  def handle_info(:run_update, _) do
+    new_stats = calculate_stat_data()
+    Process.send_after(self(), :run_update, @interval)
+    {:noreply, new_stats}
+  end
 end
--- a/lib/pleroma/upload/filter.ex
+++ b/lib/pleroma/upload/filter.ex
@ -15,7 +15,11 @@ defmodule Pleroma.Upload.Filter do

  require Logger

-  @callback filter(Pleroma.Upload.t()) :: :ok | {:ok, Pleroma.Upload.t()} | {:error, any()}
+  @callback filter(Pleroma.Upload.t()) ::
+              {:ok, :filtered}
+              | {:ok, :noop}
+              | {:ok, :filtered, Pleroma.Upload.t()}
+              | {:error, any()}

  @spec filter([module()], Pleroma.Upload.t()) :: {:ok, Pleroma.Upload.t()} | {:error, any()}

@ -25,10 +29,13 @@ defmodule Pleroma.Upload.Filter do

  def filter([filter | rest], upload) do
    case filter.filter(upload) do
-      :ok ->
+      {:ok, :filtered} ->
        filter(rest, upload)

-      {:ok, upload} ->
+      {:ok, :filtered, upload} ->
+        filter(rest, upload)
+
+      {:ok, :noop} ->
        filter(rest, upload)

      error ->
--- a/lib/pleroma/upload/filter/anonymize_filename.ex
+++ b/lib/pleroma/upload/filter/anonymize_filename.ex
@ -16,9 +16,11 @@ defmodule Pleroma.Upload.Filter.AnonymizeFilename do
  def filter(%Upload{name: name} = upload) do
    extension = List.last(String.split(name, "."))
    name = predefined_name(extension) || random(extension)
-    {:ok, %Upload{upload | name: name}}
+    {:ok, :filtered, %Upload{upload | name: name}}
  end

+  def filter(_), do: {:ok, :noop}
+
  @spec predefined_name(String.t()) :: String.t() | nil
  defp predefined_name(extension) do
    with name when not is_nil(name) <- Config.get([__MODULE__, :text]),
--- a/lib/pleroma/upload/filter/dedupe.ex
+++ b/lib/pleroma/upload/filter/dedupe.ex
@ -17,8 +17,8 @@ defmodule Pleroma.Upload.Filter.Dedupe do
      |> Base.encode16(case: :lower)

    filename = shasum <> "." <> extension
-    {:ok, %Upload{upload | id: shasum, path: filename}}
+    {:ok, :filtered, %Upload{upload | id: shasum, path: filename}}
  end

-  def filter(_), do: :ok
+  def filter(_), do: {:ok, :noop}
 end
--- a/lib/pleroma/upload/filter/exiftool.ex
+++ b/lib/pleroma/upload/filter/exiftool.ex
@ -9,11 +9,15 @@ defmodule Pleroma.Upload.Filter.Exiftool do
  """
  @behaviour Pleroma.Upload.Filter

-  @spec filter(Pleroma.Upload.t()) :: :ok | {:error, String.t()}
+  @spec filter(Pleroma.Upload.t()) :: {:ok, any()} | {:error, String.t()}
+
+  # webp is not compatible with exiftool at this time
+  def filter(%Pleroma.Upload{content_type: "image/webp"}), do: {:ok, :noop}
+
  def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do
    try do
      case System.cmd("exiftool", ["-overwrite_original", "-gps:all=", file], parallelism: true) do
-        {_response, 0} -> :ok
+        {_response, 0} -> {:ok, :filtered}
        {error, 1} -> {:error, error}
      end
    rescue
@ -22,5 +26,5 @@ defmodule Pleroma.Upload.Filter.Exiftool do
    end
  end

-  def filter(_), do: :ok
+  def filter(_), do: {:ok, :noop}
 end
--- a/lib/pleroma/upload/filter/mogrifun.ex
+++ b/lib/pleroma/upload/filter/mogrifun.ex
@ -38,16 +38,16 @@ defmodule Pleroma.Upload.Filter.Mogrifun do
    [{"fill", "yellow"}, {"tint", "40"}]
  ]

-  @spec filter(Pleroma.Upload.t()) :: :ok | {:error, String.t()}
+  @spec filter(Pleroma.Upload.t()) :: {:ok, atom()} | {:error, String.t()}
  def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do
    try do
      Filter.Mogrify.do_filter(file, [Enum.random(@filters)])
-      :ok
+      {:ok, :filtered}
    rescue
      _e in ErlangError ->
        {:error, "mogrify command not found"}
    end
  end

-  def filter(_), do: :ok
+  def filter(_), do: {:ok, :noop}
 end
--- a/lib/pleroma/upload/filter/mogrify.ex
+++ b/lib/pleroma/upload/filter/mogrify.ex
@ -8,18 +8,18 @@ defmodule Pleroma.Upload.Filter.Mogrify do
  @type conversion :: action :: String.t() | {action :: String.t(), opts :: String.t()}
  @type conversions :: conversion() | [conversion()]

-  @spec filter(Pleroma.Upload.t()) :: :ok | {:error, String.t()}
+  @spec filter(Pleroma.Upload.t()) :: {:ok, :atom} | {:error, String.t()}
  def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do
    try do
      do_filter(file, Pleroma.Config.get!([__MODULE__, :args]))
-      :ok
+      {:ok, :filtered}
    rescue
      _e in ErlangError ->
        {:error, "mogrify command not found"}
    end
  end

-  def filter(_), do: :ok
+  def filter(_), do: {:ok, :noop}

  def do_filter(file, filters) do
    file
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@ -1125,31 +1125,31 @@ defmodule Pleroma.User do
    User.Query.build(%{followers: user, deactivated: false})
  end

-  def get_followers_query(user, page) do
+  def get_followers_query(%User{} = user, page) do
    user
    |> get_followers_query(nil)
    |> User.Query.paginate(page, 20)
  end

  @spec get_followers_query(User.t()) :: Ecto.Query.t()
-  def get_followers_query(user), do: get_followers_query(user, nil)
+  def get_followers_query(%User{} = user), do: get_followers_query(user, nil)

  @spec get_followers(User.t(), pos_integer() | nil) :: {:ok, list(User.t())}
-  def get_followers(user, page \\ nil) do
+  def get_followers(%User{} = user, page \\ nil) do
    user
    |> get_followers_query(page)
    |> Repo.all()
  end

  @spec get_external_followers(User.t(), pos_integer() | nil) :: {:ok, list(User.t())}
-  def get_external_followers(user, page \\ nil) do
+  def get_external_followers(%User{} = user, page \\ nil) do
    user
    |> get_followers_query(page)
    |> User.Query.build(%{external: true})
    |> Repo.all()
  end

-  def get_followers_ids(user, page \\ nil) do
+  def get_followers_ids(%User{} = user, page \\ nil) do
    user
    |> get_followers_query(page)
    |> select([u], u.id)
@ -1161,29 +1161,29 @@ defmodule Pleroma.User do
    User.Query.build(%{friends: user, deactivated: false})
  end

-  def get_friends_query(user, page) do
+  def get_friends_query(%User{} = user, page) do
    user
    |> get_friends_query(nil)
    |> User.Query.paginate(page, 20)
  end

  @spec get_friends_query(User.t()) :: Ecto.Query.t()
-  def get_friends_query(user), do: get_friends_query(user, nil)
+  def get_friends_query(%User{} = user), do: get_friends_query(user, nil)

-  def get_friends(user, page \\ nil) do
+  def get_friends(%User{} = user, page \\ nil) do
    user
    |> get_friends_query(page)
    |> Repo.all()
  end

-  def get_friends_ap_ids(user) do
+  def get_friends_ap_ids(%User{} = user) do
    user
    |> get_friends_query(nil)
    |> select([u], u.ap_id)
    |> Repo.all()
  end

-  def get_friends_ids(user, page \\ nil) do
+  def get_friends_ids(%User{} = user, page \\ nil) do
    user
    |> get_friends_query(page)
    |> select([u], u.id)
--- a/lib/pleroma/user/search.ex
+++ b/lib/pleroma/user/search.ex
@ -115,8 +115,8 @@ defmodule Pleroma.User.Search do
    )
  end

-  defp base_query(_user, false), do: User
-  defp base_query(user, true), do: User.get_friends_query(user)
+  defp base_query(%User{} = user, true), do: User.get_friends_query(user)
+  defp base_query(_user, _following), do: User

  defp filter_invisible_users(query) do
    from(q in query, where: q.invisible == false)
--- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex
@ -399,21 +399,30 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do

  defp handle_user_activity(
         %User{} = user,
-         %{"type" => "Create", "object" => %{"type" => "Note"}} = params
+         %{"type" => "Create", "object" => %{"type" => "Note"} = object} = params
       ) do
-    object =
-      params["object"]
-      |> Map.merge(Map.take(params, ["to", "cc"]))
-      |> Map.put("attributedTo", user.ap_id())
-      |> Transmogrifier.fix_object()
+    content = if is_binary(object["content"]), do: object["content"], else: ""
+    name = if is_binary(object["name"]), do: object["name"], else: ""
+    summary = if is_binary(object["summary"]), do: object["summary"], else: ""
+    length = String.length(content <> name <> summary)

-    ActivityPub.create(%{
-      to: params["to"],
-      actor: user,
-      context: object["context"],
-      object: object,
-      additional: Map.take(params, ["cc"])
-    })
+    if length > Pleroma.Config.get([:instance, :limit]) do
+      {:error, dgettext("errors", "Note is over the character limit")}
+    else
+      object =
+        object
+        |> Map.merge(Map.take(params, ["to", "cc"]))
+        |> Map.put("attributedTo", user.ap_id())
+        |> Transmogrifier.fix_object()
+
+      ActivityPub.create(%{
+        to: params["to"],
+        actor: user,
+        context: object["context"],
+        object: object,
+        additional: Map.take(params, ["cc"])
+      })
+    end
  end

  defp handle_user_activity(%User{} = user, %{"type" => "Delete"} = params) do
--- a/lib/pleroma/web/activity_pub/mrf/force_bot_unlisted_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/force_bot_unlisted_policy.ex
@ -0,0 +1,56 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.MRF.ForceBotUnlistedPolicy do
+  alias Pleroma.User
+  @behaviour Pleroma.Web.ActivityPub.MRF
+  @moduledoc "Remove bot posts from federated timeline"
+
+  require Pleroma.Constants
+
+  defp check_by_actor_type(user), do: user.actor_type in ["Application", "Service"]
+  defp check_by_nickname(user), do: Regex.match?(~r/bot@|ebooks@/i, user.nickname)
+
+  defp check_if_bot(user), do: check_by_actor_type(user) or check_by_nickname(user)
+
+  @impl true
+  def filter(
+        %{
+          "type" => "Create",
+          "to" => to,
+          "cc" => cc,
+          "actor" => actor,
+          "object" => object
+        } = message
+      ) do
+    user = User.get_cached_by_ap_id(actor)
+    isbot = check_if_bot(user)
+
+    if isbot and Enum.member?(to, Pleroma.Constants.as_public()) do
+      to = List.delete(to, Pleroma.Constants.as_public()) ++ [user.follower_address]
+      cc = List.delete(cc, user.follower_address) ++ [Pleroma.Constants.as_public()]
+
+      object =
+        object
+        |> Map.put("to", to)
+        |> Map.put("cc", cc)
+
+      message =
+        message
+        |> Map.put("to", to)
+        |> Map.put("cc", cc)
+        |> Map.put("object", object)
+
+      {:ok, message}
+    else
+      {:ok, message}
+    end
+  end
+
+  @impl true
+  def filter(message), do: {:ok, message}
+
+  @impl true
+  def describe, do: {:ok, %{}}
+end
--- a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex
+++ b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex
@ -379,8 +379,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
    filters
    |> String.split(",")
    |> Enum.filter(&Enum.member?(@filters, &1))
-    |> Enum.map(&String.to_atom/1)
-    |> Map.new(&{&1, true})
+    |> Map.new(&{String.to_existing_atom(&1), true})
  end

  def right_add_multiple(%{assigns: %{user: admin}} = conn, %{
--- a/lib/pleroma/web/feed/tag_controller.ex
+++ b/lib/pleroma/web/feed/tag_controller.ex
@ -9,7 +9,15 @@ defmodule Pleroma.Web.Feed.TagController do
  alias Pleroma.Web.ActivityPub.ActivityPub
  alias Pleroma.Web.Feed.FeedView

-  def feed(conn, %{"tag" => raw_tag} = params) do
+  def feed(conn, params) do
+    unless Pleroma.Config.restrict_unauthenticated_access?(:activities, :local) do
+      render_feed(conn, params)
+    else
+      render_error(conn, :not_found, "Not found")
+    end
+  end
+
+  def render_feed(conn, %{"tag" => raw_tag} = params) do
    {format, tag} = parse_tag(raw_tag)

    activities =
--- a/lib/pleroma/web/feed/user_controller.ex
+++ b/lib/pleroma/web/feed/user_controller.ex
@ -37,7 +37,15 @@ defmodule Pleroma.Web.Feed.UserController do
    end
  end

-  def feed(conn, %{"nickname" => nickname} = params) do
+  def feed(conn, params) do
+    unless Pleroma.Config.restrict_unauthenticated_access?(:profiles, :local) do
+      render_feed(conn, params)
+    else
+      errors(conn, {:error, :not_found})
+    end
+  end
+
+  def render_feed(conn, %{"nickname" => nickname} = params) do
    format = get_format(conn)

    format =
--- a/lib/pleroma/web/metadata.ex
+++ b/lib/pleroma/web/metadata.ex
@ -7,8 +7,9 @@ defmodule Pleroma.Web.Metadata do

  def build_tags(params) do
    providers = [
+      Pleroma.Web.Metadata.Providers.RelMe,
      Pleroma.Web.Metadata.Providers.RestrictIndexing
-      | Pleroma.Config.get([__MODULE__, :providers], [])
+      | activated_providers()
    ]

    Enum.reduce(providers, "", fn parser, acc ->
@ -42,4 +43,12 @@ defmodule Pleroma.Web.Metadata do
  def activity_nsfw?(_) do
    false
  end
+
+  defp activated_providers do
+    unless Pleroma.Config.restrict_unauthenticated_access?(:activities, :local) do
+      [Pleroma.Web.Metadata.Providers.Feed | Pleroma.Config.get([__MODULE__, :providers], [])]
+    else
+      []
+    end
+  end
 end
--- a/lib/pleroma/web/rel_me.ex
+++ b/lib/pleroma/web/rel_me.ex
@ -5,7 +5,8 @@
 defmodule Pleroma.Web.RelMe do
  @options [
    pool: :media,
-    max_body: 2_000_000
+    max_body: 2_000_000,
+    recv_timeout: 2_000
  ]

  if Pleroma.Config.get(:env) == :test do
@ -23,18 +24,8 @@ defmodule Pleroma.Web.RelMe do
  def parse(_), do: {:error, "No URL provided"}

  defp parse_url(url) do
-    opts =
-      if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do
-        Keyword.merge(@options,
-          recv_timeout: 2_000,
-          with_body: true
-        )
-      else
-        @options
-      end
-
    with {:ok, %Tesla.Env{body: html, status: status}} when status in 200..299 <-
-           Pleroma.HTTP.get(url, [], adapter: opts),
+           Pleroma.HTTP.get(url, [], @options),
         {:ok, html_tree} <- Floki.parse_document(html),
         data <-
           Floki.attribute(html_tree, "link[rel~=me]", "href") ++
--- a/lib/pleroma/web/rich_media/helpers.ex
+++ b/lib/pleroma/web/rich_media/helpers.ex
@ -9,14 +9,15 @@ defmodule Pleroma.Web.RichMedia.Helpers do
  alias Pleroma.Object
  alias Pleroma.Web.RichMedia.Parser

-  @rich_media_options [
+  @options [
    pool: :media,
-    max_body: 2_000_000
+    max_body: 2_000_000,
+    recv_timeout: 2_000
  ]

  @spec validate_page_url(URI.t() | binary()) :: :ok | :error
  defp validate_page_url(page_url) when is_binary(page_url) do
-    validate_tld = Pleroma.Config.get([Pleroma.Formatter, :validate_tld])
+    validate_tld = Config.get([Pleroma.Formatter, :validate_tld])

    page_url
    |> Linkify.Parser.url?(validate_tld: validate_tld)
@ -58,7 +59,7 @@ defmodule Pleroma.Web.RichMedia.Helpers do
    with true <- Config.get([:rich_media, :enabled]),
         false <- object.data["sensitive"] || false,
         {:ok, page_url} <-
-           HTML.extract_first_external_url(object, object.data["content"]),
+           HTML.extract_first_external_url_from_object(object),
         :ok <- validate_page_url(page_url),
         {:ok, rich_media} <- Parser.parse(page_url) do
      %{page_url: page_url, rich_media: rich_media}
@ -86,16 +87,6 @@ defmodule Pleroma.Web.RichMedia.Helpers do
  def rich_media_get(url) do
    headers = [{"user-agent", Pleroma.Application.user_agent() <> "; Bot"}]

-    options =
-      if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do
-        Keyword.merge(@rich_media_options,
-          recv_timeout: 2_000,
-          with_body: true
-        )
-      else
-        @rich_media_options
-      end
-
-    Pleroma.HTTP.get(url, headers, adapter: options)
+    Pleroma.HTTP.get(url, headers, @options)
  end
 end
--- a/lib/pleroma/web/rich_media/parser.ex
+++ b/lib/pleroma/web/rich_media/parser.ex
@ -21,8 +21,13 @@ defmodule Pleroma.Web.RichMedia.Parser do
           {:ok, _} <- set_ttl_based_on_image(data, url) do
        {:ok, data}
      else
+        {:error, {:invalid_metadata, data}} = e ->
+          Logger.debug(fn -> "Incomplete or invalid metadata for #{url}: #{inspect(data)}" end)
+          e
+
        error ->
-          Logger.error(fn -> "Rich media error: #{inspect(error)}" end)
+          Logger.error(fn -> "Rich media error for #{url}: #{inspect(error)}" end)
+          error
      end
    end

@ -90,7 +95,7 @@ defmodule Pleroma.Web.RichMedia.Parser do
    end)
  end

-  defp parse_url(url) do
+  def parse_url(url) do
    with {:ok, %Tesla.Env{body: html}} <- Pleroma.Web.RichMedia.Helpers.rich_media_get(url),
         {:ok, html} <- Floki.parse_document(html) do
      html
@ -116,7 +121,7 @@ defmodule Pleroma.Web.RichMedia.Parser do
  end

  defp check_parsed_data(data) do
-    {:error, "Found metadata was invalid or incomplete: #{inspect(data)}"}
+    {:error, {:invalid_metadata, data}}
  end

  defp clean_parsed_data(data) do
--- a/lib/pleroma/workers/cron/stats_worker.ex
+++ b/lib/pleroma/workers/cron/stats_worker.ex
@ -1,17 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Workers.Cron.StatsWorker do
-  @moduledoc """
-  The worker to update peers statistics.
-  """
-
-  use Oban.Worker, queue: "background"
-
-  @impl Oban.Worker
-  def perform(_job) do
-    Pleroma.Stats.do_collect()
-    :ok
-  end
-end