Merge remote-tracking branch 'origin/develop' into oban/backup

lib/pleroma/application.ex

@@ -14,7 +14,6 @@ defmodule Pleroma.Application do
   @name Mix.Project.config()[:name]
   @version Mix.Project.config()[:version]
   @repository Mix.Project.config()[:source_url]
-  @compile_env Mix.env()
 
   def name, do: @name
   def version, do: @version
@@ -53,7 +52,7 @@ defmodule Pleroma.Application do
     Pleroma.Config.Oban.warn()
     Config.DeprecationWarnings.warn()
 
-    if @compile_env != :test do
+    if Config.get([Pleroma.Web.Plugs.HTTPSecurityPlug, :enable], true) do
       Pleroma.Web.Plugs.HTTPSecurityPlug.warn_if_disabled()
     end
 

lib/pleroma/http.ex

@@ -68,7 +68,9 @@ defmodule Pleroma.HTTP do
 
     adapter = Application.get_env(:tesla, :adapter)
 
-    client = Tesla.client(adapter_middlewares(adapter), adapter)
+    extra_middleware = options[:tesla_middleware] || []
+
+    client = Tesla.client(adapter_middlewares(adapter, extra_middleware), adapter)
 
     maybe_limit(
       fn ->
@@ -102,20 +104,21 @@ defmodule Pleroma.HTTP do
     fun.()
   end
 
-  defp adapter_middlewares(Tesla.Adapter.Gun) do
-    [Tesla.Middleware.FollowRedirects, Pleroma.Tesla.Middleware.ConnectionPool]
+  defp adapter_middlewares(Tesla.Adapter.Gun, extra_middleware) do
+    [Tesla.Middleware.FollowRedirects, Pleroma.Tesla.Middleware.ConnectionPool] ++
+      extra_middleware
   end
 
-  defp adapter_middlewares({Tesla.Adapter.Finch, _}) do
-    [Tesla.Middleware.FollowRedirects]
+  defp adapter_middlewares({Tesla.Adapter.Finch, _}, extra_middleware) do
+    [Tesla.Middleware.FollowRedirects] ++ extra_middleware
   end
 
-  defp adapter_middlewares(_) do
+  defp adapter_middlewares(_, extra_middleware) do
     if Pleroma.Config.get(:env) == :test do
       # Emulate redirects in test env, which are handled by adapters in other environments
       [Tesla.Middleware.FollowRedirects]
     else
-      []
+      extra_middleware
     end
   end
 end

lib/pleroma/web/rich_media/backfill.ex

@@ -36,13 +36,9 @@ defmodule Pleroma.Web.RichMedia.Backfill do
         :ok
 
       {:error, type} = error
-      when type in [:invalid_metadata, :body_too_large, :content_type, :validate] ->
+      when type in [:invalid_metadata, :body_too_large, :content_type, :validate, :get, :head] ->
         negative_cache(url_hash)
         error
-
-      {:error, type} = error
-      when type in [:get, :head] ->
-        error
     end
   end
 
@@ -68,5 +64,5 @@ defmodule Pleroma.Web.RichMedia.Backfill do
   defp warm_cache(key, val), do: @cachex.put(:rich_media_cache, key, val)
 
   defp negative_cache(key, ttl \\ :timer.minutes(15)),
-    do: @cachex.put(:rich_media_cache, key, nil, ttl: ttl)
+    do: @cachex.put(:rich_media_cache, key, :error, ttl: ttl)
 end

lib/pleroma/web/rich_media/helpers.ex

@@ -69,9 +69,12 @@ defmodule Pleroma.Web.RichMedia.Helpers do
   end
 
   defp http_options do
+    timeout = Config.get!([:rich_media, :timeout])
+
     [
       pool: :rich_media,
-      max_body: Config.get([:rich_media, :max_body], 5_000_000)
+      max_body: Config.get([:rich_media, :max_body], 5_000_000),
+      tesla_middleware: [{Tesla.Middleware.Timeout, timeout: timeout}]
     ]
   end
 end

lib/pleroma/workers/rich_media_worker.ex

@@ -3,6 +3,7 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Workers.RichMediaWorker do
+  alias Pleroma.Config
   alias Pleroma.Web.RichMedia.Backfill
   alias Pleroma.Web.RichMedia.Card
 
@@ -19,18 +20,21 @@ defmodule Pleroma.Workers.RichMediaWorker do
         :ok
 
       {:error, type}
-      when type in [:invalid_metadata, :body_too_large, :content_type, :validate] ->
+      when type in [:invalid_metadata, :body_too_large, :content_type, :validate, :get, :head] ->
         {:cancel, type}
 
-      {:error, type}
-      when type in [:get, :head] ->
-        {:error, type}
-
       error ->
         {:error, error}
     end
   end
 
+  # There is timeout value enforced by Tesla.Middleware.Timeout
+  # which can be found in the RichMedia.Helpers module to allow us to detect
+  # a slow/infinite data stream and insert a negative cache entry for the URL
+  # We pad it by 2 seconds to be certain a slow connection is detected and we
+  # can inject a negative cache entry for the URL
   @impl Oban.Worker
-  def timeout(_job), do: :timer.seconds(5)
+  def timeout(_job) do
+    Config.get!([:rich_media, :timeout]) + :timer.seconds(2)
+  end
 end