Merge branch 'media-preview-proxy-nostream' into 'develop'

Media preview proxy

See merge request pleroma/pleroma!3001

test/web/activity_pub/mrf/mediaproxy_warming_policy_test.exs

@@ -22,6 +22,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicyTest do
     }
   }
 
+  setup do: clear_config([:media_proxy, :enabled], true)
+
   test "it prefetches media proxy URIs" do
     with_mock HTTP, get: fn _, _, _ -> {:ok, []} end do
       MediaProxyWarmingPolicy.filter(@message)

test/web/mastodon_api/views/account_view_test.exs

@@ -5,6 +5,7 @@
 defmodule Pleroma.Web.MastodonAPI.AccountViewTest do
   use Pleroma.DataCase
 
+  alias Pleroma.Config
   alias Pleroma.User
   alias Pleroma.UserRelationship
   alias Pleroma.Web.CommonAPI
@@ -540,8 +541,9 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do
     end
   end
 
-  test "uses mediaproxy urls when it's enabled" do
+  test "uses mediaproxy urls when it's enabled (regardless of media preview proxy state)" do
     clear_config([:media_proxy, :enabled], true)
+    clear_config([:media_preview_proxy, :enabled])
 
     user =
       insert(:user,
@@ -550,20 +552,24 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do
         emoji: %{"joker_smile" => "https://evil.website/society.png"}
       )
 
-    AccountView.render("show.json", %{user: user, skip_visibility_check: true})
-    |> Enum.all?(fn
-      {key, url} when key in [:avatar, :avatar_static, :header, :header_static] ->
-        String.starts_with?(url, Pleroma.Web.base_url())
+    with media_preview_enabled <- [false, true] do
+      Config.put([:media_preview_proxy, :enabled], media_preview_enabled)
 
-      {:emojis, emojis} ->
-        Enum.all?(emojis, fn %{url: url, static_url: static_url} ->
-          String.starts_with?(url, Pleroma.Web.base_url()) &&
-            String.starts_with?(static_url, Pleroma.Web.base_url())
-        end)
+      AccountView.render("show.json", %{user: user, skip_visibility_check: true})
+      |> Enum.all?(fn
+        {key, url} when key in [:avatar, :avatar_static, :header, :header_static] ->
+          String.starts_with?(url, Pleroma.Web.base_url())
 
-      _ ->
-        true
-    end)
-    |> assert()
+        {:emojis, emojis} ->
+          Enum.all?(emojis, fn %{url: url, static_url: static_url} ->
+            String.starts_with?(url, Pleroma.Web.base_url()) &&
+              String.starts_with?(static_url, Pleroma.Web.base_url())
+          end)
+
+        _ ->
+          true
+      end)
+      |> assert()
+    end
   end
 end

test/web/media_proxy/media_proxy_controller_test.exs

@@ -8,34 +8,34 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyControllerTest do
   import Mock
 
   alias Pleroma.Web.MediaProxy
-  alias Pleroma.Web.MediaProxy.MediaProxyController
   alias Plug.Conn
 
   setup do
     on_exit(fn -> Cachex.clear(:banned_urls_cache) end)
   end
 
-  test "it returns 404 when MediaProxy disabled", %{conn: conn} do
-    clear_config([:media_proxy, :enabled], false)
-
-    assert %Conn{
-             status: 404,
-             resp_body: "Not Found"
-           } = get(conn, "/proxy/hhgfh/eeeee")
-
-    assert %Conn{
-             status: 404,
-             resp_body: "Not Found"
-           } = get(conn, "/proxy/hhgfh/eeee/fff")
-  end
-
-  describe "" do
+  describe "Media Proxy" do
     setup do
       clear_config([:media_proxy, :enabled], true)
       clear_config([Pleroma.Web.Endpoint, :secret_key_base], "00000000000")
+
       [url: MediaProxy.encode_url("https://google.fn/test.png")]
     end
 
+    test "it returns 404 when disabled", %{conn: conn} do
+      clear_config([:media_proxy, :enabled], false)
+
+      assert %Conn{
+               status: 404,
+               resp_body: "Not Found"
+             } = get(conn, "/proxy/hhgfh/eeeee")
+
+      assert %Conn{
+               status: 404,
+               resp_body: "Not Found"
+             } = get(conn, "/proxy/hhgfh/eeee/fff")
+    end
+
     test "it returns 403 for invalid signature", %{conn: conn, url: url} do
       Pleroma.Config.put([Pleroma.Web.Endpoint, :secret_key_base], "000")
       %{path: path} = URI.parse(url)
@@ -56,7 +56,7 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyControllerTest do
              } = get(conn, "/proxy/hhgfh/eeee/fff")
     end
 
-    test "redirects on valid url when filename is invalidated", %{conn: conn, url: url} do
+    test "redirects to valid url when filename is invalidated", %{conn: conn, url: url} do
       invalid_url = String.replace(url, "test.png", "test-file.png")
       response = get(conn, invalid_url)
       assert response.status == 302
@@ -80,42 +80,248 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyControllerTest do
     end
   end
 
-  describe "filename_matches/3" do
-    test "preserves the encoded or decoded path" do
-      assert MediaProxyController.filename_matches(
-               %{"filename" => "/Hello world.jpg"},
-               "/Hello world.jpg",
-               "http://pleroma.social/Hello world.jpg"
-             ) == :ok
+  describe "Media Preview Proxy" do
+    setup do
+      clear_config([:media_proxy, :enabled], true)
+      clear_config([:media_preview_proxy, :enabled], true)
+      clear_config([Pleroma.Web.Endpoint, :secret_key_base], "00000000000")
 
-      assert MediaProxyController.filename_matches(
-               %{"filename" => "/Hello%20world.jpg"},
-               "/Hello%20world.jpg",
-               "http://pleroma.social/Hello%20world.jpg"
-             ) == :ok
+      original_url = "https://google.fn/test.png"
 
-      assert MediaProxyController.filename_matches(
-               %{"filename" => "/my%2Flong%2Furl%2F2019%2F07%2FS.jpg"},
-               "/my%2Flong%2Furl%2F2019%2F07%2FS.jpg",
-               "http://pleroma.social/my%2Flong%2Furl%2F2019%2F07%2FS.jpg"
-             ) == :ok
-
-      assert MediaProxyController.filename_matches(
-               %{"filename" => "/my%2Flong%2Furl%2F2019%2F07%2FS.jp"},
-               "/my%2Flong%2Furl%2F2019%2F07%2FS.jp",
-               "http://pleroma.social/my%2Flong%2Furl%2F2019%2F07%2FS.jpg"
-             ) == {:wrong_filename, "my%2Flong%2Furl%2F2019%2F07%2FS.jpg"}
+      [
+        url: MediaProxy.encode_preview_url(original_url),
+        media_proxy_url: MediaProxy.encode_url(original_url)
+      ]
     end
 
-    test "encoded url are tried to match for proxy as `conn.request_path` encodes the url" do
-      # conn.request_path will return encoded url
-      request_path = "/ANALYSE-DAI-_-LE-STABLECOIN-100-D%C3%89CENTRALIS%C3%89-BQ.jpg"
+    test "returns 404 when media proxy is disabled", %{conn: conn} do
+      clear_config([:media_proxy, :enabled], false)
 
-      assert MediaProxyController.filename_matches(
-               true,
-               request_path,
-               "https://mydomain.com/uploads/2019/07/ANALYSE-DAI-_-LE-STABLECOIN-100-DÉCENTRALISÉ-BQ.jpg"
-             ) == :ok
+      assert %Conn{
+               status: 404,
+               resp_body: "Not Found"
+             } = get(conn, "/proxy/preview/hhgfh/eeeee")
+
+      assert %Conn{
+               status: 404,
+               resp_body: "Not Found"
+             } = get(conn, "/proxy/preview/hhgfh/fff")
+    end
+
+    test "returns 404 when disabled", %{conn: conn} do
+      clear_config([:media_preview_proxy, :enabled], false)
+
+      assert %Conn{
+               status: 404,
+               resp_body: "Not Found"
+             } = get(conn, "/proxy/preview/hhgfh/eeeee")
+
+      assert %Conn{
+               status: 404,
+               resp_body: "Not Found"
+             } = get(conn, "/proxy/preview/hhgfh/fff")
+    end
+
+    test "it returns 403 for invalid signature", %{conn: conn, url: url} do
+      Pleroma.Config.put([Pleroma.Web.Endpoint, :secret_key_base], "000")
+      %{path: path} = URI.parse(url)
+
+      assert %Conn{
+               status: 403,
+               resp_body: "Forbidden"
+             } = get(conn, path)
+
+      assert %Conn{
+               status: 403,
+               resp_body: "Forbidden"
+             } = get(conn, "/proxy/preview/hhgfh/eeee")
+
+      assert %Conn{
+               status: 403,
+               resp_body: "Forbidden"
+             } = get(conn, "/proxy/preview/hhgfh/eeee/fff")
+    end
+
+    test "redirects to valid url when filename is invalidated", %{conn: conn, url: url} do
+      invalid_url = String.replace(url, "test.png", "test-file.png")
+      response = get(conn, invalid_url)
+      assert response.status == 302
+      assert redirected_to(response) == url
+    end
+
+    test "responds with 424 Failed Dependency if HEAD request to media proxy fails", %{
+      conn: conn,
+      url: url,
+      media_proxy_url: media_proxy_url
+    } do
+      Tesla.Mock.mock(fn
+        %{method: "head", url: ^media_proxy_url} ->
+          %Tesla.Env{status: 500, body: ""}
+      end)
+
+      response = get(conn, url)
+      assert response.status == 424
+      assert response.resp_body == "Can't fetch HTTP headers (HTTP 500)."
+    end
+
+    test "redirects to media proxy URI on unsupported content type", %{
+      conn: conn,
+      url: url,
+      media_proxy_url: media_proxy_url
+    } do
+      Tesla.Mock.mock(fn
+        %{method: "head", url: ^media_proxy_url} ->
+          %Tesla.Env{status: 200, body: "", headers: [{"content-type", "application/pdf"}]}
+      end)
+
+      response = get(conn, url)
+      assert response.status == 302
+      assert redirected_to(response) == media_proxy_url
+    end
+
+    test "with `static=true` and GIF image preview requested, responds with JPEG image", %{
+      conn: conn,
+      url: url,
+      media_proxy_url: media_proxy_url
+    } do
+      # Setting a high :min_content_length to ensure this scenario is not affected by its logic
+      clear_config([:media_preview_proxy, :min_content_length], 1_000_000_000)
+
+      Tesla.Mock.mock(fn
+        %{method: "head", url: ^media_proxy_url} ->
+          %Tesla.Env{
+            status: 200,
+            body: "",
+            headers: [{"content-type", "image/gif"}, {"content-length", "1001718"}]
+          }
+
+        %{method: :get, url: ^media_proxy_url} ->
+          %Tesla.Env{status: 200, body: File.read!("test/fixtures/image.gif")}
+      end)
+
+      response = get(conn, url <> "?static=true")
+
+      assert response.status == 200
+      assert Conn.get_resp_header(response, "content-type") == ["image/jpeg"]
+      assert response.resp_body != ""
+    end
+
+    test "with GIF image preview requested and no `static` param, redirects to media proxy URI",
+         %{
+           conn: conn,
+           url: url,
+           media_proxy_url: media_proxy_url
+         } do
+      Tesla.Mock.mock(fn
+        %{method: "head", url: ^media_proxy_url} ->
+          %Tesla.Env{status: 200, body: "", headers: [{"content-type", "image/gif"}]}
+      end)
+
+      response = get(conn, url)
+
+      assert response.status == 302
+      assert redirected_to(response) == media_proxy_url
+    end
+
+    test "with `static` param and non-GIF image preview requested, " <>
+           "redirects to media preview proxy URI without `static` param",
+         %{
+           conn: conn,
+           url: url,
+           media_proxy_url: media_proxy_url
+         } do
+      Tesla.Mock.mock(fn
+        %{method: "head", url: ^media_proxy_url} ->
+          %Tesla.Env{status: 200, body: "", headers: [{"content-type", "image/jpeg"}]}
+      end)
+
+      response = get(conn, url <> "?static=true")
+
+      assert response.status == 302
+      assert redirected_to(response) == url
+    end
+
+    test "with :min_content_length setting not matched by Content-Length header, " <>
+           "redirects to media proxy URI",
+         %{
+           conn: conn,
+           url: url,
+           media_proxy_url: media_proxy_url
+         } do
+      clear_config([:media_preview_proxy, :min_content_length], 100_000)
+
+      Tesla.Mock.mock(fn
+        %{method: "head", url: ^media_proxy_url} ->
+          %Tesla.Env{
+            status: 200,
+            body: "",
+            headers: [{"content-type", "image/gif"}, {"content-length", "5000"}]
+          }
+      end)
+
+      response = get(conn, url)
+
+      assert response.status == 302
+      assert redirected_to(response) == media_proxy_url
+    end
+
+    test "thumbnails PNG images into PNG", %{
+      conn: conn,
+      url: url,
+      media_proxy_url: media_proxy_url
+    } do
+      Tesla.Mock.mock(fn
+        %{method: "head", url: ^media_proxy_url} ->
+          %Tesla.Env{status: 200, body: "", headers: [{"content-type", "image/png"}]}
+
+        %{method: :get, url: ^media_proxy_url} ->
+          %Tesla.Env{status: 200, body: File.read!("test/fixtures/image.png")}
+      end)
+
+      response = get(conn, url)
+
+      assert response.status == 200
+      assert Conn.get_resp_header(response, "content-type") == ["image/png"]
+      assert response.resp_body != ""
+    end
+
+    test "thumbnails JPEG images into JPEG", %{
+      conn: conn,
+      url: url,
+      media_proxy_url: media_proxy_url
+    } do
+      Tesla.Mock.mock(fn
+        %{method: "head", url: ^media_proxy_url} ->
+          %Tesla.Env{status: 200, body: "", headers: [{"content-type", "image/jpeg"}]}
+
+        %{method: :get, url: ^media_proxy_url} ->
+          %Tesla.Env{status: 200, body: File.read!("test/fixtures/image.jpg")}
+      end)
+
+      response = get(conn, url)
+
+      assert response.status == 200
+      assert Conn.get_resp_header(response, "content-type") == ["image/jpeg"]
+      assert response.resp_body != ""
+    end
+
+    test "redirects to media proxy URI in case of thumbnailing error", %{
+      conn: conn,
+      url: url,
+      media_proxy_url: media_proxy_url
+    } do
+      Tesla.Mock.mock(fn
+        %{method: "head", url: ^media_proxy_url} ->
+          %Tesla.Env{status: 200, body: "", headers: [{"content-type", "image/jpeg"}]}
+
+        %{method: :get, url: ^media_proxy_url} ->
+          %Tesla.Env{status: 200, body: "<html><body>error</body></html>"}
+      end)
+
+      response = get(conn, url)
+
+      assert response.status == 302
+      assert redirected_to(response) == media_proxy_url
     end
   end
 end

test/web/media_proxy/media_proxy_test.exs

@@ -6,9 +6,16 @@ defmodule Pleroma.Web.MediaProxyTest do
   use ExUnit.Case
   use Pleroma.Tests.Helpers
 
+  alias Pleroma.Config
   alias Pleroma.Web.Endpoint
   alias Pleroma.Web.MediaProxy
 
+  defp decode_result(encoded) do
+    [_, "proxy", sig, base64 | _] = URI.parse(encoded).path |> String.split("/")
+    {:ok, decoded} = MediaProxy.decode_url(sig, base64)
+    decoded
+  end
+
   describe "when enabled" do
     setup do: clear_config([:media_proxy, :enabled], true)
 
@@ -35,7 +42,7 @@ defmodule Pleroma.Web.MediaProxyTest do
 
       assert String.starts_with?(
                encoded,
-               Pleroma.Config.get([:media_proxy, :base_url], Pleroma.Web.base_url())
+               Config.get([:media_proxy, :base_url], Pleroma.Web.base_url())
              )
 
       assert String.ends_with?(encoded, "/logo.png")
@@ -75,6 +82,64 @@ defmodule Pleroma.Web.MediaProxyTest do
       assert MediaProxy.decode_url(sig, base64) == {:error, :invalid_signature}
     end
 
+    def test_verify_request_path_and_url(request_path, url, expected_result) do
+      assert MediaProxy.verify_request_path_and_url(request_path, url) == expected_result
+
+      assert MediaProxy.verify_request_path_and_url(
+               %Plug.Conn{
+                 params: %{"filename" => Path.basename(request_path)},
+                 request_path: request_path
+               },
+               url
+             ) == expected_result
+    end
+
+    test "if first arg of `verify_request_path_and_url/2` is a Plug.Conn without \"filename\" " <>
+           "parameter, `verify_request_path_and_url/2` returns :ok " do
+      assert MediaProxy.verify_request_path_and_url(
+               %Plug.Conn{params: %{}, request_path: "/some/path"},
+               "https://instance.com/file.jpg"
+             ) == :ok
+
+      assert MediaProxy.verify_request_path_and_url(
+               %Plug.Conn{params: %{}, request_path: "/path/to/file.jpg"},
+               "https://instance.com/file.jpg"
+             ) == :ok
+    end
+
+    test "`verify_request_path_and_url/2` preserves the encoded or decoded path" do
+      test_verify_request_path_and_url(
+        "/Hello world.jpg",
+        "http://pleroma.social/Hello world.jpg",
+        :ok
+      )
+
+      test_verify_request_path_and_url(
+        "/Hello%20world.jpg",
+        "http://pleroma.social/Hello%20world.jpg",
+        :ok
+      )
+
+      test_verify_request_path_and_url(
+        "/my%2Flong%2Furl%2F2019%2F07%2FS.jpg",
+        "http://pleroma.social/my%2Flong%2Furl%2F2019%2F07%2FS.jpg",
+        :ok
+      )
+
+      test_verify_request_path_and_url(
+        # Note: `conn.request_path` returns encoded url
+        "/ANALYSE-DAI-_-LE-STABLECOIN-100-D%C3%89CENTRALIS%C3%89-BQ.jpg",
+        "https://mydomain.com/uploads/2019/07/ANALYSE-DAI-_-LE-STABLECOIN-100-DÉCENTRALISÉ-BQ.jpg",
+        :ok
+      )
+
+      test_verify_request_path_and_url(
+        "/my%2Flong%2Furl%2F2019%2F07%2FS",
+        "http://pleroma.social/my%2Flong%2Furl%2F2019%2F07%2FS.jpg",
+        {:wrong_filename, "my%2Flong%2Furl%2F2019%2F07%2FS.jpg"}
+      )
+    end
+
     test "uses the configured base_url" do
       base_url = "https://cache.pleroma.social"
       clear_config([:media_proxy, :base_url], base_url)
@@ -124,12 +189,6 @@ defmodule Pleroma.Web.MediaProxyTest do
     end
   end
 
-  defp decode_result(encoded) do
-    [_, "proxy", sig, base64 | _] = URI.parse(encoded).path |> String.split("/")
-    {:ok, decoded} = MediaProxy.decode_url(sig, base64)
-    decoded
-  end
-
   describe "whitelist" do
     setup do: clear_config([:media_proxy, :enabled], true)