diff --git a/changelog.d/max-body-hackney.fix b/changelog.d/max-body-hackney.fix new file mode 100644 index 000000000..cc41a617b --- /dev/null +++ b/changelog.d/max-body-hackney.fix @@ -0,0 +1 @@ +Enforce the max_body HTTP option again, which Hackney 4 silently stopped honouring because it always returns a fully read body diff --git a/lib/pleroma/http.ex b/lib/pleroma/http.ex index bdeb2171e..c230cea46 100644 --- a/lib/pleroma/http.ex +++ b/lib/pleroma/http.ex @@ -130,5 +130,9 @@ defmodule Pleroma.HTTP do end defp default_middleware, - do: [Tesla.Middleware.FollowRedirects, Pleroma.Tesla.Middleware.EncodeUrl] + do: [ + Tesla.Middleware.FollowRedirects, + Pleroma.Tesla.Middleware.EncodeUrl, + Pleroma.Tesla.Middleware.MaxBody + ] end diff --git a/lib/pleroma/tesla/middleware/max_body.ex b/lib/pleroma/tesla/middleware/max_body.ex new file mode 100644 index 000000000..7e74f0b56 --- /dev/null +++ b/lib/pleroma/tesla/middleware/max_body.ex @@ -0,0 +1,72 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Tesla.Middleware.MaxBody do + @moduledoc """ + Enforces the `:max_body` adapter option for every adapter. + + `Pleroma.HTTP.Adapter.Gun` applies the limit as it accumulates a `:plain` + body, so an oversized response is abandoned mid-transfer. Hackney cannot: + since 4.0 it ignores `with_body` and always hands back a fully read binary + (`hackney:request/5`), so `Tesla.Adapter.Hackney` never takes the branch that + would honour `:max_body`. Without this middleware the option is silently + unenforced there. + + The check therefore bounds what callers are handed, and refuses up front when + `content-length` already announces too much. It cannot bound peak memory on + the hackney path, because the adapter has read the body before any middleware + runs. + + Streaming bodies are left alone: neither adapter limits those, and the caller + that owns the stream is the one that has to drain and release it. + """ + + @behaviour Tesla.Middleware + + @impl Tesla.Middleware + def call(%Tesla.Env{opts: opts} = env, next, _opts) do + case opts[:adapter][:max_body] do + limit when is_integer(limit) and limit >= 0 -> run(env, next, limit) + _ -> Tesla.run(env, next) + end + end + + defp run(env, next, limit) do + with {:ok, env} <- Tesla.run(env, next), + :ok <- check(env, limit) do + {:ok, env} + end + end + + defp check(%Tesla.Env{body: body} = env, limit) when is_binary(body) do + with :ok <- check_content_length(env, limit) do + check_body(body, limit) + end + end + + # body_as: :chunks and response: :stream hand back a client the caller drains + # itself, applying its own limit (Pleroma.Web.RichMedia.Helpers). There is + # nothing to measure, and failing on the header alone would strand the pool + # lease that only the caller holds a handle to. + defp check(_env, _limit), do: :ok + + # A bodyless response may still announce the entity's size. + defp check_content_length(%Tesla.Env{method: :head}, _limit), do: :ok + + defp check_content_length(%Tesla.Env{status: status}, _limit) when status in [204, 304], + do: :ok + + defp check_content_length(env, limit) do + with value when is_binary(value) <- Tesla.get_header(env, "content-length"), + {length, ""} <- Integer.parse(value) do + if length > limit, do: {:error, :body_too_large}, else: :ok + else + _ -> :ok + end + end + + defp check_body(body, limit) do + if byte_size(body) > limit, do: {:error, :body_too_large}, else: :ok + end +end diff --git a/test/pleroma/http/hackney_max_body_regression_test.exs b/test/pleroma/http/hackney_max_body_regression_test.exs new file mode 100644 index 000000000..4e533c53f --- /dev/null +++ b/test/pleroma/http/hackney_max_body_regression_test.exs @@ -0,0 +1,165 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +# Hackney 4 ignores `with_body` and always returns a fully read binary, so +# `Tesla.Adapter.Hackney` never reaches the clause that honours `:max_body`. +# The limit is enforced by Pleroma.Tesla.Middleware.MaxBody instead; these +# exercise it against the real adapter. +defmodule Pleroma.HTTP.HackneyMaxBodyRegressionTest do + use ExUnit.Case, async: false + + alias Pleroma.HTTP.AdapterHelper.Hackney, as: HackneyAdapterHelper + + @body_bytes 64 * 1024 + + setup do + {:ok, _} = Application.ensure_all_started(:hackney) + + {:ok, server} = start_server() + on_exit(fn -> stop_server(server) end) + + {:ok, server: server} + end + + defp request(url, extra_opts) do + adapter_opts = + HackneyAdapterHelper.options( + [pool: :media, no_proxy_env: true] ++ extra_opts, + URI.parse(url) + ) + + client = + Tesla.client( + [Tesla.Middleware.FollowRedirects, Pleroma.Tesla.Middleware.MaxBody], + Tesla.Adapter.Hackney + ) + + Tesla.request(client, method: :get, url: url, opts: [adapter: adapter_opts]) + end + + test "a body over max_body is refused", %{server: server} do + assert {:error, :body_too_large} = + request("#{server.base_url}/big", max_body: 1024) + end + + # Without a Content-Length the header pre-check cannot fire, so this is the + # case that proves hackney's fully-read binary is measured. + test "a chunked body over max_body is refused on size alone", %{server: server} do + assert {:error, :body_too_large} = + request("#{server.base_url}/chunked", max_body: 1024) + end + + test "a body under max_body is returned", %{server: server} do + assert {:ok, %Tesla.Env{status: 200, body: body}} = + request("#{server.base_url}/big", max_body: @body_bytes * 2) + + assert byte_size(body) == @body_bytes + end + + test "without max_body the body is returned whatever its size", %{server: server} do + assert {:ok, %Tesla.Env{status: 200, body: body}} = request("#{server.base_url}/big", []) + assert byte_size(body) == @body_bytes + end + + # Pins why the middleware is needed. Tesla.Adapter.Hackney only honours + # :max_body on the branch where hackney hands back a ref, which 4.x never + # does for a synchronous request. Should this ever start failing, hackney or + # Tesla have regained streaming and the middleware can go. + test "the bare adapter does not enforce max_body", %{server: server} do + url = "#{server.base_url}/big" + + adapter_opts = + HackneyAdapterHelper.options( + [pool: :media, no_proxy_env: true, max_body: 1024], + URI.parse(url) + ) + + client = Tesla.client([], Tesla.Adapter.Hackney) + + assert {:ok, %Tesla.Env{status: 200, body: body}} = + Tesla.request(client, method: :get, url: url, opts: [adapter: adapter_opts]) + + assert byte_size(body) == @body_bytes + end + + defp start_server do + {:ok, listener} = + :gen_tcp.listen(0, [ + :binary, + active: false, + packet: :raw, + reuseaddr: true, + ip: {127, 0, 0, 1} + ]) + + {:ok, {{127, 0, 0, 1}, port}} = :inet.sockname(listener) + {:ok, acceptor} = Task.start(fn -> accept_loop(listener) end) + + {:ok, %{listener: listener, acceptor: acceptor, base_url: "http://127.0.0.1:#{port}"}} + end + + # Closing the listener makes the blocked accept/1 return {:error, :closed}, + # which ends the loop. + defp stop_server(%{listener: listener, acceptor: acceptor}) do + :ok = :gen_tcp.close(listener) + if Process.alive?(acceptor), do: Process.exit(acceptor, :kill) + end + + defp accept_loop(listener) do + case :gen_tcp.accept(listener) do + {:ok, socket} -> + serve(socket) + accept_loop(listener) + + {:error, _} -> + :ok + end + end + + defp serve(socket) do + with {:ok, data} <- recv_headers(socket) do + body = :binary.copy("x", @body_bytes) + + if String.contains?(data, "/chunked") do + # No Content-Length, so only the body-size check can catch this. + :gen_tcp.send(socket, [ + "HTTP/1.1 200 OK\r\n", + "Content-Type: text/plain\r\n", + "Transfer-Encoding: chunked\r\n", + "Connection: close\r\n\r\n", + :erlang.integer_to_list(byte_size(body), 16), + "\r\n", + body, + "\r\n0\r\n\r\n" + ]) + else + :gen_tcp.send(socket, [ + "HTTP/1.1 200 OK\r\n", + "Content-Type: text/plain\r\n", + "Content-Length: #{byte_size(body)}\r\n", + "Connection: close\r\n\r\n", + body + ]) + end + end + + :gen_tcp.close(socket) + end + + defp recv_headers(socket, acc \\ <<>>) do + case :gen_tcp.recv(socket, 0, 1_000) do + {:ok, data} -> + acc = acc <> data + + cond do + :binary.match(acc, "\r\n\r\n") != :nomatch -> {:ok, acc} + byte_size(acc) > 8_192 -> {:error, :too_large} + true -> recv_headers(socket, acc) + end + + {:error, _} = error -> + error + end + end +end diff --git a/test/pleroma/tesla/middleware/max_body_test.exs b/test/pleroma/tesla/middleware/max_body_test.exs new file mode 100644 index 000000000..a12e6c8ca --- /dev/null +++ b/test/pleroma/tesla/middleware/max_body_test.exs @@ -0,0 +1,87 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Tesla.Middleware.MaxBodyTest do + use ExUnit.Case, async: true + + alias Pleroma.Tesla.Middleware.MaxBody + + defp client(adapter_opts, response) do + Tesla.client([{MaxBody, nil}], fn env -> + {:ok, %{env | status: 200, headers: response.headers, body: response.body}} + end) + |> then(fn c -> {c, [opts: [adapter: adapter_opts]]} end) + end + + defp get(adapter_opts, response) do + {c, opts} = client(adapter_opts, response) + Tesla.get(c, "http://example.com/", opts) + end + + test "passes a body within the limit through untouched" do + assert {:ok, %Tesla.Env{status: 200, body: "hello"}} = + get([max_body: 100], %{headers: [], body: "hello"}) + end + + test "rejects a body larger than the limit" do + assert {:error, :body_too_large} = + get([max_body: 4], %{headers: [], body: "hello"}) + end + + test "rejects when content-length exceeds the limit" do + assert {:error, :body_too_large} = + get([max_body: 4], %{headers: [{"content-length", "1000"}], body: ""}) + end + + test "allows a content-length equal to the limit" do + assert {:ok, %Tesla.Env{status: 200}} = + get([max_body: 5], %{headers: [{"content-length", "5"}], body: "hello"}) + end + + test "ignores an unparsable content-length" do + assert {:ok, %Tesla.Env{status: 200}} = + get([max_body: 100], %{headers: [{"content-length", "banana"}], body: "hi"}) + end + + test "does nothing without a max_body option" do + big = String.duplicate("x", 10_000) + + assert {:ok, %Tesla.Env{body: ^big}} = + get([], %{headers: [{"content-length", "10000"}], body: big}) + end + + # body_as: :chunks hands back a client map, not a binary; the caller streams + # it and enforces its own limit (see Pleroma.Web.RichMedia.Helpers). + test "leaves a non-binary streaming body alone" do + body = %{pid: self(), stream: make_ref()} + + assert {:ok, %Tesla.Env{body: ^body}} = + get([max_body: 1], %{headers: [], body: body}) + end + + # Rejecting a stream on the header alone would strand the connection lease: + # the env carrying the pid/stream is the caller's only handle to release it. + test "does not reject a streaming body on content-length alone" do + body = %{pid: self(), stream: make_ref()} + + assert {:ok, %Tesla.Env{body: ^body}} = + get([max_body: 1], %{headers: [{"content-length", "20000000"}], body: body}) + end + + test "ignores content-length on a HEAD response" do + {c, opts} = client([max_body: 4], %{headers: [{"content-length", "9000000"}], body: ""}) + + assert {:ok, %Tesla.Env{status: 200}} = Tesla.head(c, "http://example.com/", opts) + end + + test "ignores content-length on a bodyless status" do + client = + Tesla.client([{MaxBody, nil}], fn env -> + {:ok, %{env | status: 304, headers: [{"content-length", "9000000"}], body: ""}} + end) + + assert {:ok, %Tesla.Env{status: 304}} = + Tesla.get(client, "http://example.com/", opts: [adapter: [max_body: 4]]) + end +end