24bdd9774aThree paths obtained a client ref and abandoned it without closing: the content-length rejection in call/3, the invalid-status clause in request/4, and the success branch of response/6. Under Hackney 4 an abandoned ref is a permanent process leak. A conn that finishes or errors without an explicit stop parks in `closed` or `connected` and never terminates: closed/3 only arms its stop timer for pooled conns, `connected` gets no state_timeout when idle_timeout is infinity, and a non-pooled conn monitors hackney_conn_sup rather than its caller, so the owner-DOWN cleanup never fires. Each parked process pins every refc binary it read, which for proxied video is tens of MB. Releasing on every outcome means the gun client must stop releasing in stream_body/1, or the ref would be removed twice. Hackney does reclaim a pooled connection itself for a response that cannot carry a body (204, 304, HEAD), so those refs are marked and left alone; stopping one would evict a connection another request may already hold. The mark is gated on the request actually being pooled, since a direct or proxy-tunnelled conn is never checked in and leaks unless we stop it.
586 lines
18 KiB
Elixir
586 lines
18 KiB
Elixir
# Pleroma: A lightweight social networking server
|
|
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
|
|
# SPDX-License-Identifier: AGPL-3.0-only
|
|
|
|
defmodule Pleroma.ReverseProxyTest do
|
|
use Pleroma.Web.ConnCase
|
|
import ExUnit.CaptureLog
|
|
import Mox
|
|
|
|
alias Pleroma.ReverseProxy
|
|
alias Pleroma.ReverseProxy.ClientMock
|
|
alias Plug.Conn
|
|
|
|
setup_all do
|
|
{:ok, _} = Registry.start_link(keys: :unique, name: ClientMock)
|
|
:ok
|
|
end
|
|
|
|
setup :verify_on_exit!
|
|
|
|
# Every path that obtains a client ref releases it, so tests that do not assert
|
|
# on the release still need close/1 to be callable. Recording each call lets
|
|
# the tests below pin *exactly* once: a Mox `expect` would silently fall
|
|
# through to this stub on a second call.
|
|
setup do
|
|
test_pid = self()
|
|
|
|
stub(ClientMock, :close, fn client ->
|
|
send(test_pid, {:closed, client})
|
|
:ok
|
|
end)
|
|
|
|
:ok
|
|
end
|
|
|
|
defp request_mock(invokes) do
|
|
ClientMock
|
|
|> expect(:request, fn :get, url, headers, _body, _opts ->
|
|
Registry.register(ClientMock, url, 0)
|
|
body = headers |> Enum.into(%{}) |> Jason.encode!()
|
|
|
|
{:ok, 200,
|
|
[
|
|
{"content-type", "application/json"},
|
|
{"content-length", byte_size(body) |> to_string()}
|
|
], %{url: url, body: body}}
|
|
end)
|
|
|> expect(:stream_body, invokes, fn %{url: url, body: body} = client ->
|
|
case Registry.lookup(ClientMock, url) do
|
|
[{_, 0}] ->
|
|
Registry.update_value(ClientMock, url, &(&1 + 1))
|
|
{:ok, body, client}
|
|
|
|
[{_, 1}] ->
|
|
Registry.unregister(ClientMock, url)
|
|
:done
|
|
end
|
|
end)
|
|
end
|
|
|
|
describe "reverse proxy" do
|
|
test "do not track successful request", %{conn: conn} do
|
|
request_mock(2)
|
|
url = "/success"
|
|
|
|
conn = ReverseProxy.call(conn, url)
|
|
|
|
assert conn.status == 200
|
|
assert Cachex.get(:failed_proxy_url_cache, url) == {:ok, nil}
|
|
end
|
|
|
|
test "uses max_read_duration as the default HTTP receive timeout", %{conn: conn} do
|
|
ClientMock
|
|
|> expect(:request, fn :get, "/timeout", _, _, opts ->
|
|
assert opts[:recv_timeout] == 123
|
|
{:error, :timeout}
|
|
end)
|
|
|
|
capture_log(fn -> ReverseProxy.call(conn, "/timeout", max_read_duration: 123) end)
|
|
end
|
|
|
|
test "preserves an explicit HTTP receive timeout", %{conn: conn} do
|
|
ClientMock
|
|
|> expect(:request, fn :get, "/timeout", _, _, opts ->
|
|
assert opts[:recv_timeout] == 456
|
|
{:error, :timeout}
|
|
end)
|
|
|
|
capture_log(fn ->
|
|
ReverseProxy.call(conn, "/timeout", max_read_duration: 123, http: [recv_timeout: 456])
|
|
end)
|
|
end
|
|
|
|
test "keeps unlimited read durations unlimited", %{conn: conn} do
|
|
for {max_read_duration, url} <- [{0, "/zero-timeout"}, {:infinity, "/infinite-timeout"}] do
|
|
ClientMock
|
|
|> expect(:request, fn :get, ^url, _, _, opts ->
|
|
assert opts[:recv_timeout] == :infinity
|
|
{:error, :timeout}
|
|
end)
|
|
|
|
capture_log(fn ->
|
|
ReverseProxy.call(conn, url, max_read_duration: max_read_duration)
|
|
end)
|
|
end
|
|
end
|
|
end
|
|
|
|
# The connection behind a client ref is not tied to the lifetime of this
|
|
# request. Hackney 4 in particular parks a non-pooled `hackney_conn` process
|
|
# in the `closed` state forever when nobody stops it, pinning every refc
|
|
# binary it read, so abandoning a ref leaks the whole response body.
|
|
describe "client ref release" do
|
|
test "closes the client after the body is fully streamed", %{conn: conn} do
|
|
ClientMock
|
|
|> expect(:request, fn :get, "/release", _, _, _ ->
|
|
{:ok, 200, [{"content-type", "image/png"}], %{url: "/release"}}
|
|
end)
|
|
|> expect(:stream_body, fn _ -> :done end)
|
|
|
|
conn = ReverseProxy.call(conn, "/release")
|
|
|
|
assert conn.halted
|
|
assert_received {:closed, %{url: "/release"}}
|
|
refute_received {:closed, _}
|
|
end
|
|
|
|
test "closes the client when content-length exceeds max_body_length", %{conn: conn} do
|
|
ClientMock
|
|
|> expect(:request, fn :get, "/release-huge", _, _, _ ->
|
|
{:ok, 200, [{"content-length", "102227307"}], %{url: "/release-huge"}}
|
|
end)
|
|
|
|
assert capture_log(fn ->
|
|
ReverseProxy.call(conn, "/release-huge", max_body_length: 25 * 1_048_576)
|
|
end) =~ ":body_too_large"
|
|
|
|
assert_received {:closed, %{url: "/release-huge"}}
|
|
refute_received {:closed, _}
|
|
end
|
|
|
|
test "closes the client when the upstream status is not proxiable", %{conn: conn} do
|
|
ClientMock
|
|
|> expect(:request, fn :get, "/release-418", _, _, _ ->
|
|
{:ok, 418, [], %{url: "/release-418"}}
|
|
end)
|
|
|
|
assert capture_log(fn -> ReverseProxy.call(conn, "/release-418") end) =~
|
|
"failed with HTTP status 418"
|
|
|
|
assert_received {:closed, %{url: "/release-418"}}
|
|
refute_received {:closed, _}
|
|
end
|
|
|
|
# The hackney client answers HEAD with a ref too, so this path must release
|
|
# it as well; only the ref-less 3-tuple reply has nothing to close.
|
|
test "closes the client when a HEAD response comes back with a ref", %{conn: conn} do
|
|
ClientMock
|
|
|> expect(:request, fn :head, "/release-head", _, _, _ ->
|
|
{:ok, 200, [{"content-type", "image/png"}], %{url: "/release-head"}}
|
|
end)
|
|
|> expect(:stream_body, fn _ -> :done end)
|
|
|
|
conn = ReverseProxy.call(Map.put(conn, :method, "HEAD"), "/release-head")
|
|
|
|
assert conn.halted
|
|
assert_received {:closed, %{url: "/release-head"}}
|
|
refute_received {:closed, _}
|
|
end
|
|
|
|
test "has nothing to close when the reply carries no ref", %{conn: conn} do
|
|
ClientMock
|
|
|> expect(:request, fn :head, "/release-headless", _, _, _ ->
|
|
{:ok, 200, [{"content-type", "image/png"}]}
|
|
end)
|
|
|
|
conn = ReverseProxy.call(Map.put(conn, :method, "HEAD"), "/release-headless")
|
|
|
|
assert conn.status == 200
|
|
refute_received {:closed, _}
|
|
end
|
|
end
|
|
|
|
test "use Pleroma's user agent in the request; don't pass the client's", %{conn: conn} do
|
|
request_mock(2)
|
|
|
|
conn =
|
|
conn
|
|
|> Plug.Conn.put_req_header("user-agent", "fake/1.0")
|
|
|> ReverseProxy.call("/user-agent")
|
|
|
|
# Convert the response to a map without relying on json_response
|
|
body = conn.resp_body
|
|
assert conn.status == 200
|
|
response = Jason.decode!(body)
|
|
assert response == %{"user-agent" => Pleroma.Application.user_agent()}
|
|
end
|
|
|
|
test "closed connection", %{conn: conn} do
|
|
ClientMock
|
|
|> expect(:request, fn :get, "/closed", _, _, _ -> {:ok, 200, [], %{}} end)
|
|
|> expect(:stream_body, fn _ -> {:error, :closed} end)
|
|
|> expect(:close, fn _ -> :ok end)
|
|
|
|
conn = ReverseProxy.call(conn, "/closed")
|
|
assert conn.halted
|
|
end
|
|
|
|
defp stream_mock(invokes) do
|
|
ClientMock
|
|
|> expect(:request, fn :get, "/stream-bytes/" <> length, _, _, _ ->
|
|
Registry.register(ClientMock, "/stream-bytes/" <> length, 0)
|
|
|
|
{:ok, 200, [{"content-type", "application/octet-stream"}],
|
|
%{url: "/stream-bytes/" <> length}}
|
|
end)
|
|
|> expect(:stream_body, invokes, fn %{url: "/stream-bytes/" <> length} = client ->
|
|
max = String.to_integer(length)
|
|
|
|
case Registry.lookup(ClientMock, "/stream-bytes/" <> length) do
|
|
[{_, current}] when current < max ->
|
|
Registry.update_value(
|
|
ClientMock,
|
|
"/stream-bytes/" <> length,
|
|
&(&1 + 10)
|
|
)
|
|
|
|
{:ok, "0123456789", client}
|
|
|
|
[{_, ^max}] ->
|
|
Registry.unregister(ClientMock, "/stream-bytes/" <> length)
|
|
:done
|
|
end
|
|
end)
|
|
end
|
|
|
|
describe "max_body" do
|
|
test "length returns error if content-length more than option", %{conn: conn} do
|
|
request_mock(0)
|
|
|
|
assert capture_log(fn ->
|
|
ReverseProxy.call(conn, "/huge-file", max_body_length: 4)
|
|
end) =~
|
|
"[error] Elixir.Pleroma.ReverseProxy: request to \"/huge-file\" failed: :body_too_large"
|
|
|
|
assert {:ok, true} == Cachex.get(:failed_proxy_url_cache, "/huge-file")
|
|
|
|
assert capture_log(fn ->
|
|
ReverseProxy.call(conn, "/huge-file", max_body_length: 4)
|
|
end) == ""
|
|
end
|
|
|
|
test "max_body_length returns error if streaming body more than that option", %{conn: conn} do
|
|
stream_mock(3)
|
|
|
|
assert capture_log(fn ->
|
|
ReverseProxy.call(conn, "/stream-bytes/50", max_body_length: 30)
|
|
end) =~
|
|
"Elixir.Pleroma.ReverseProxy request to /stream-bytes/50 failed while reading/chunking: :body_too_large"
|
|
|
|
assert_received {:closed, %{url: "/stream-bytes/50"}}
|
|
refute_received {:closed, _}
|
|
end
|
|
end
|
|
|
|
describe "HEAD requests" do
|
|
test "common", %{conn: conn} do
|
|
ClientMock
|
|
|> expect(:request, fn :head, "/head", _, _, _ ->
|
|
{:ok, 200, [{"content-type", "image/png"}]}
|
|
end)
|
|
|
|
conn = ReverseProxy.call(Map.put(conn, :method, "HEAD"), "/head")
|
|
|
|
assert conn.status == 200
|
|
assert Conn.get_resp_header(conn, "content-type") == ["image/png"]
|
|
assert conn.resp_body == ""
|
|
end
|
|
end
|
|
|
|
defp error_mock(status) when is_integer(status) do
|
|
ClientMock
|
|
|> expect(:request, fn :get, "/status/" <> _, _, _, _ ->
|
|
{:error, status}
|
|
end)
|
|
end
|
|
|
|
describe "returns error on" do
|
|
test "500", %{conn: conn} do
|
|
error_mock(500)
|
|
url = "/status/500"
|
|
|
|
capture_log(fn -> ReverseProxy.call(conn, url) end) =~
|
|
"[error] Elixir.Pleroma.ReverseProxy: request to /status/500 failed with HTTP status 500"
|
|
|
|
assert Cachex.get(:failed_proxy_url_cache, url) == {:ok, true}
|
|
|
|
{:ok, ttl} = Cachex.ttl(:failed_proxy_url_cache, url)
|
|
assert ttl <= 60_000
|
|
end
|
|
|
|
test "400", %{conn: conn} do
|
|
error_mock(400)
|
|
url = "/status/400"
|
|
|
|
capture_log(fn -> ReverseProxy.call(conn, url) end) =~
|
|
"[error] Elixir.Pleroma.ReverseProxy: request to /status/400 failed with HTTP status 400"
|
|
|
|
assert Cachex.get(:failed_proxy_url_cache, url) == {:ok, true}
|
|
assert Cachex.ttl(:failed_proxy_url_cache, url) == {:ok, nil}
|
|
end
|
|
|
|
test "403", %{conn: conn} do
|
|
error_mock(403)
|
|
url = "/status/403"
|
|
|
|
capture_log(fn ->
|
|
ReverseProxy.call(conn, url, failed_request_ttl: :timer.seconds(120))
|
|
end) =~
|
|
"[error] Elixir.Pleroma.ReverseProxy: request to /status/403 failed with HTTP status 403"
|
|
|
|
{:ok, ttl} = Cachex.ttl(:failed_proxy_url_cache, url)
|
|
assert ttl > 100_000
|
|
end
|
|
|
|
test "204", %{conn: conn} do
|
|
url = "/status/204"
|
|
expect(ClientMock, :request, fn :get, _url, _, _, _ -> {:ok, 204, [], %{}} end)
|
|
|
|
capture_log(fn ->
|
|
conn = ReverseProxy.call(conn, url)
|
|
assert conn.resp_body == "Request failed: No Content"
|
|
assert conn.halted
|
|
end) =~
|
|
"[error] Elixir.Pleroma.ReverseProxy: request to \"/status/204\" failed with HTTP status 204"
|
|
|
|
assert Cachex.get(:failed_proxy_url_cache, url) == {:ok, true}
|
|
assert Cachex.ttl(:failed_proxy_url_cache, url) == {:ok, nil}
|
|
end
|
|
end
|
|
|
|
test "streaming", %{conn: conn} do
|
|
stream_mock(21)
|
|
conn = ReverseProxy.call(conn, "/stream-bytes/200")
|
|
assert conn.state == :chunked
|
|
assert byte_size(conn.resp_body) == 200
|
|
assert Conn.get_resp_header(conn, "content-type") == ["application/octet-stream"]
|
|
end
|
|
|
|
defp headers_mock(_) do
|
|
ClientMock
|
|
|> expect(:request, fn :get, "/headers", headers, _, _ ->
|
|
Registry.register(ClientMock, "/headers", 0)
|
|
{:ok, 200, [{"content-type", "application/json"}], %{url: "/headers", headers: headers}}
|
|
end)
|
|
|> expect(:stream_body, 2, fn %{url: url, headers: headers} = client ->
|
|
case Registry.lookup(ClientMock, url) do
|
|
[{_, 0}] ->
|
|
Registry.update_value(ClientMock, url, &(&1 + 1))
|
|
headers = for {k, v} <- headers, into: %{}, do: {String.capitalize(k), v}
|
|
{:ok, Jason.encode!(%{headers: headers}), client}
|
|
|
|
[{_, 1}] ->
|
|
Registry.unregister(ClientMock, url)
|
|
:done
|
|
end
|
|
end)
|
|
|
|
:ok
|
|
end
|
|
|
|
describe "keep request headers" do
|
|
setup [:headers_mock]
|
|
|
|
test "header passes", %{conn: conn} do
|
|
conn =
|
|
Conn.put_req_header(
|
|
conn,
|
|
"accept",
|
|
"text/html"
|
|
)
|
|
|> ReverseProxy.call("/headers")
|
|
|
|
body = conn.resp_body
|
|
assert conn.status == 200
|
|
response = Jason.decode!(body)
|
|
headers = response["headers"]
|
|
assert headers["Accept"] == "text/html"
|
|
end
|
|
|
|
test "header is filtered", %{conn: conn} do
|
|
conn =
|
|
Conn.put_req_header(
|
|
conn,
|
|
"accept-language",
|
|
"en-US"
|
|
)
|
|
|> ReverseProxy.call("/headers")
|
|
|
|
body = conn.resp_body
|
|
assert conn.status == 200
|
|
response = Jason.decode!(body)
|
|
headers = response["headers"]
|
|
refute headers["Accept-Language"]
|
|
end
|
|
end
|
|
|
|
test "returns 400 on non GET, HEAD requests", %{conn: conn} do
|
|
conn = ReverseProxy.call(Map.put(conn, :method, "POST"), "/ip")
|
|
assert conn.status == 400
|
|
end
|
|
|
|
describe "cache resp headers" do
|
|
test "add cache-control", %{conn: conn} do
|
|
ClientMock
|
|
|> expect(:request, fn :get, "/cache", _, _, _ ->
|
|
{:ok, 200, [{"ETag", "some ETag"}], %{}}
|
|
end)
|
|
|> expect(:stream_body, fn _ -> :done end)
|
|
|
|
conn = ReverseProxy.call(conn, "/cache")
|
|
assert {"cache-control", "public, max-age=1209600, immutable"} in conn.resp_headers
|
|
end
|
|
end
|
|
|
|
defp disposition_headers_mock(headers) do
|
|
ClientMock
|
|
|> expect(:request, fn :get, "/disposition", _, _, _ ->
|
|
Registry.register(ClientMock, "/disposition", 0)
|
|
|
|
{:ok, 200, headers, %{url: "/disposition"}}
|
|
end)
|
|
|> expect(:stream_body, 2, fn %{url: "/disposition"} = client ->
|
|
case Registry.lookup(ClientMock, "/disposition") do
|
|
[{_, 0}] ->
|
|
Registry.update_value(ClientMock, "/disposition", &(&1 + 1))
|
|
{:ok, "", client}
|
|
|
|
[{_, 1}] ->
|
|
Registry.unregister(ClientMock, "/disposition")
|
|
:done
|
|
end
|
|
end)
|
|
end
|
|
|
|
describe "response content disposition header" do
|
|
test "not attachment", %{conn: conn} do
|
|
disposition_headers_mock([
|
|
{"content-type", "image/gif"},
|
|
{"content-length", "0"}
|
|
])
|
|
|
|
conn = ReverseProxy.call(conn, "/disposition")
|
|
|
|
assert {"content-type", "image/gif"} in conn.resp_headers
|
|
end
|
|
|
|
test "with content-disposition header", %{conn: conn} do
|
|
disposition_headers_mock([
|
|
{"content-disposition", "attachment; filename=\"filename.jpg\""},
|
|
{"content-length", "0"}
|
|
])
|
|
|
|
conn = ReverseProxy.call(conn, "/disposition")
|
|
|
|
assert {"content-disposition", "attachment; filename=\"filename.jpg\""} in conn.resp_headers
|
|
end
|
|
end
|
|
|
|
describe "content-type sanitisation" do
|
|
test "preserves allowed image type", %{conn: conn} do
|
|
ClientMock
|
|
|> expect(:request, fn :get, "/content", _, _, _ ->
|
|
{:ok, 200, [{"content-type", "image/png"}], %{url: "/content"}}
|
|
end)
|
|
|> expect(:stream_body, fn _ -> :done end)
|
|
|
|
conn = ReverseProxy.call(conn, "/content")
|
|
|
|
assert conn.status == 200
|
|
assert Conn.get_resp_header(conn, "content-type") == ["image/png"]
|
|
end
|
|
|
|
test "preserves allowed video type", %{conn: conn} do
|
|
ClientMock
|
|
|> expect(:request, fn :get, "/content", _, _, _ ->
|
|
{:ok, 200, [{"content-type", "video/mp4"}], %{url: "/content"}}
|
|
end)
|
|
|> expect(:stream_body, fn _ -> :done end)
|
|
|
|
conn = ReverseProxy.call(conn, "/content")
|
|
|
|
assert conn.status == 200
|
|
assert Conn.get_resp_header(conn, "content-type") == ["video/mp4"]
|
|
end
|
|
|
|
test "sanitizes ActivityPub content type", %{conn: conn} do
|
|
ClientMock
|
|
|> expect(:request, fn :get, "/content", _, _, _ ->
|
|
{:ok, 200, [{"content-type", "application/activity+json"}], %{url: "/content"}}
|
|
end)
|
|
|> expect(:stream_body, fn _ -> :done end)
|
|
|
|
conn = ReverseProxy.call(conn, "/content")
|
|
|
|
assert conn.status == 200
|
|
assert Conn.get_resp_header(conn, "content-type") == ["application/octet-stream"]
|
|
end
|
|
|
|
test "sanitizes LD-JSON content type", %{conn: conn} do
|
|
ClientMock
|
|
|> expect(:request, fn :get, "/content", _, _, _ ->
|
|
{:ok, 200, [{"content-type", "application/ld+json"}], %{url: "/content"}}
|
|
end)
|
|
|> expect(:stream_body, fn _ -> :done end)
|
|
|
|
conn = ReverseProxy.call(conn, "/content")
|
|
|
|
assert conn.status == 200
|
|
assert Conn.get_resp_header(conn, "content-type") == ["application/octet-stream"]
|
|
end
|
|
end
|
|
|
|
# Hackney is used for Reverse Proxy when Hackney or Finch is the Tesla Adapter
|
|
# Gun is able to proxy through Tesla, so it does not need testing as the
|
|
# test cases in the Pleroma.HTTPTest module are sufficient
|
|
describe "Hackney URL encoding:" do
|
|
setup do
|
|
ClientMock
|
|
|> expect(:request, fn
|
|
:get,
|
|
"https://example.com/emoji/Pack%201/koronebless.png?foo=bar+baz",
|
|
_headers,
|
|
_body,
|
|
_opts ->
|
|
{:ok, 200, [{"content-type", "image/png"}], "It works!"}
|
|
|
|
:get,
|
|
"https://example.com/media/foo/bar%20!$&'()*+,;=/:%20@a%20%5Bbaz%5D.mp4",
|
|
_headers,
|
|
_body,
|
|
_opts ->
|
|
{:ok, 200, [{"content-type", "video/mp4"}], "Allowed reserved chars."}
|
|
|
|
:get, "https://example.com/media/unicode%20%F0%9F%99%82%20.gif", _headers, _body, _opts ->
|
|
{:ok, 200, [{"content-type", "image/gif"}], "Unicode emoji in path"}
|
|
end)
|
|
|> stub(:stream_body, fn _ -> :done end)
|
|
|> stub(:close, fn _ -> :ok end)
|
|
|
|
:ok
|
|
end
|
|
|
|
test "properly encodes URLs with spaces", %{conn: conn} do
|
|
url_with_space = "https://example.com/emoji/Pack 1/koronebless.png?foo=bar baz"
|
|
|
|
result = ReverseProxy.call(conn, url_with_space)
|
|
|
|
assert result.status == 200
|
|
end
|
|
|
|
test "properly encoded URL should not be altered", %{conn: conn} do
|
|
properly_encoded_url = "https://example.com/emoji/Pack%201/koronebless.png?foo=bar+baz"
|
|
|
|
result = ReverseProxy.call(conn, properly_encoded_url)
|
|
|
|
assert result.status == 200
|
|
end
|
|
|
|
test "properly encodes URLs with allowed reserved characters", %{conn: conn} do
|
|
url_with_reserved_chars = "https://example.com/media/foo/bar !$&'()*+,;=/: @a [baz].mp4"
|
|
|
|
result = ReverseProxy.call(conn, url_with_reserved_chars)
|
|
|
|
assert result.status == 200
|
|
end
|
|
|
|
test "properly encodes URLs with unicode in path", %{conn: conn} do
|
|
url_with_unicode = "https://example.com/media/unicode 🙂 .gif"
|
|
|
|
result = ReverseProxy.call(conn, url_with_unicode)
|
|
|
|
assert result.status == 200
|
|
end
|
|
end
|
|
end
|