adding gun adapter

lib/pleroma/http/adapter.ex

@@ -0,0 +1,64 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.HTTP.Adapter do
+  alias Pleroma.HTTP.Connection
+
+  @type proxy ::
+          {Connection.host(), pos_integer()}
+          | {Connection.proxy_type(), pos_integer()}
+  @type host_type :: :domain | :ip
+
+  @callback options(keyword(), URI.t()) :: keyword()
+  @callback after_request(keyword()) :: :ok
+
+  @spec options(keyword(), URI.t()) :: keyword()
+  def options(opts, _uri) do
+    proxy = Pleroma.Config.get([:http, :proxy_url], nil)
+    maybe_add_proxy(opts, format_proxy(proxy))
+  end
+
+  @spec maybe_get_conn(URI.t(), keyword()) :: keyword()
+  def maybe_get_conn(_uri, opts), do: opts
+
+  @spec after_request(keyword()) :: :ok
+  def after_request(_opts), do: :ok
+
+  @spec format_proxy(String.t() | tuple() | nil) :: proxy() | nil
+  def format_proxy(nil), do: nil
+
+  def format_proxy(proxy_url) do
+    with {:ok, host, port} <- Connection.parse_proxy(proxy_url) do
+      {host, port}
+    else
+      {:ok, type, host, port} -> {type, host, port}
+      _ -> nil
+    end
+  end
+
+  @spec maybe_add_proxy(keyword(), proxy() | nil) :: keyword()
+  def maybe_add_proxy(opts, nil), do: opts
+  def maybe_add_proxy(opts, proxy), do: Keyword.put_new(opts, :proxy, proxy)
+
+  @spec domain_or_fallback(String.t()) :: charlist()
+  def domain_or_fallback(host) do
+    case domain_or_ip(host) do
+      {:domain, domain} -> domain
+      {:ip, _ip} -> to_charlist(host)
+    end
+  end
+
+  @spec domain_or_ip(String.t()) :: {host_type(), Connection.host()}
+  def domain_or_ip(host) do
+    charlist = to_charlist(host)
+
+    case :inet.parse_address(charlist) do
+      {:error, :einval} ->
+        {:domain, :idna.encode(charlist)}
+
+      {:ok, ip} when is_tuple(ip) and tuple_size(ip) in [4, 8] ->
+        {:ip, ip}
+    end
+  end
+end

lib/pleroma/http/adapter/gun.ex

@@ -0,0 +1,123 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.HTTP.Adapter.Gun do
+  @behaviour Pleroma.HTTP.Adapter
+
+  alias Pleroma.HTTP.Adapter
+
+  require Logger
+
+  alias Pleroma.Pool.Connections
+
+  @defaults [
+    connect_timeout: 20_000,
+    domain_lookup_timeout: 5_000,
+    tls_handshake_timeout: 5_000,
+    retry_timeout: 100,
+    await_up_timeout: 5_000
+  ]
+
+  @spec options(keyword(), URI.t()) :: keyword()
+  def options(connection_opts \\ [], %URI{} = uri) do
+    proxy = Pleroma.Config.get([:http, :proxy_url], nil)
+
+    @defaults
+    |> Keyword.merge(Pleroma.Config.get([:http, :adapter], []))
+    |> add_original(uri)
+    |> add_scheme_opts(uri)
+    |> Adapter.maybe_add_proxy(Adapter.format_proxy(proxy))
+    |> maybe_get_conn(uri, connection_opts)
+  end
+
+  @spec after_request(keyword()) :: :ok
+  def after_request(opts) do
+    with conn when not is_nil(conn) <- opts[:conn],
+         body_as when body_as != :chunks <- opts[:body_as] do
+      Connections.checkout(conn, self(), :gun_connections)
+    end
+
+    :ok
+  end
+
+  defp add_original(opts, %URI{host: host, port: port}) do
+    formatted_host = Adapter.domain_or_fallback(host)
+
+    Keyword.put(opts, :original, "#{formatted_host}:#{port}")
+  end
+
+  defp add_scheme_opts(opts, %URI{scheme: "http"}), do: opts
+
+  defp add_scheme_opts(opts, %URI{scheme: "https", host: host, port: port}) do
+    adapter_opts = [
+      certificates_verification: true,
+      tls_opts: [
+        verify: :verify_peer,
+        cacertfile: CAStore.file_path(),
+        depth: 20,
+        reuse_sessions: false,
+        verify_fun:
+          {&:ssl_verify_hostname.verify_fun/3, [check_hostname: Adapter.domain_or_fallback(host)]}
+      ]
+    ]
+
+    adapter_opts =
+      if port != 443 do
+        Keyword.put(adapter_opts, :transport, :tls)
+      else
+        adapter_opts
+      end
+
+    Keyword.merge(opts, adapter_opts)
+  end
+
+  defp maybe_get_conn(adapter_opts, uri, connection_opts) do
+    {receive_conn?, opts} =
+      adapter_opts
+      |> Keyword.merge(connection_opts)
+      |> Keyword.pop(:receive_conn, true)
+
+    if Connections.alive?(:gun_connections) and receive_conn? do
+      try_to_get_conn(uri, opts)
+    else
+      opts
+    end
+  end
+
+  defp try_to_get_conn(uri, opts) do
+    try do
+      case Connections.checkin(uri, :gun_connections) do
+        nil ->
+          Logger.info(
+            "Gun connections pool checkin was not succesfull. Trying to open conn for next request."
+          )
+
+          :ok = Connections.open_conn(uri, :gun_connections, opts)
+          opts
+
+        conn when is_pid(conn) ->
+          Logger.debug("received conn #{inspect(conn)} #{Connections.compose_uri(uri)}")
+
+          opts
+          |> Keyword.put(:conn, conn)
+          |> Keyword.put(:close_conn, false)
+      end
+    rescue
+      error ->
+        Logger.warn("Gun connections pool checkin caused error #{inspect(error)}")
+        opts
+    catch
+      :exit, {:timeout, _} ->
+        Logger.info(
+          "Gun connections pool checkin with timeout error #{Connections.compose_uri(uri)}"
+        )
+
+        opts
+
+      :exit, error ->
+        Logger.warn("Gun pool checkin exited with error #{inspect(error)}")
+        opts
+    end
+  end
+end

lib/pleroma/http/adapter/hackney.ex

@@ -0,0 +1,41 @@
+defmodule Pleroma.HTTP.Adapter.Hackney do
+  @behaviour Pleroma.HTTP.Adapter
+
+  @defaults [
+    connect_timeout: 10_000,
+    recv_timeout: 20_000,
+    follow_redirect: true,
+    force_redirect: true,
+    pool: :federation
+  ]
+
+  @spec options(keyword(), URI.t()) :: keyword()
+  def options(connection_opts \\ [], %URI{} = uri) do
+    proxy = Pleroma.Config.get([:http, :proxy_url], nil)
+
+    @defaults
+    |> Keyword.merge(Pleroma.Config.get([:http, :adapter], []))
+    |> Keyword.merge(connection_opts)
+    |> add_scheme_opts(uri)
+    |> Pleroma.HTTP.Adapter.maybe_add_proxy(proxy)
+  end
+
+  defp add_scheme_opts(opts, %URI{scheme: "http"}), do: opts
+
+  defp add_scheme_opts(opts, %URI{scheme: "https", host: host}) do
+    ssl_opts = [
+      ssl_options: [
+        # Workaround for remote server certificate chain issues
+        partial_chain: &:hackney_connect.partial_chain/1,
+
+        # We don't support TLS v1.3 yet
+        versions: [:tlsv1, :"tlsv1.1", :"tlsv1.2"],
+        server_name_indication: to_charlist(host)
+      ]
+    ]
+
+    Keyword.merge(opts, ssl_opts)
+  end
+
+  def after_request(_), do: :ok
+end

lib/pleroma/http/connection.ex

@@ -4,40 +4,99 @@
 
 defmodule Pleroma.HTTP.Connection do
   @moduledoc """
-  Connection for http-requests.
+  Configure Tesla.Client with default and customized adapter options.
   """
+  @type ip_address :: ipv4_address() | ipv6_address()
+  @type ipv4_address :: {0..255, 0..255, 0..255, 0..255}
+  @type ipv6_address ::
+          {0..65_535, 0..65_535, 0..65_535, 0..65_535, 0..65_535, 0..65_535, 0..65_535, 0..65_535}
+  @type proxy_type() :: :socks4 | :socks5
+  @type host() :: charlist() | ip_address()
 
-  @hackney_options [
-    connect_timeout: 10_000,
-    recv_timeout: 20_000,
-    follow_redirect: true,
-    force_redirect: true,
-    pool: :federation
-  ]
-  @adapter Application.get_env(:tesla, :adapter)
+  @defaults [pool: :federation]
+
+  require Logger
+
+  alias Pleroma.Config
+  alias Pleroma.HTTP.Adapter
 
   @doc """
-  Configure a client connection
-
-  # Returns
-
-  Tesla.Env.client
+  Merge default connection & adapter options with received ones.
   """
-  @spec new(Keyword.t()) :: Tesla.Env.client()
-  def new(opts \\ []) do
-    Tesla.client([], {@adapter, hackney_options(opts)})
+
+  @spec options(URI.t(), keyword()) :: keyword()
+  def options(%URI{} = uri, opts \\ []) do
+    @defaults
+    |> pool_timeout()
+    |> Keyword.merge(opts)
+    |> adapter().options(uri)
   end
 
-  # fetch Hackney options
-  #
-  def hackney_options(opts) do
-    options = Keyword.get(opts, :adapter, [])
-    adapter_options = Pleroma.Config.get([:http, :adapter], [])
-    proxy_url = Pleroma.Config.get([:http, :proxy_url], nil)
+  defp pool_timeout(opts) do
+    timeout =
+      Config.get([:pools, opts[:pool], :timeout]) || Config.get([:pools, :default, :timeout])
 
-    @hackney_options
-    |> Keyword.merge(adapter_options)
-    |> Keyword.merge(options)
-    |> Keyword.merge(proxy: proxy_url)
+    Keyword.merge(opts, timeout: timeout)
+  end
+
+  @spec after_request(keyword()) :: :ok
+  def after_request(opts), do: adapter().after_request(opts)
+
+  defp adapter do
+    case Application.get_env(:tesla, :adapter) do
+      Tesla.Adapter.Gun -> Adapter.Gun
+      Tesla.Adapter.Hackney -> Adapter.Hackney
+      _ -> Adapter
+    end
+  end
+
+  @spec parse_proxy(String.t() | tuple() | nil) ::
+          {:ok, host(), pos_integer()}
+          | {:ok, proxy_type(), host(), pos_integer()}
+          | {:error, atom()}
+          | nil
+
+  def parse_proxy(nil), do: nil
+
+  def parse_proxy(proxy) when is_binary(proxy) do
+    with [host, port] <- String.split(proxy, ":"),
+         {port, ""} <- Integer.parse(port) do
+      {:ok, parse_host(host), port}
+    else
+      {_, _} ->
+        Logger.warn("parsing port in proxy fail #{inspect(proxy)}")
+        {:error, :error_parsing_port_in_proxy}
+
+      :error ->
+        Logger.warn("parsing port in proxy fail #{inspect(proxy)}")
+        {:error, :error_parsing_port_in_proxy}
+
+      _ ->
+        Logger.warn("parsing proxy fail #{inspect(proxy)}")
+        {:error, :error_parsing_proxy}
+    end
+  end
+
+  def parse_proxy(proxy) when is_tuple(proxy) do
+    with {type, host, port} <- proxy do
+      {:ok, type, parse_host(host), port}
+    else
+      _ ->
+        Logger.warn("parsing proxy fail #{inspect(proxy)}")
+        {:error, :error_parsing_proxy}
+    end
+  end
+
+  @spec parse_host(String.t() | atom() | charlist()) :: charlist() | ip_address()
+  def parse_host(host) when is_list(host), do: host
+  def parse_host(host) when is_atom(host), do: to_charlist(host)
+
+  def parse_host(host) when is_binary(host) do
+    host = to_charlist(host)
+
+    case :inet.parse_address(host) do
+      {:error, :einval} -> host
+      {:ok, ip} -> ip
+    end
   end
 end

lib/pleroma/http/http.ex

@@ -4,21 +4,47 @@
 
 defmodule Pleroma.HTTP do
   @moduledoc """
-
+    Wrapper for `Tesla.request/2`.
   """
 
   alias Pleroma.HTTP.Connection
+  alias Pleroma.HTTP.Request
   alias Pleroma.HTTP.RequestBuilder, as: Builder
+  alias Tesla.Client
+  alias Tesla.Env
+
+  require Logger
 
   @type t :: __MODULE__
 
   @doc """
-  Builds and perform http request.
+  Performs GET request.
+
+  See `Pleroma.HTTP.request/5`
+  """
+  @spec get(Request.url() | nil, Request.headers(), keyword()) ::
+          nil | {:ok, Env.t()} | {:error, any()}
+  def get(url, headers \\ [], options \\ [])
+  def get(nil, _, _), do: nil
+  def get(url, headers, options), do: request(:get, url, "", headers, options)
+
+  @doc """
+  Performs POST request.
+
+  See `Pleroma.HTTP.request/5`
+  """
+  @spec post(Request.url(), String.t(), Request.headers(), keyword()) ::
+          {:ok, Env.t()} | {:error, any()}
+  def post(url, body, headers \\ [], options \\ []),
+    do: request(:post, url, body, headers, options)
+
+  @doc """
+  Builds and performs http request.
 
   # Arguments:
   `method` - :get, :post, :put, :delete
-  `url`
-  `body`
+  `url` - full url
+  `body` - request body
   `headers` - a keyworld list of headers, e.g. `[{"content-type", "text/plain"}]`
   `options` - custom, per-request middleware or adapter options
 
@@ -26,23 +52,78 @@ defmodule Pleroma.HTTP do
   `{:ok, %Tesla.Env{}}` or `{:error, error}`
 
   """
-  def request(method, url, body \\ "", headers \\ [], options \\ []) do
+  @spec request(atom(), Request.url(), String.t(), Request.headers(), keyword()) ::
+          {:ok, Env.t()} | {:error, any()}
+  def request(method, url, body, headers, options) when is_binary(url) do
+    with uri <- URI.parse(url),
+         received_adapter_opts <- Keyword.get(options, :adapter, []),
+         adapter_opts <- Connection.options(uri, received_adapter_opts),
+         options <- put_in(options[:adapter], adapter_opts),
+         params <- Keyword.get(options, :params, []),
+         request <- build_request(method, headers, options, url, body, params),
+         client <- Tesla.client([Tesla.Middleware.FollowRedirects], tesla_adapter()),
+         pid <- Process.whereis(adapter_opts[:pool]) do
+      pool_alive? =
+        if tesla_adapter() == Tesla.Adapter.Gun do
+          if pid, do: Process.alive?(pid), else: false
+        else
+          false
+        end
+
+      request_opts =
+        adapter_opts
+        |> Enum.into(%{})
+        |> Map.put(:env, Pleroma.Config.get([:env]))
+        |> Map.put(:pool_alive?, pool_alive?)
+
+      response =
+        request(
+          client,
+          request,
+          request_opts
+        )
+
+      Connection.after_request(adapter_opts)
+
+      response
+    end
+  end
+
+  @spec request(Client.t(), keyword(), map()) :: {:ok, Env.t()} | {:error, any()}
+  def request(%Client{} = client, request, %{env: :test}), do: request_try(client, request)
+
+  def request(%Client{} = client, request, %{body_as: :chunks}) do
+    request_try(client, request)
+  end
+
+  def request(%Client{} = client, request, %{pool_alive?: false}) do
+    request_try(client, request)
+  end
+
+  def request(%Client{} = client, request, %{pool: pool, timeout: timeout}) do
     try do
-      options =
-        process_request_options(options)
-        |> process_sni_options(url)
+      :poolboy.transaction(
+        pool,
+        &Pleroma.Pool.Request.execute(&1, client, request, timeout + 500),
+        timeout + 1_000
+      )
+    rescue
+      e ->
+        {:error, e}
+    catch
+      :exit, {:timeout, _} ->
+        Logger.warn("Receive response from pool failed #{request[:url]}")
+        {:error, :recv_pool_timeout}
 
-      params = Keyword.get(options, :params, [])
+      :exit, e ->
+        {:error, e}
+    end
+  end
 
-      %{}
-      |> Builder.method(method)
-      |> Builder.headers(headers)
-      |> Builder.opts(options)
-      |> Builder.url(url)
-      |> Builder.add_param(:body, :body, body)
-      |> Builder.add_param(:query, :query, params)
-      |> Enum.into([])
-      |> (&Tesla.request(Connection.new(options), &1)).()
+  @spec request_try(Client.t(), keyword()) :: {:ok, Env.t()} | {:error, any()}
+  def request_try(client, request) do
+    try do
+      Tesla.request(client, request)
     rescue
       e ->
         {:error, e}
@@ -52,35 +133,16 @@ defmodule Pleroma.HTTP do
     end
   end
 
-  defp process_sni_options(options, nil), do: options
-
-  defp process_sni_options(options, url) do
-    uri = URI.parse(url)
-    host = uri.host |> to_charlist()
-
-    case uri.scheme do
-      "https" -> options ++ [ssl: [server_name_indication: host]]
-      _ -> options
-    end
+  defp build_request(method, headers, options, url, body, params) do
+    Builder.new()
+    |> Builder.method(method)
+    |> Builder.headers(headers)
+    |> Builder.opts(options)
+    |> Builder.url(url)
+    |> Builder.add_param(:body, :body, body)
+    |> Builder.add_param(:query, :query, params)
+    |> Builder.convert_to_keyword()
   end
 
-  def process_request_options(options) do
-    Keyword.merge(Pleroma.HTTP.Connection.hackney_options([]), options)
-  end
-
-  @doc """
-  Performs GET request.
-
-  See `Pleroma.HTTP.request/5`
-  """
-  def get(url, headers \\ [], options \\ []),
-    do: request(:get, url, "", headers, options)
-
-  @doc """
-  Performs POST request.
-
-  See `Pleroma.HTTP.request/5`
-  """
-  def post(url, body, headers \\ [], options \\ []),
-    do: request(:post, url, body, headers, options)
+  defp tesla_adapter, do: Application.get_env(:tesla, :adapter)
 end

lib/pleroma/http/request.ex

@@ -0,0 +1,23 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.HTTP.Request do
+  @moduledoc """
+  Request struct.
+  """
+  defstruct method: :get, url: "", query: [], headers: [], body: "", opts: []
+
+  @type method :: :head | :get | :delete | :trace | :options | :post | :put | :patch
+  @type url :: String.t()
+  @type headers :: [{String.t(), String.t()}]
+
+  @type t :: %__MODULE__{
+          method: method(),
+          url: url(),
+          query: keyword(),
+          headers: headers(),
+          body: String.t(),
+          opts: keyword()
+        }
+end

lib/pleroma/http/request_builder.ex

@@ -7,77 +7,54 @@ defmodule Pleroma.HTTP.RequestBuilder do
   Helper functions for building Tesla requests
   """
 
+  alias Pleroma.HTTP.Request
+  alias Tesla.Multipart
+
   @doc """
-  Specify the request method when building a request
-
-  ## Parameters
-
-  - request (Map) - Collected request options
-  - m (atom) - Request method
-
-  ## Returns
-
-  Map
+  Creates new request
   """
-  @spec method(map(), atom) :: map()
-  def method(request, m) do
-    Map.put_new(request, :method, m)
-  end
+  @spec new(Request.t()) :: Request.t()
+  def new(%Request{} = request \\ %Request{}), do: request
 
   @doc """
   Specify the request method when building a request
-
-  ## Parameters
-
-  - request (Map) - Collected request options
-  - u (String) - Request URL
-
-  ## Returns
-
-  Map
   """
-  @spec url(map(), String.t()) :: map()
-  def url(request, u) do
-    Map.put_new(request, :url, u)
-  end
+  @spec method(Request.t(), Request.method()) :: Request.t()
+  def method(request, m), do: %{request | method: m}
+
+  @doc """
+  Specify the request method when building a request
+  """
+  @spec url(Request.t(), Request.url()) :: Request.t()
+  def url(request, u), do: %{request | url: u}
 
   @doc """
   Add headers to the request
   """
-  @spec headers(map(), list(tuple)) :: map()
-  def headers(request, header_list) do
-    header_list =
+  @spec headers(Request.t(), Request.headers()) :: Request.t()
+  def headers(request, headers) do
+    headers_list =
       if Pleroma.Config.get([:http, :send_user_agent]) do
-        header_list ++ [{"User-Agent", Pleroma.Application.user_agent()}]
+        headers ++ [{"user-agent", Pleroma.Application.user_agent()}]
       else
-        header_list
+        headers
       end
 
-    Map.put_new(request, :headers, header_list)
+    %{request | headers: headers_list}
   end
 
   @doc """
   Add custom, per-request middleware or adapter options to the request
   """
-  @spec opts(map(), Keyword.t()) :: map()
-  def opts(request, options) do
-    Map.put_new(request, :opts, options)
-  end
+  @spec opts(Request.t(), keyword()) :: Request.t()
+  def opts(request, options), do: %{request | opts: options}
 
+  # NOTE: isn't used anywhere
   @doc """
   Add optional parameters to the request
 
-  ## Parameters
-
-  - request (Map) - Collected request options
-  - definitions (Map) - Map of parameter name to parameter location.
-  - options (KeywordList) - The provided optional parameters
-
-  ## Returns
-
-  Map
   """
-  @spec add_optional_params(map(), %{optional(atom) => atom}, keyword()) :: map()
+  @spec add_optional_params(Request.t(), %{optional(atom) => atom}, keyword()) :: map()
   def add_optional_params(request, _, []), do: request
 
   def add_optional_params(request, definitions, [{key, value} | tail]) do
@@ -94,49 +71,43 @@ defmodule Pleroma.HTTP.RequestBuilder do
 
   @doc """
   Add optional parameters to the request
-
-  ## Parameters
-
-  - request (Map) - Collected request options
-  - location (atom) - Where to put the parameter
-  - key (atom) - The name of the parameter
-  - value (any) - The value of the parameter
-
-  ## Returns
-
-  Map
   """
-  @spec add_param(map(), atom, atom, any()) :: map()
-  def add_param(request, :query, :query, values), do: Map.put(request, :query, values)
+  @spec add_param(Request.t(), atom(), atom(), any()) :: Request.t()
+  def add_param(request, :query, :query, values), do: %{request | query: values}
 
-  def add_param(request, :body, :body, value), do: Map.put(request, :body, value)
+  def add_param(request, :body, :body, value), do: %{request | body: value}
 
   def add_param(request, :body, key, value) do
     request
-    |> Map.put_new_lazy(:body, &Tesla.Multipart.new/0)
+    |> Map.put(:body, Multipart.new())
     |> Map.update!(
       :body,
-      &Tesla.Multipart.add_field(
+      &Multipart.add_field(
         &1,
         key,
         Jason.encode!(value),
-        headers: [{:"Content-Type", "application/json"}]
+        headers: [{"content-type", "application/json"}]
       )
     )
   end
 
   def add_param(request, :file, name, path) do
     request
-    |> Map.put_new_lazy(:body, &Tesla.Multipart.new/0)
-    |> Map.update!(:body, &Tesla.Multipart.add_file(&1, path, name: name))
+    |> Map.put(:body, Multipart.new())
+    |> Map.update!(:body, &Multipart.add_file(&1, path, name: name))
   end
 
   def add_param(request, :form, name, value) do
-    request
-    |> Map.update(:body, %{name => value}, &Map.put(&1, name, value))
+    Map.update(request, :body, %{name => value}, &Map.put(&1, name, value))
   end
 
   def add_param(request, location, key, value) do
     Map.update(request, location, [{key, value}], &(&1 ++ [{key, value}]))
   end
+
+  def convert_to_keyword(request) do
+    request
+    |> Map.from_struct()
+    |> Enum.into([])
+  end
 end