Merge branch 'audit-mix-audit-report' (#69) into release/2.11

# Conflicts:
#	lib/pleroma/http/adapter_helper/hackney.ex

changelog.d/gun-default.change

@@ -0,0 +1 @@
+Gun is now the default HTTP client instead of Hackney

changelog.d/hackney-http1.fix

@@ -0,0 +1 @@
+Hackney: use HTTP/1.1 only, so closing one media proxy response no longer breaks other downloads from the same host

changelog.d/hackney-proxy-ip-host.fix

@@ -0,0 +1 @@
+Hackney: use the configured proxy when its host is an IP address; Hackney 4 ignored such proxies and connected directly

changelog.d/scrubber-hidden-schemes.security

@@ -0,0 +1 @@
+Scrubbers: remove links and images whose scheme is hidden with tabs, newlines or leading control characters (for example java<TAB>script:), which browsers still run as javascript:. Custom scrubbers should use Pleroma.HTML.URIAttributes.allow_tag_with_uri_attributes instead of the FastSanitize macro

config/config.exs

@@ -161,7 +161,11 @@ config :mime, :types, %{
   "image/apng" => ["apng"]
 }
 
-config :tesla, adapter: Tesla.Adapter.Hackney
+config :tesla, adapter: Tesla.Adapter.Gun
+
+# Hackney 4 shares one HTTP/2 connection per host, and closing one response
+# (as the media proxy does) stops it for every other stream on it.
+config :hackney, default_protocols: [:http1]
 
 # Configures http settings, upstream proxy etc.
 config :pleroma, :http,

docs/configuration/cheatsheet.md

@@ -541,6 +541,15 @@ Available caches:
 
 ## HTTP client
 
+Pleroma uses [Gun](https://github.com/ninenines/gun) for outgoing HTTP requests. To use Hackney instead:
+
+```elixir
+config :tesla, adapter: Tesla.Adapter.Hackney
+```
+
+!!! note
+    With Hackney, size limits for fetched responses (for example for link previews) are checked only after Hackney has read the whole response. They limit what Pleroma keeps, not how much it downloads or holds in memory. Gun stops reading at the limit.
+
 ### :http
 
 * `proxy_url`: an upstream proxy to fetch posts and/or media with, (default: `nil`)
@@ -550,6 +559,8 @@ Available caches:
 
 ### :hackney_pools
 
+*For `hackney` adapter*
+
 Advanced. Tweaks Hackney (http client) connections pools.
 
 There's three pools used:

docs/installation/debian_based_jp.md

@@ -14,7 +14,7 @@ Note: This article is potentially outdated because at this time we may not have
 
 - PostgreSQL 11.0以上 (Ubuntu16.04では9.5しか提供されていないので，[](https://www.postgresql.org/download/linux/ubuntu/)こちらから新しいバージョンを入手してください)
 - `postgresql-contrib` 11.0以上 (同上)
-- Elixir 1.15 以上 ([Debianのリポジトリからインストールしないこと！！！ ここからインストールすること！](https://elixir-lang.org/install.html#unix-and-unix-like)。または [asdf](https://github.com/asdf-vm/asdf) をpleromaユーザーでインストールしてください)
+- Elixir 1.17 以上 (Erlang/OTP 27 以上) ([Debianのリポジトリからインストールしないこと！！！ ここからインストールすること！](https://elixir-lang.org/install.html#unix-and-unix-like)。または [asdf](https://github.com/asdf-vm/asdf) をpleromaユーザーでインストールしてください)
 - `erlang-dev`
 - `erlang-nox`
 - `git`

docs/installation/erlang_elixir_requirements.include

@@ -1,2 +1,2 @@
-* Elixir >=1.15.0, <1.20
+* Elixir >=1.17.0, <1.20
-* Erlang OTP >=24.0.0 (supported: <29)
+* Erlang OTP >=27.0.0 (supported: <29)

docs/installation/migrating_from_rebased.md

@@ -46,7 +46,7 @@ present, `uploads`, and `instance`. Diff and restore any other non-stock
 configuration files, preserve their ownership, and review them before starting
 Pleroma.
 
-Pleroma requires Elixir 1.15 or newer. Install supported Erlang and Elixir
+Pleroma requires Elixir 1.17 or newer and Erlang/OTP 27 or newer. Install supported Erlang and Elixir
 versions as described in the installation guide for your distribution before
 running any Mix command. Do not reuse Rebased's `.tool-versions`. If you keep
 using [`asdf`](https://asdf-vm.com/), select supported versions in the

lib/pleroma/formatter.ex

@@ -169,7 +169,8 @@ defmodule Pleroma.Formatter do
     Enum.map(nodes, &strip_email_autolinks(&1, emails))
   end
 
-  defp strip_email_autolinks({"a", attrs, [email], _meta} = node, emails) do
+  defp strip_email_autolinks({"a", attrs, [email], _meta} = node, emails)
+       when is_binary(email) do
     if List.keyfind(attrs, "href", 0) == {"href", "mailto:" <> email} and
          MapSet.member?(emails, email) do
       email
@@ -251,9 +252,13 @@ defmodule Pleroma.Formatter do
 
   defp sanitize_markdown_url(value, name) do
     if String.downcase(to_string(name)) in ["href", "src"] do
-      value
+      # Percent-encode instead of cutting, so the URL stays complete but can
-      |> String.split(~r/["<>]/, parts: 2)
+      # never close the attribute or open a tag.
-      |> hd()
+      String.replace(value, ~r/["<>]/, fn
+        "\"" -> "%22"
+        "<" -> "%3C"
+        ">" -> "%3E"
+      end)
     else
       value
     end

lib/pleroma/html/uri_attributes.ex

@@ -0,0 +1,40 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.HTML.URIAttributes do
+  @moduledoc """
+  URI attribute rules for the scrubbers.
+
+  Works like `FastSanitize.Sanitizer.Meta.allow_tag_with_uri_attributes/3`, but
+  checks the scheme the way a browser reads the URL. Browsers remove tabs and
+  newlines anywhere and control characters and spaces at the start, so
+  `java\\tscript:` is a `javascript:` link to them, while `URI.parse/1` finds no
+  scheme and FastSanitize lets it through.
+  """
+
+  defmacro allow_tag_with_uri_attributes(tag, attrs, valid_schemes) do
+    for attr <- attrs do
+      quote do
+        def scrub_attribute(unquote(tag), {unquote(attr), "&" <> _}), do: nil
+
+        def scrub_attribute(unquote(tag), {unquote(attr), uri} = attribute) do
+          if Pleroma.HTML.URIAttributes.valid_scheme?(uri, unquote(valid_schemes)),
+            do: attribute
+        end
+      end
+    end
+  end
+
+  @spec valid_scheme?(String.t(), [String.t()]) :: boolean()
+  def valid_scheme?(uri, valid_schemes) do
+    scheme =
+      uri
+      |> String.replace(~r/[\t\n\r]/, "")
+      |> String.replace(~r/\A[\x00-\x20]+/, "")
+      |> URI.parse()
+      |> Map.get(:scheme)
+
+    is_nil(scheme) or scheme in valid_schemes
+  end
+end

lib/pleroma/http/adapter_helper/gun.ex

@@ -28,6 +28,7 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do
 
     @defaults
     |> Keyword.merge(config_opts)
+    |> ssl_options_to_tls_opts()
     |> add_scheme_opts(uri)
     |> AdapterHelper.maybe_add_proxy(proxy)
     |> Keyword.merge(incoming_opts)
@@ -35,6 +36,15 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do
     |> maybe_stream()
   end
 
+  # [:http, :adapter, :ssl_options] is Hackney's name for the TLS options,
+  # and AdminFE sets it. Keep those settings when Gun is the adapter.
+  defp ssl_options_to_tls_opts(opts) do
+    case Keyword.pop(opts, :ssl_options) do
+      {nil, opts} -> opts
+      {ssl_options, opts} -> Keyword.put_new(opts, :tls_opts, ssl_options)
+    end
+  end
+
   defp add_scheme_opts(opts, %{scheme: "http"}), do: opts
 
   defp add_scheme_opts(opts, %{scheme: "https"}) do

lib/pleroma/http/adapter_helper/hackney.ex

@@ -30,9 +30,25 @@ defmodule Pleroma.HTTP.AdapterHelper.Hackney do
     |> Pleroma.HTTP.AdapterHelper.maybe_add_proxy(proxy)
     |> Keyword.put(:follow_redirect, false)
     |> Keyword.put(:force_redirect, false)
+    |> normalize_proxy()
     |> Pleroma.HTTP.TLS.put_hackney_ssl_options()
   end
 
+  # Hackney 4 ignores a proxy whose host is an IP tuple and connects directly.
+  # ConfigDB turns a numeric proxy host into such a tuple.
+  defp normalize_proxy(opts) do
+    case Keyword.fetch(opts, :proxy) do
+      {:ok, {type, host, port}} when is_tuple(host) ->
+        Keyword.put(opts, :proxy, {type, :inet.ntoa(host), port})
+
+      {:ok, {host, port}} when is_tuple(host) ->
+        Keyword.put(opts, :proxy, {:inet.ntoa(host), port})
+
+      _ ->
+        opts
+    end
+  end
+
   defp add_scheme_opts(opts, _), do: opts
 
   defp maybe_add_with_body(opts) do

priv/scrubbers/default.ex

@@ -6,6 +6,7 @@ defmodule Pleroma.HTML.Scrubber.Default do
   @doc "The default HTML scrubbing policy: no "
 
   require FastSanitize.Sanitizer.Meta
+  require Pleroma.HTML.URIAttributes
   alias FastSanitize.Sanitizer.Meta
 
   # credo:disable-for-previous-line
@@ -15,7 +16,11 @@ defmodule Pleroma.HTML.Scrubber.Default do
 
   Meta.strip_comments()
 
-  Meta.allow_tag_with_uri_attributes(:a, ["href", "data-user", "data-tag"], @valid_schemes)
+  Pleroma.HTML.URIAttributes.allow_tag_with_uri_attributes(
+    :a,
+    ["href", "data-user", "data-tag"],
+    @valid_schemes
+  )
 
   Meta.allow_tag_with_this_attribute_values(:a, "class", [
     "hashtag",
@@ -145,7 +150,7 @@ defmodule Pleroma.HTML.Scrubber.Default do
     Meta.allow_tag_with_this_attribute_values(:img, "class", ["emoji"])
 
     # restrict img tags to http/https only, because of MediaProxy.
-    Meta.allow_tag_with_uri_attributes(:img, ["src"], ["http", "https"])
+    Pleroma.HTML.URIAttributes.allow_tag_with_uri_attributes(:img, ["src"], ["http", "https"])
 
     Meta.allow_tag_with_these_attributes(:img, [
       "width",

priv/scrubbers/links_only.ex

@@ -10,12 +10,13 @@ defmodule Pleroma.HTML.Scrubber.LinksOnly do
   @valid_schemes Pleroma.Config.get([:uri_schemes, :valid_schemes], [])
 
   require FastSanitize.Sanitizer.Meta
+  require Pleroma.HTML.URIAttributes
   alias FastSanitize.Sanitizer.Meta
 
   Meta.strip_comments()
 
   # links
-  Meta.allow_tag_with_uri_attributes(:a, ["href"], @valid_schemes)
+  Pleroma.HTML.URIAttributes.allow_tag_with_uri_attributes(:a, ["href"], @valid_schemes)
 
   Meta.allow_tag_with_this_attribute_values(:a, "rel", [
     "tag",

priv/scrubbers/twitter_text.ex

@@ -11,12 +11,17 @@ defmodule Pleroma.HTML.Scrubber.TwitterText do
   @valid_schemes Pleroma.Config.get([:uri_schemes, :valid_schemes], [])
 
   require FastSanitize.Sanitizer.Meta
+  require Pleroma.HTML.URIAttributes
   alias FastSanitize.Sanitizer.Meta
 
   Meta.strip_comments()
 
   # links
-  Meta.allow_tag_with_uri_attributes(:a, ["href", "data-user", "data-tag"], @valid_schemes)
+  Pleroma.HTML.URIAttributes.allow_tag_with_uri_attributes(
+    :a,
+    ["href", "data-user", "data-tag"],
+    @valid_schemes
+  )
 
   Meta.allow_tag_with_this_attribute_values(:a, "class", [
     "hashtag",
@@ -49,7 +54,7 @@ defmodule Pleroma.HTML.Scrubber.TwitterText do
     Meta.allow_tag_with_this_attribute_values(:img, "class", ["emoji"])
 
     # restrict img tags to http/https only, because of MediaProxy.
-    Meta.allow_tag_with_uri_attributes(:img, ["src"], ["http", "https"])
+    Pleroma.HTML.URIAttributes.allow_tag_with_uri_attributes(:img, ["src"], ["http", "https"])
 
     Meta.allow_tag_with_these_attributes(:img, [
       "width",

test/pleroma/formatter_test.exs

@@ -375,13 +375,22 @@ defmodule Pleroma.FormatterTest do
   end
 
   describe "markdown_to_html/2" do
+    test "keeps formatted link text when stripping email autolinks" do
+      opts = %{breaks: true, strip_email_autolinks: true}
+
+      assert Formatter.markdown_to_html("[**hello**](https://example.org/)", opts) ==
+               ~s(<p><a href="https://example.org/"><strong>hello</strong></a></p>)
+
+      assert Formatter.markdown_to_html("[`code`](https://example.org/)", opts) ==
+               ~s(<p><a href="https://example.org/"><code class="inline">code</code></a></p>)
+    end
+
     test "escapes link attribute values" do
       markdown = ~s|[click](http://example.com/?a=x" onerror="alert(1))|
 
-      html = Formatter.markdown_to_html(markdown)
+      # The quotes are percent-encoded, so everything stays inside href.
-
+      assert Formatter.markdown_to_html(markdown) ==
-      assert html =~ ~s|href="http://example.com/?a=x"|
+               ~s|<p><a href="http://example.com/?a=x%22 onerror=%22alert(1)">click</a></p>|
-      refute html =~ "onerror"
     end
 
     test "preserves entities without rendering dangerous raw HTML" do
@@ -406,10 +415,15 @@ defmodule Pleroma.FormatterTest do
     test "rejects entity-encoded URL delimiters" do
       markdown = "[click](http://example.com/?a=x&#34;onerror=alert(1))"
 
-      html = Formatter.markdown_to_html(markdown)
+      assert Formatter.markdown_to_html(markdown) ==
+               ~s|<p><a href="http://example.com/?a=x%22onerror=alert(1)">click</a></p>|
+    end
 
-      assert html =~ ~s|href="http://example.com/?a=x"|
+    test "keeps URLs with encoded quotes complete" do
-      refute html =~ "onerror"
+      markdown = "[search](https://example.org/?q=&quot;cat&quot;&page=2)"
+
+      assert Formatter.markdown_to_html(markdown) ==
+               ~s(<p><a href="https://example.org/?q=%22cat%22&amp;page=2">search</a></p>)
     end
 
     test "strips email autolinks without changing explicit mailto links" do

test/pleroma/html_test.exs

@@ -141,6 +141,44 @@ defmodule Pleroma.HTMLTest do
     end
   end
 
+  describe "link schemes as a browser sees them" do
+    # Browsers remove tabs and newlines in URLs and control characters and
+    # spaces at the start, so these are javascript: links to them.
+    test "removes javascript: links hidden with control characters" do
+      for href <- [
+            "java\tscript:alert(1)",
+            "java&#x09;script:alert(1)",
+            "java\nscript:alert(1)",
+            "&#x0d;javascript:alert(1)",
+            " javascript:alert(1)",
+            "\u0001javascript:alert(1)"
+          ],
+          scrubber <- [
+            Pleroma.HTML.Scrubber.Default,
+            Pleroma.HTML.Scrubber.TwitterText,
+            Pleroma.HTML.Scrubber.LinksOnly
+          ] do
+        html = HTML.filter_tags(~s(<a href="#{href}">x</a>), scrubber)
+
+        refute html =~ "href", "#{inspect(href)} survived #{inspect(scrubber)}: #{html}"
+      end
+    end
+
+    test "keeps normal links" do
+      for href <- ["https://example.org/a?b=c", "/relative/path", "#fragment"] do
+        assert HTML.filter_tags(~s(<a href="#{href}">x</a>), Pleroma.HTML.Scrubber.Default) ==
+                 ~s(<a href="#{href}">x</a>)
+      end
+    end
+
+    test "removes images with hidden schemes" do
+      html =
+        HTML.filter_tags(~s|<img src="java\tscript:alert(1)"/>|, Pleroma.HTML.Scrubber.Default)
+
+      refute html =~ "src"
+    end
+  end
+
   describe "default scrubber" do
     test "normalizes HTML as expected" do
       expected = """

test/pleroma/http/adapter_helper/gun_test.exs

@@ -50,6 +50,15 @@ defmodule Pleroma.HTTP.AdapterHelper.GunTest do
       assert Keyword.has_key?(defaults, :b)
     end
 
+    test "uses Hackney's ssl_options from the config as Gun's tls_opts" do
+      clear_config([:http, :adapter], ssl_options: [versions: [:"tlsv1.3"]])
+
+      opts = Gun.options([], URI.parse("https://example.com"))
+
+      assert opts[:tls_opts] == [versions: [:"tlsv1.3"]]
+      refute Keyword.has_key?(opts, :ssl_options)
+    end
+
     test "parses string proxy host & port" do
       clear_config([:http, :proxy_url], "localhost:8123")
 

test/pleroma/http/adapter_helper/hackney_test.exs

@@ -13,6 +13,12 @@ defmodule Pleroma.HTTP.AdapterHelper.HackneyTest do
     {:ok, uri: uri}
   end
 
+  # Hackney 4 shares one HTTP/2 connection per host, and closing a response
+  # stops that connection for every stream on it.
+  test "Hackney only speaks HTTP/1.1" do
+    assert :hackney_util.default_protocols() == [:http1]
+  end
+
   describe "options/2" do
     setup do: clear_config([:http, :adapter], a: 1, b: 2)
 
@@ -39,6 +45,19 @@ defmodule Pleroma.HTTP.AdapterHelper.HackneyTest do
       assert opts[:proxy] == "localhost:8123"
     end
 
+    test "gives Hackney a string host for a proxy with an IP address", %{uri: uri} do
+      clear_config([:http, :proxy_url], {:socks5, {127, 0, 0, 1}, 9050})
+      assert Hackney.options([], uri)[:proxy] == {:socks5, ~c"127.0.0.1", 9050}
+
+      clear_config([:http, :proxy_url], {{10, 0, 0, 1}, 8123})
+      assert Hackney.options([], uri)[:proxy] == {~c"10.0.0.1", 8123}
+
+      clear_config([:http, :proxy_url], nil)
+
+      assert Hackney.options([proxy: {:socks5, {0, 0, 0, 0, 0, 0, 0, 1}, 9050}], uri)[:proxy] ==
+               {:socks5, ~c"::1", 9050}
+    end
+
     test "respect connection opts and no proxy", %{uri: uri} do
       opts = Hackney.options([a: 2, b: 1], uri)
 

test/pleroma/http/default_adapter_test.exs

@@ -0,0 +1,13 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.HTTP.DefaultAdapterTest do
+  use ExUnit.Case, async: true
+
+  # The test environment uses Tesla.Mock, and config.exs imports the config
+  # of the current Mix env, so check the shipped default in the file itself.
+  test "Gun is the default HTTP client" do
+    assert File.read!("config/config.exs") =~ ~r/^config :tesla, adapter: Tesla.Adapter.Gun$/m
+  end
+end

test/pleroma/web/common_api/utils_test.exs

@@ -232,7 +232,7 @@ defmodule Pleroma.Web.CommonAPI.UtilsTest do
       code = ~s|[click](http://example.com/?a=x" onerror="alert(1))|
       {result, [], []} = Utils.format_input(code, "text/markdown")
 
-      assert result =~ ~s(<a href="http://example.com/?a=x">)
+      assert result =~ ~s|<a href="http://example.com/?a=x%22 onerror=%22alert(1)">|
       refute result =~ ~s|onerror="alert(1)|
       refute result =~ ~s(<a href="http://example.com/?a=x" onerror=)
     end