pleroma/test/pleroma/formatter_test.exs

# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only

defmodule Pleroma.FormatterTest do
  alias Pleroma.Formatter
  alias Pleroma.User
  use Pleroma.DataCase

  import Mock
  import Pleroma.Factory

  setup_all do
    clear_config(Pleroma.Formatter)
    Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end)
    :ok
  end

  describe ".add_hashtag_links" do
    test "turns hashtags into links" do
      text = "I love #cofe and #2hu"

      expected_text =
        ~s(I love <a class="hashtag" data-tag="cofe" href="http://localhost:4001/tag/cofe" rel="tag ugc">#cofe</a> and <a class="hashtag" data-tag="2hu" href="http://localhost:4001/tag/2hu" rel="tag ugc">#2hu</a>)

      assert {^expected_text, [], _tags} = Formatter.linkify(text)
    end

    test "does not turn html characters to tags" do
      text = "#fact_3: pleroma does what mastodon't"

      expected_text =
        ~s(<a class="hashtag" data-tag="fact_3" href="http://localhost:4001/tag/fact_3" rel="tag ugc">#fact_3</a>: pleroma does what mastodon't)

      assert {^expected_text, [], _tags} = Formatter.linkify(text)
    end
  end

  describe ".add_links" do
    test "turning urls into links" do
      text = "Hey, check out https://www.youtube.com/watch?v=8Zg1-TufF%20zY?x=1&y=2#blabla ."

      expected =
        ~S(Hey, check out <a href="https://www.youtube.com/watch?v=8Zg1-TufF%20zY?x=1&y=2#blabla" rel="ugc">https://www.youtube.com/watch?v=8Zg1-TufF%20zY?x=1&y=2#blabla</a> .)

      assert {^expected, [], []} = Formatter.linkify(text)

      text = "https://mastodon.social/@lambadalambda"

      expected =
        ~S(<a href="https://mastodon.social/@lambadalambda" rel="ugc">https://mastodon.social/@lambadalambda</a>)

      assert {^expected, [], []} = Formatter.linkify(text)

      text = "https://mastodon.social:4000/@lambadalambda"

      expected =
        ~S(<a href="https://mastodon.social:4000/@lambadalambda" rel="ugc">https://mastodon.social:4000/@lambadalambda</a>)

      assert {^expected, [], []} = Formatter.linkify(text)

      text = "@lambadalambda"
      expected = "@lambadalambda"

      assert {^expected, [], []} = Formatter.linkify(text)

      text = "http://www.cs.vu.nl/~ast/intel/"

      expected =
        ~S(<a href="http://www.cs.vu.nl/~ast/intel/" rel="ugc">http://www.cs.vu.nl/~ast/intel/</a>)

      assert {^expected, [], []} = Formatter.linkify(text)

      text = "https://forum.zdoom.org/viewtopic.php?f=44&t=57087"

      expected =
        "<a href=\"https://forum.zdoom.org/viewtopic.php?f=44&t=57087\" rel=\"ugc\">https://forum.zdoom.org/viewtopic.php?f=44&t=57087</a>"

      assert {^expected, [], []} = Formatter.linkify(text)

      text = "https://en.wikipedia.org/wiki/Sophia_(Gnosticism)#Mythos_of_the_soul"

      expected =
        "<a href=\"https://en.wikipedia.org/wiki/Sophia_(Gnosticism)#Mythos_of_the_soul\" rel=\"ugc\">https://en.wikipedia.org/wiki/Sophia_(Gnosticism)#Mythos_of_the_soul</a>"

      assert {^expected, [], []} = Formatter.linkify(text)

      text = "https://www.google.co.jp/search?q=Nasim+Aghdam"

      expected =
        "<a href=\"https://www.google.co.jp/search?q=Nasim+Aghdam\" rel=\"ugc\">https://www.google.co.jp/search?q=Nasim+Aghdam</a>"

      assert {^expected, [], []} = Formatter.linkify(text)

      text = "https://en.wikipedia.org/wiki/Duff's_device"

      expected =
        "<a href=\"https://en.wikipedia.org/wiki/Duff's_device\" rel=\"ugc\">https://en.wikipedia.org/wiki/Duff's_device</a>"

      assert {^expected, [], []} = Formatter.linkify(text)

      text = "https://pleroma.com https://pleroma.com/sucks"

      expected =
        "<a href=\"https://pleroma.com\" rel=\"ugc\">https://pleroma.com</a> <a href=\"https://pleroma.com/sucks\" rel=\"ugc\">https://pleroma.com/sucks</a>"

      assert {^expected, [], []} = Formatter.linkify(text)

      text = "xmpp:contact@hacktivis.me"

      expected = "<a href=\"xmpp:contact@hacktivis.me\" rel=\"ugc\">xmpp:contact@hacktivis.me</a>"

      assert {^expected, [], []} = Formatter.linkify(text)

      text =
        "magnet:?xt=urn:btih:7ec9d298e91d6e4394d1379caf073c77ff3e3136&tr=udp%3A%2F%2Fopentor.org%3A2710&tr=udp%3A%2F%2Ftracker.blackunicorn.xyz%3A6969&tr=udp%3A%2F%2Ftracker.ccc.de%3A80&tr=udp%3A%2F%2Ftracker.coppersurfer.tk%3A6969&tr=udp%3A%2F%2Ftracker.leechers-paradise.org%3A6969&tr=udp%3A%2F%2Ftracker.openbittorrent.com%3A80&tr=wss%3A%2F%2Ftracker.btorrent.xyz&tr=wss%3A%2F%2Ftracker.fastcast.nz&tr=wss%3A%2F%2Ftracker.openwebtorrent.com"

      expected = "<a href=\"#{text}\" rel=\"ugc\">#{text}</a>"

      assert {^expected, [], []} = Formatter.linkify(text)
    end
  end

  describe "Formatter.linkify" do
    test "correctly finds mentions that contain the domain name" do
      _user = insert(:user, %{nickname: "lain"})
      _remote_user = insert(:user, %{nickname: "lain@lain.com", local: false})

      text = "hey @lain@lain.com what's up"

      {_text, mentions, []} = Formatter.linkify(text)
      [{username, user}] = mentions

      assert username == "@lain@lain.com"
      assert user.nickname == "lain@lain.com"
    end

    test "gives a replacement for user links, using local nicknames in user links text" do
      text = "@gsimg According to @archa_eme_, that is @daggsy. Also hello @archaeme@archae.me"
      gsimg = insert(:user, %{nickname: "gsimg"})

      archaeme =
        insert(:user,
          nickname: "archa_eme_",
          uri: "https://archeme/@archa_eme_"
        )

      archaeme_remote = insert(:user, %{nickname: "archaeme@archae.me"})

      {text, mentions, []} = Formatter.linkify(text)

      assert length(mentions) == 3

      expected_text =
        ~s(<span class="h-card"><a class="u-url mention" data-user="#{gsimg.id}" href="#{gsimg.ap_id}" rel="ugc">@<span>gsimg</span></a></span> According to <span class="h-card"><a class="u-url mention" data-user="#{archaeme.id}" href="#{"https://archeme/@archa_eme_"}" rel="ugc">@<span>archa_eme_</span></a></span>, that is @daggsy. Also hello <span class="h-card"><a class="u-url mention" data-user="#{archaeme_remote.id}" href="#{archaeme_remote.ap_id}" rel="ugc">@<span>archaeme</span></a></span>)

      assert expected_text == text
    end

    test "gives a replacement for user links when the user is using Osada" do
      {:ok, mike} = User.get_or_fetch("mike@osada.macgirvin.com")

      text = "@mike@osada.macgirvin.com test"

      {text, mentions, []} = Formatter.linkify(text)

      assert length(mentions) == 1

      expected_text =
        ~s(<span class="h-card"><a class="u-url mention" data-user="#{mike.id}" href="#{mike.ap_id}" rel="ugc">@<span>mike</span></a></span> test)

      assert expected_text == text
    end

    test "gives a replacement for single-character local nicknames" do
      text = "@o hi"
      o = insert(:user, %{nickname: "o"})

      {text, mentions, []} = Formatter.linkify(text)

      assert length(mentions) == 1

      expected_text =
        ~s(<span class="h-card"><a class="u-url mention" data-user="#{o.id}" href="#{o.ap_id}" rel="ugc">@<span>o</span></a></span> hi)

      assert expected_text == text
    end

    test "does not give a replacement for single-character local nicknames who don't exist" do
      text = "@a hi"

      expected_text = "@a hi"
      assert {^expected_text, [] = _mentions, [] = _tags} = Formatter.linkify(text)
    end

    test "given the 'safe_mention' option, it will only mention people in the beginning" do
      user = insert(:user)
      other_user = insert(:user)
      third_user = insert(:user)
      text = " @#{user.nickname} @#{other_user.nickname} hey dudes i hate @#{third_user.nickname}"
      {expected_text, mentions, [] = _tags} = Formatter.linkify(text, safe_mention: true)

      assert mentions == [{"@#{user.nickname}", user}, {"@#{other_user.nickname}", other_user}]

      assert expected_text ==
               ~s(<span class="h-card"><a class="u-url mention" data-user="#{user.id}" href="#{user.ap_id}" rel="ugc">@<span>#{user.nickname}</span></a></span> <span class="h-card"><a class="u-url mention" data-user="#{other_user.id}" href="#{other_user.ap_id}" rel="ugc">@<span>#{other_user.nickname}</span></a></span> hey dudes i hate <span class="h-card"><a class="u-url mention" data-user="#{third_user.id}" href="#{third_user.ap_id}" rel="ugc">@<span>#{third_user.nickname}</span></a></span>)
    end

    test "given the 'safe_mention' option, it will still work without any mention" do
      text = "A post without any mention"
      {expected_text, mentions, [] = _tags} = Formatter.linkify(text, safe_mention: true)

      assert mentions == []
      assert expected_text == text
    end

    test "given the 'safe_mention' option, it will keep text after newlines" do
      user = insert(:user)
      text = " @#{user.nickname}\n hey dude\n\nhow are you doing?"

      {expected_text, _, _} = Formatter.linkify(text, safe_mention: true)

      assert expected_text =~ "how are you doing?"
    end

    test "it can parse mentions and return the relevant users" do
      text =
        "@@gsimg According to @archaeme, that is @daggsy. Also hello @archaeme@archae.me and @o and @@@jimm"

      o = insert(:user, %{nickname: "o"})
      _jimm = insert(:user, %{nickname: "jimm"})
      _gsimg = insert(:user, %{nickname: "gsimg"})
      archaeme = insert(:user, %{nickname: "archaeme"})
      archaeme_remote = insert(:user, %{nickname: "archaeme@archae.me"})

      expected_mentions = [
        {"@archaeme", archaeme},
        {"@archaeme@archae.me", archaeme_remote},
        {"@o", o}
      ]

      assert {_text, ^expected_mentions, []} = Formatter.linkify(text)
    end

    test "it parses URL containing local mention" do
      _user = insert(:user, %{nickname: "lain"})

      text = "https://example.com/@lain"

      expected = ~S(<a href="https://example.com/@lain" rel="ugc">https://example.com/@lain</a>)

      assert {^expected, [], []} = Formatter.linkify(text)
    end

    test "it correctly parses angry face D:< with mention" do
      lain =
        insert(:user, %{
          nickname: "lain@lain.com",
          ap_id: "https://lain.com/users/lain",
          id: "9qrWmR0cKniB0YU0TA"
        })

      text = "@lain@lain.com D:<"

      expected_text =
        ~S(<span class="h-card"><a class="u-url mention" data-user="9qrWmR0cKniB0YU0TA" href="https://lain.com/users/lain" rel="ugc">@<span>lain</span></a></span> D:<)

      expected_mentions = [
        {"@lain@lain.com", lain}
      ]

      assert {^expected_text, ^expected_mentions, []} = Formatter.linkify(text)
    end

    test "correctly parses mentions in html" do
      text = "<p>@lain hello</p>"
      lain = insert(:user, %{nickname: "lain"})

      {text, mentions, []} = Formatter.linkify(text)

      assert length(mentions) == 1

      expected_text =
        ~s(<p><span class="h-card"><a class="u-url mention" data-user="#{lain.id}" href="#{lain.ap_id}" rel="ugc">@<span>lain</span></a></span> hello</p>)

      assert expected_text == text
    end

    test "correctly parses mentions on the last line of html" do
      text = "<p>Hello</p><p>@lain</p>"
      lain = insert(:user, %{nickname: "lain"})

      {text, mentions, []} = Formatter.linkify(text)

      assert length(mentions) == 1

      expected_text =
        ~s(<p>Hello</p><p><span class="h-card"><a class="u-url mention" data-user="#{lain.id}" href="#{lain.ap_id}" rel="ugc">@<span>lain</span></a></span></p>)

      assert expected_text == text
    end
  end

  describe ".parse_tags" do
    test "parses tags in the text" do
      text = "Here's a #Test. Maybe these are #working or not. What about #漢字? And #は｡"

      expected_tags = [
        {"#Test", "test"},
        {"#working", "working"},
        {"#は", "は"},
        {"#漢字", "漢字"}
      ]

      assert {_text, [], ^expected_tags} = Formatter.linkify(text)
    end

    test "parses tags in html" do
      text = "<p>This is a #test</p>"

      expected_tags = [
        {"#test", "test"}
      ]

      assert {_text, [], ^expected_tags} = Formatter.linkify(text)
    end

    test "parses multiple tags in html" do
      text = "<p>#tag1 #tag2 #tag3 #tag4</p>"

      expected_tags = [
        {"#tag1", "tag1"},
        {"#tag2", "tag2"},
        {"#tag3", "tag3"},
        {"#tag4", "tag4"}
      ]

      assert {_text, [], ^expected_tags} = Formatter.linkify(text)
    end

    test "parses tags on the last line of html" do
      text = "<p>This is a</p><p>#test</p>"

      expected_tags = [
        {"#test", "test"}
      ]

      assert {_text, [], ^expected_tags} = Formatter.linkify(text)
    end

    test "parses multiple tags on multiple lines in html" do
      text =
        "<p>testing...</p><p>#tag1 #tag2 #tag3 #tag4</p><p>paragraph</p><p>#tag5 #tag6 #tag7 #tag8</p>"

      expected_tags = [
        {"#tag1", "tag1"},
        {"#tag2", "tag2"},
        {"#tag3", "tag3"},
        {"#tag4", "tag4"},
        {"#tag5", "tag5"},
        {"#tag6", "tag6"},
        {"#tag7", "tag7"},
        {"#tag8", "tag8"}
      ]

      assert {_text, [], ^expected_tags} = Formatter.linkify(text)
    end
  end

  test "it escapes HTML in plain text" do
    text = "hello & world google.com/?a=b&c=d \n http://test.com/?a=b&c=d 1"
    expected = "hello &amp; world google.com/?a=b&c=d \n http://test.com/?a=b&c=d 1"

    assert Formatter.html_escape(text, "text/plain") == expected
  end

  describe "markdown_to_html/2" do
    test "keeps formatted link text when stripping email autolinks" do
      opts = %{breaks: true, strip_email_autolinks: true}

      assert Formatter.markdown_to_html("[**hello**](https://example.org/)", opts) ==
               ~s(<p><a href="https://example.org/"><strong>hello</strong></a></p>)

      assert Formatter.markdown_to_html("[`code`](https://example.org/)", opts) ==
               ~s(<p><a href="https://example.org/"><code class="inline">code</code></a></p>)
    end

    test "escapes link attribute values" do
      markdown = ~s|[click](http://example.com/?a=x" onerror="alert(1))|

      # The quotes are percent-encoded, so everything stays inside href.
      assert Formatter.markdown_to_html(markdown) ==
               ~s|<p><a href="http://example.com/?a=x%22 onerror=%22alert(1)">click</a></p>|
    end

    test "preserves entities without rendering dangerous raw HTML" do
      markdown = "safe &amp; sound<script>alert(1)</script>"

      assert Formatter.markdown_to_html(markdown) == "<p>safe &amp; sound</p>"

      assert Formatter.markdown_to_html("<div><script>alert(1)</script>safe</div>") ==
               "<div>safe</div>"

      assert Formatter.markdown_to_html("<ScRiPt>alert(1)</ScRiPt>") == ""
      assert Formatter.markdown_to_html("<BR>") == "<BR>"
    end

    test "escapes dangerous HTML inside code" do
      markdown = "`<script>&amp;</script>`"

      assert Formatter.markdown_to_html(markdown) ==
               ~s(<p><code class="inline">&lt;script&gt;&amp;amp;&lt;/script&gt;</code></p>)
    end

    test "rejects entity-encoded URL delimiters" do
      markdown = "[click](http://example.com/?a=x&#34;onerror=alert(1))"

      assert Formatter.markdown_to_html(markdown) ==
               ~s|<p><a href="http://example.com/?a=x%22onerror=alert(1)">click</a></p>|
    end

    test "keeps URLs with encoded quotes complete" do
      markdown = "[search](https://example.org/?q=&quot;cat&quot;&page=2)"

      assert Formatter.markdown_to_html(markdown) ==
               ~s(<p><a href="https://example.org/?q=%22cat%22&amp;page=2">search</a></p>)
    end

    test "strips email autolinks without changing explicit mailto links" do
      opts = %{strip_email_autolinks: true}

      assert Formatter.markdown_to_html("<foo@example.com>", opts) ==
               "<p>foo@example.com</p>"

      assert Formatter.markdown_to_html("[email](mailto:foo@example.com)", opts) ==
               ~s(<p><a href="mailto:foo@example.com">email</a></p>)
    end

    test "handles malformed Markdown that crashed older EarmarkParser releases" do
      assert Formatter.markdown_to_html("\\[{:}") == "<p>[{:}</p>"
    end

    test "handles a malformed Markdown regression corpus" do
      markdown_samples = [
        "- hello\n{:.claass .klass}",
        "[link](\"",
        "<script",
        "</style>",
        "~~~",
        "<![CDATA[",
        <<0>>
      ]

      for markdown <- markdown_samples do
        html = Formatter.markdown_to_html(markdown)

        assert is_binary(html)
        refute html =~ <<0>>
      end
    end

    test "falls back to escaped text when parsing fails" do
      markdown = "<script>alert(1)</script>\nnext"

      with_mock EarmarkParser, as_ast: fn _text, _opts -> raise "parser failed" end do
        assert Formatter.markdown_to_html(markdown, %{breaks: true}) ==
                 "<p>&lt;script&gt;alert(1)&lt;/script&gt;<br>next</p>"
      end
    end
  end
end