Add backend MFM support

lib/pleroma/formatter.ex

@@ -127,6 +127,13 @@ defmodule Pleroma.Formatter do
     Earmark.as_html!(text, %Earmark.Options{compact_output: true, smartypants: false})
   end
 
+  def markdown_to_html(text, opts) do
+    Earmark.as_html!(
+      text,
+      %Earmark.Options{compact_output: true, smartypants: false} |> Map.merge(opts)
+    )
+  end
+
   def html_escape({text, mentions, hashtags}, type) do
     {html_escape(text, type), mentions, hashtags}
   end
@@ -135,6 +142,10 @@ defmodule Pleroma.Formatter do
     HTML.filter_tags(text)
   end
 
+  def html_escape(text, "text/x.misskeymarkdown") do
+    HTML.filter_tags(text)
+  end
+
   def html_escape(text, "text/plain") do
     Regex.split(@link_regex, text, include_captures: true)
     |> Enum.map_every(2, fn chunk ->

lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex

@@ -6,6 +6,9 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidator do
   use Ecto.Schema
 
   alias Pleroma.EctoType.ActivityPub.ObjectValidators
+  alias Pleroma.HTML
+  alias Pleroma.User
+  alias Pleroma.Web.CommonAPI.Utils
   alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes
   alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations
   alias Pleroma.Web.ActivityPub.Transmogrifier
@@ -26,6 +29,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidator do
     end
 
     field(:replies, {:array, ObjectValidators.ObjectID}, default: [])
+    field(:source, :map)
   end
 
   def cast_and_apply(data) do
@@ -80,6 +84,113 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidator do
 
   def fix_attachments(data), do: data
 
+  defp remote_mention_resolver(
+         %{"id" => ap_id, "tag" => tags},
+         "@" <> nickname = mention,
+         buffer,
+         opts,
+         acc
+       )
+       when is_binary(ap_id) and is_list(tags) do
+    initial_host =
+      ap_id
+      |> URI.parse()
+      |> Map.get(:host)
+
+    with mention_tag when not is_nil(mention_tag) <-
+           Enum.find(tags, &mention_tag?(&1, mention, initial_host)),
+         href when is_binary(href) <- mention_tag["href"],
+         %User{} = user <- User.get_cached_by_ap_id(href) do
+      link = Pleroma.Formatter.mention_from_user(user, opts)
+      {link, %{acc | mentions: MapSet.put(acc.mentions, {"@" <> nickname, user})}}
+    else
+      _ -> {buffer, acc}
+    end
+  end
+
+  defp remote_mention_resolver(_object, _mention, buffer, _opts, acc), do: {buffer, acc}
+
+  defp mention_tag?(%{"type" => "Mention", "name" => name}, mention, initial_host)
+       when is_binary(name) do
+    name == mention || mention == "#{name}@#{initial_host}"
+  end
+
+  defp mention_tag?(_tag, _mention, _initial_host), do: false
+
+  defp scrub_content(%{"content" => content} = object) when is_binary(content) do
+    Map.put(object, "content", HTML.filter_tags(content))
+  end
+
+  defp scrub_content(object), do: object
+
+  defp mfm_parse_limit do
+    min(Pleroma.Config.get([:instance, :limit]), Pleroma.Config.get([:instance, :remote_limit]))
+  end
+
+  defp normalize_source(%{"source" => source} = object) when is_binary(source) do
+    object
+    |> Map.put("source", %{"content" => source})
+    |> normalize_source()
+  end
+
+  defp normalize_source(%{"source" => source} = object) when is_map(source) do
+    source =
+      case source["content"] do
+        content when is_binary(content) ->
+          if String.length(content) <= mfm_parse_limit() do
+            source
+          else
+            Map.delete(source, "content")
+          end
+
+        nil ->
+          source
+
+        _ ->
+          Map.delete(source, "content")
+      end
+
+    Map.put(object, "source", source)
+  end
+
+  defp normalize_source(object), do: object
+
+  defp fix_misskey_content(%{"htmlMfm" => true, "content" => content} = object)
+       when is_binary(content) do
+    Map.put(object, "content", HTML.filter_tags(content))
+  end
+
+  defp fix_misskey_content(%{"htmlMfm" => true} = object), do: object
+
+  defp fix_misskey_content(
+         %{"source" => %{"mediaType" => "text/x.misskeymarkdown", "content" => content}} = object
+       )
+       when is_binary(content) do
+    mention_handler = fn nick, buffer, opts, acc ->
+      remote_mention_resolver(object, nick, buffer, opts, acc)
+    end
+
+    {linked, _mentions, _tags} =
+      Utils.format_input(content, "text/x.misskeymarkdown", mention_handler: mention_handler)
+
+    Map.put(object, "content", linked)
+  end
+
+  defp fix_misskey_content(%{"source" => %{"mediaType" => "text/x.misskeymarkdown"}} = object),
+    do: scrub_content(object)
+
+  defp fix_misskey_content(%{"_misskey_content" => content} = object) when is_binary(content) do
+    object
+    |> Map.put("source", %{
+      "content" => content,
+      "mediaType" => "text/x.misskeymarkdown"
+    })
+    |> Map.delete("_misskey_content")
+    |> fix_misskey_content()
+  end
+
+  defp fix_misskey_content(object), do: object
+
   defp fix(data) do
     data
     |> CommonFixes.fix_actor()
@@ -88,6 +199,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidator do
     |> fix_tag()
     |> fix_replies()
     |> fix_attachments()
+    |> normalize_source()
+    |> fix_misskey_content()
     |> CommonFixes.fix_quote_url()
     |> CommonFixes.fix_likes()
     |> Transmogrifier.fix_emoji()

lib/pleroma/web/activity_pub/object_validators/common_fields.ex

@@ -32,6 +32,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFields do
     quote bind_quoted: binding() do
       field(:content, :string)
       field(:contentMap, ObjectValidators.ContentLanguageMap)
+      field(:htmlMfm, :boolean)
 
       field(:published, ObjectValidators.DateTime)
       field(:updated, ObjectValidators.DateTime)

lib/pleroma/web/activity_pub/utils.ex

@@ -120,7 +120,8 @@ defmodule Pleroma.Web.ActivityPub.Utils do
         "https://www.w3.org/ns/activitystreams",
         "#{Endpoint.url()}/schemas/litepub-0.1.jsonld",
         %{
-          "@language" => get_language(data)
+          "@language" => get_language(data),
+          "htmlMfm" => "https://w3id.org/fep/c16b#htmlMfm"
         }
       ]
     }

lib/pleroma/web/common_api/activity_draft.ex

@@ -317,6 +317,7 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do
 
     emoji = Map.merge(emoji, summary_emoji)
 
+    media_type = Utils.get_content_type(draft.params[:content_type])
     {:ok, note_data, _meta} = Builder.note(draft)
 
     object =
@@ -324,14 +325,18 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do
       |> Map.put("emoji", emoji)
       |> Map.put("source", %{
         "content" => draft.status,
-        "mediaType" => Utils.get_content_type(draft.params[:content_type])
+        "mediaType" => media_type
       })
+      |> maybe_put("htmlMfm", true, media_type == "text/x.misskeymarkdown")
       |> Map.put("generator", draft.params[:generator])
       |> Map.put("language", draft.language)
 
     %{draft | object: object}
   end
 
+  defp maybe_put(map, key, value, true), do: Map.put(map, key, value)
+  defp maybe_put(map, _key, _value, _condition), do: map
+
   defp preview?(%__MODULE__{} = draft) do
     preview? = Pleroma.Web.Utils.Params.truthy_param?(draft.params[:preview])
     %{draft | preview?: preview?}

lib/pleroma/web/common_api/utils.ex

@@ -322,6 +322,14 @@ defmodule Pleroma.Web.CommonAPI.Utils do
     |> Formatter.linkify(options)
   end
 
+  def format_input(text, "text/x.misskeymarkdown", options) do
+    text
+    |> Formatter.markdown_to_html(%{breaks: true})
+    |> safe_mfm_to_html()
+    |> Formatter.linkify(options)
+    |> Formatter.html_escape("text/x.misskeymarkdown")
+  end
+
   def format_input(text, "text/markdown", options) do
     text
     |> Formatter.mentions_escape(options)
@@ -330,6 +338,16 @@ defmodule Pleroma.Web.CommonAPI.Utils do
     |> Formatter.html_escape("text/html")
   end
 
+  defp safe_mfm_to_html(html) do
+    html
+    |> MfmParser.Parser.parse()
+    |> MfmParser.Encoder.to_html()
+  rescue
+    _ -> html
+  catch
+    _, _ -> html
+  end
+
   def format_naive_asctime(date) do
     date |> DateTime.from_naive!("Etc/UTC") |> format_asctime
   end