diff --git a/config/config.exs b/config/config.exs index 82bdd7750..2d38e3ebe 100644 --- a/config/config.exs +++ b/config/config.exs @@ -203,7 +203,8 @@ config :pleroma, :instance, "text/plain", "text/html", "text/markdown", - "text/bbcode" + "text/bbcode", + "text/x.misskeymarkdown" ], autofollowed_nicknames: [], autofollowing_nicknames: [], diff --git a/config/description.exs b/config/description.exs index c388d17c3..6e4348907 100644 --- a/config/description.exs +++ b/config/description.exs @@ -815,7 +815,8 @@ config :pleroma, :config_description, [ "text/plain", "text/html", "text/markdown", - "text/bbcode" + "text/bbcode", + "text/x.misskeymarkdown" ] }, %{ @@ -1394,7 +1395,13 @@ config :pleroma, :config_description, [ label: "Post Content Type", type: {:dropdown, :atom}, description: "Default post formatting option", - suggestions: ["text/plain", "text/html", "text/markdown", "text/bbcode"] + suggestions: [ + "text/plain", + "text/html", + "text/markdown", + "text/bbcode", + "text/x.misskeymarkdown" + ] }, %{ key: :redirectRootNoLogin, diff --git a/lib/pleroma/formatter.ex b/lib/pleroma/formatter.ex index 11d5af2fb..4bf2f6b95 100644 --- a/lib/pleroma/formatter.ex +++ b/lib/pleroma/formatter.ex @@ -127,6 +127,13 @@ defmodule Pleroma.Formatter do Earmark.as_html!(text, %Earmark.Options{compact_output: true, smartypants: false}) end + def markdown_to_html(text, opts) do + Earmark.as_html!( + text, + %Earmark.Options{compact_output: true, smartypants: false} |> Map.merge(opts) + ) + end + def html_escape({text, mentions, hashtags}, type) do {html_escape(text, type), mentions, hashtags} end @@ -135,6 +142,10 @@ defmodule Pleroma.Formatter do HTML.filter_tags(text) end + def html_escape(text, "text/x.misskeymarkdown") do + HTML.filter_tags(text) + end + def html_escape(text, "text/plain") do Regex.split(@link_regex, text, include_captures: true) |> Enum.map_every(2, fn chunk -> diff --git a/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex b/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex index c0626ce4d..ec9259f95 100644 --- a/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex @@ -6,6 +6,9 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidator do use Ecto.Schema alias Pleroma.EctoType.ActivityPub.ObjectValidators + alias Pleroma.HTML + alias Pleroma.User + alias Pleroma.Web.CommonAPI.Utils alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations alias Pleroma.Web.ActivityPub.Transmogrifier @@ -26,6 +29,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidator do end field(:replies, {:array, ObjectValidators.ObjectID}, default: []) + field(:source, :map) end def cast_and_apply(data) do @@ -80,6 +84,113 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidator do def fix_attachments(data), do: data + defp remote_mention_resolver( + %{"id" => ap_id, "tag" => tags}, + "@" <> nickname = mention, + buffer, + opts, + acc + ) + when is_binary(ap_id) and is_list(tags) do + initial_host = + ap_id + |> URI.parse() + |> Map.get(:host) + + with mention_tag when not is_nil(mention_tag) <- + Enum.find(tags, &mention_tag?(&1, mention, initial_host)), + href when is_binary(href) <- mention_tag["href"], + %User{} = user <- User.get_cached_by_ap_id(href) do + link = Pleroma.Formatter.mention_from_user(user, opts) + {link, %{acc | mentions: MapSet.put(acc.mentions, {"@" <> nickname, user})}} + else + _ -> {buffer, acc} + end + end + + defp remote_mention_resolver(_object, _mention, buffer, _opts, acc), do: {buffer, acc} + + defp mention_tag?(%{"type" => "Mention", "name" => name}, mention, initial_host) + when is_binary(name) do + name == mention || mention == "#{name}@#{initial_host}" + end + + defp mention_tag?(_tag, _mention, _initial_host), do: false + + defp scrub_content(%{"content" => content} = object) when is_binary(content) do + Map.put(object, "content", HTML.filter_tags(content)) + end + + defp scrub_content(object), do: object + + defp mfm_parse_limit do + min(Pleroma.Config.get([:instance, :limit]), Pleroma.Config.get([:instance, :remote_limit])) + end + + defp normalize_source(%{"source" => source} = object) when is_binary(source) do + object + |> Map.put("source", %{"content" => source}) + |> normalize_source() + end + + defp normalize_source(%{"source" => source} = object) when is_map(source) do + source = + case source["content"] do + content when is_binary(content) -> + if String.length(content) <= mfm_parse_limit() do + source + else + Map.delete(source, "content") + end + + nil -> + source + + _ -> + Map.delete(source, "content") + end + + Map.put(object, "source", source) + end + + defp normalize_source(object), do: object + + defp fix_misskey_content(%{"htmlMfm" => true, "content" => content} = object) + when is_binary(content) do + Map.put(object, "content", HTML.filter_tags(content)) + end + + defp fix_misskey_content(%{"htmlMfm" => true} = object), do: object + + defp fix_misskey_content( + %{"source" => %{"mediaType" => "text/x.misskeymarkdown", "content" => content}} = object + ) + when is_binary(content) do + mention_handler = fn nick, buffer, opts, acc -> + remote_mention_resolver(object, nick, buffer, opts, acc) + end + + {linked, _mentions, _tags} = + Utils.format_input(content, "text/x.misskeymarkdown", mention_handler: mention_handler) + + Map.put(object, "content", linked) + end + + defp fix_misskey_content(%{"source" => %{"mediaType" => "text/x.misskeymarkdown"}} = object), + do: scrub_content(object) + + defp fix_misskey_content(%{"_misskey_content" => content} = object) when is_binary(content) do + object + |> Map.put("source", %{ + "content" => content, + "mediaType" => "text/x.misskeymarkdown" + }) + |> Map.delete("_misskey_content") + |> fix_misskey_content() + end + + defp fix_misskey_content(object), do: object + defp fix(data) do data |> CommonFixes.fix_actor() @@ -88,6 +199,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidator do |> fix_tag() |> fix_replies() |> fix_attachments() + |> normalize_source() + |> fix_misskey_content() |> CommonFixes.fix_quote_url() |> CommonFixes.fix_likes() |> Transmogrifier.fix_emoji() diff --git a/lib/pleroma/web/activity_pub/object_validators/common_fields.ex b/lib/pleroma/web/activity_pub/object_validators/common_fields.ex index 22cf0cc05..9b8580200 100644 --- a/lib/pleroma/web/activity_pub/object_validators/common_fields.ex +++ b/lib/pleroma/web/activity_pub/object_validators/common_fields.ex @@ -32,6 +32,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFields do quote bind_quoted: binding() do field(:content, :string) field(:contentMap, ObjectValidators.ContentLanguageMap) + field(:htmlMfm, :boolean) field(:published, ObjectValidators.DateTime) field(:updated, ObjectValidators.DateTime) diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index 43c0f456d..0af4ceaf5 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -120,7 +120,8 @@ defmodule Pleroma.Web.ActivityPub.Utils do "https://www.w3.org/ns/activitystreams", "#{Endpoint.url()}/schemas/litepub-0.1.jsonld", %{ - "@language" => get_language(data) + "@language" => get_language(data), + "htmlMfm" => "https://w3id.org/fep/c16b#htmlMfm" } ] } diff --git a/lib/pleroma/web/common_api/activity_draft.ex b/lib/pleroma/web/common_api/activity_draft.ex index 16489663a..6072fff6b 100644 --- a/lib/pleroma/web/common_api/activity_draft.ex +++ b/lib/pleroma/web/common_api/activity_draft.ex @@ -317,6 +317,7 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do emoji = Map.merge(emoji, summary_emoji) + media_type = Utils.get_content_type(draft.params[:content_type]) {:ok, note_data, _meta} = Builder.note(draft) object = @@ -324,14 +325,18 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do |> Map.put("emoji", emoji) |> Map.put("source", %{ "content" => draft.status, - "mediaType" => Utils.get_content_type(draft.params[:content_type]) + "mediaType" => media_type }) + |> maybe_put("htmlMfm", true, media_type == "text/x.misskeymarkdown") |> Map.put("generator", draft.params[:generator]) |> Map.put("language", draft.language) %{draft | object: object} end + defp maybe_put(map, key, value, true), do: Map.put(map, key, value) + defp maybe_put(map, _key, _value, _condition), do: map + defp preview?(%__MODULE__{} = draft) do preview? = Pleroma.Web.Utils.Params.truthy_param?(draft.params[:preview]) %{draft | preview?: preview?} diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index 32572a721..26034d685 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -322,6 +322,14 @@ defmodule Pleroma.Web.CommonAPI.Utils do |> Formatter.linkify(options) end + def format_input(text, "text/x.misskeymarkdown", options) do + text + |> Formatter.markdown_to_html(%{breaks: true}) + |> safe_mfm_to_html() + |> Formatter.linkify(options) + |> Formatter.html_escape("text/x.misskeymarkdown") + end + def format_input(text, "text/markdown", options) do text |> Formatter.mentions_escape(options) @@ -330,6 +338,16 @@ defmodule Pleroma.Web.CommonAPI.Utils do |> Formatter.html_escape("text/html") end + defp safe_mfm_to_html(html) do + html + |> MfmParser.Parser.parse() + |> MfmParser.Encoder.to_html() + rescue + _ -> html + catch + _, _ -> html + end + def format_naive_asctime(date) do date |> DateTime.from_naive!("Etc/UTC") |> format_asctime end diff --git a/mix.exs b/mix.exs index bb7bfb9da..88f98d54e 100644 --- a/mix.exs +++ b/mix.exs @@ -160,6 +160,9 @@ defmodule Pleroma.Mixfile do {:sweet_xml, "~> 0.7.5"}, {:earmark, "1.4.46"}, {:bbcode_pleroma, "~> 0.2.0"}, + {:mfm_parser, + git: "https://akkoma.dev/AkkomaGang/mfm-parser.git", + ref: "360a30267a847810a63ab48f606ba227b2ca05f0"}, {:cors_plug, "~> 2.0"}, {:web_push_encryption, "~> 0.3.1"}, {:swoosh, "~> 1.16.12"}, diff --git a/mix.lock b/mix.lock index dd5f65f11..976a856b0 100644 --- a/mix.lock +++ b/mix.lock @@ -79,6 +79,7 @@ "makeup_erlang": {:hex, :makeup_erlang, "1.0.2", "03e1804074b3aa64d5fad7aa64601ed0fb395337b982d9bcf04029d68d51b6a7", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "af33ff7ef368d5893e4a267933e7744e46ce3cf1f61e2dccf53a111ed3aa3727"}, "meck": {:hex, :meck, "0.9.2", "85ccbab053f1db86c7ca240e9fc718170ee5bda03810a6292b5306bf31bae5f5", [:rebar3], [], "hexpm", "81344f561357dc40a8344afa53767c32669153355b626ea9fcbc8da6b3045826"}, "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"}, + "mfm_parser": {:git, "https://akkoma.dev/AkkomaGang/mfm-parser.git", "360a30267a847810a63ab48f606ba227b2ca05f0", [ref: "360a30267a847810a63ab48f606ba227b2ca05f0"]}, "mime": {:hex, :mime, "1.6.0", "dabde576a497cef4bbdd60aceee8160e02a6c89250d6c0b29e56c0dfb00db3d2", [:mix], [], "hexpm", "31a1a8613f8321143dde1dafc36006a17d28d02bdfecb9e95a880fa7aabd19a7"}, "mimerl": {:hex, :mimerl, "1.5.0", "f35aca6f23242339b3666e0ac0702379e362b469d0aea167f6cc713547e777ed", [:rebar3], [], "hexpm", "db648ce065bae14ea84ca8b5dd123f42f49417cef693541110bf6f9e9be9ecc4"}, "mint": {:hex, :mint, "1.7.1", "113fdb2b2f3b59e47c7955971854641c61f378549d73e829e1768de90fc1abf1", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1 or ~> 0.2.0 or ~> 1.0", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "fceba0a4d0f24301ddee3024ae116df1c3f4bb7a563a731f45fdfeb9d39a231b"}, diff --git a/priv/scrubbers/default.ex b/priv/scrubbers/default.ex index 0defdc74e..342ef9944 100644 --- a/priv/scrubbers/default.ex +++ b/priv/scrubbers/default.ex @@ -82,12 +82,50 @@ defmodule Pleroma.HTML.Scrubber.Default do "recipients-inline", "quote-inline", "invisible", - "ellipsis" + "ellipsis", + "mfm-center", + "mfm-flip", + "mfm-font", + "mfm-blur", + "mfm-rotate", + "mfm-x2", + "mfm-x3", + "mfm-x4", + "mfm-position", + "mfm-scale", + "mfm-fg", + "mfm-bg", + "mfm-jelly", + "mfm-twitch", + "mfm-shake", + "mfm-spin", + "mfm-jump", + "mfm-bounce", + "mfm-rainbow", + "mfm-tada", + "mfm-sparkle" ]) Meta.allow_tag_with_this_attribute_values(:p, "class", ["quote-inline"]) - Meta.allow_tag_with_these_attributes(:span, ["lang"]) + Meta.allow_tag_with_these_attributes(:span, [ + "lang", + "data-mfm-h", + "data-mfm-v", + "data-mfm-x", + "data-mfm-y", + "data-mfm-alternate", + "data-mfm-speed", + "data-mfm-deg", + "data-mfm-left", + "data-mfm-serif", + "data-mfm-monospace", + "data-mfm-cursive", + "data-mfm-fantasy", + "data-mfm-emoji", + "data-mfm-math", + "data-mfm-color" + ]) Meta.allow_tag_with_this_attribute_values(:code, "class", ["inline"]) diff --git a/test/mix/tasks/pleroma/config_test.exs b/test/mix/tasks/pleroma/config_test.exs index ef1adc235..f672d8c13 100644 --- a/test/mix/tasks/pleroma/config_test.exs +++ b/test/mix/tasks/pleroma/config_test.exs @@ -144,7 +144,13 @@ defmodule Mix.Tasks.Pleroma.ConfigTest do quarantined_instances: [], managed_config: true, static_dir: "instance/static/", - allowed_post_formats: ["text/plain", "text/html", "text/markdown", "text/bbcode"], + allowed_post_formats: [ + "text/plain", + "text/html", + "text/markdown", + "text/bbcode", + "text/x.misskeymarkdown" + ], autofollowed_nicknames: [], max_pinned_statuses: 1, attachment_links: false, diff --git a/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs b/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs index c32811c5b..bf9c70fb6 100644 --- a/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs +++ b/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs @@ -149,6 +149,171 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidatorTest %{valid?: true} = ArticleNotePageValidator.cast_and_validate(note) end + test "a Misskey MFM note is rendered from source content" do + user = insert(:user, ap_id: "https://misskey.example/users/alice") + + note = %{ + "id" => "https://misskey.example/notes/1", + "type" => "Note", + "actor" => user.ap_id, + "attributedTo" => user.ap_id, + "to" => ["https://www.w3.org/ns/activitystreams#Public"], + "cc" => [], + "content" => "original content", + "context" => Utils.generate_context_id(), + "source" => %{ + "content" => "$[spin.speed=1s mfm goes here] ", + "mediaType" => "text/x.misskeymarkdown" + } + } + + %{valid?: true, changes: %{content: content, source: source}} = + ArticleNotePageValidator.cast_and_validate(note) + + assert source["mediaType"] == "text/x.misskeymarkdown" + assert content =~ ~s(class="mfm-spin") + assert content =~ ~s(data-mfm-speed="1s") + assert content =~ "mfm goes here" + refute content =~ "original content" + refute content =~ "", + "htmlMfm" => true, + "context" => Utils.generate_context_id(), + "source" => %{ + "content" => String.duplicate("x", 5_001), + "mediaType" => "text/x.misskeymarkdown" + } + } + + %{valid?: true, changes: %{content: content, htmlMfm: true, source: source}} = + ArticleNotePageValidator.cast_and_validate(note) + + assert content == "already renderedalert('xss')" + refute Map.has_key?(source, "content") + end + test "a Note with validated likes collection validates" do insert(:user, ap_id: "https://pol.social/users/mkljczk") diff --git a/test/pleroma/web/activity_pub/utils_test.exs b/test/pleroma/web/activity_pub/utils_test.exs index 3b77f0867..93234a015 100644 --- a/test/pleroma/web/activity_pub/utils_test.exs +++ b/test/pleroma/web/activity_pub/utils_test.exs @@ -180,7 +180,8 @@ defmodule Pleroma.Web.ActivityPub.UtilsTest do "https://www.w3.org/ns/activitystreams", "http://localhost:4001/schemas/litepub-0.1.jsonld", %{ - "@language" => "und" + "@language" => "und", + "htmlMfm" => "https://w3id.org/fep/c16b#htmlMfm" } ] } @@ -192,7 +193,8 @@ defmodule Pleroma.Web.ActivityPub.UtilsTest do "https://www.w3.org/ns/activitystreams", "http://localhost:4001/schemas/litepub-0.1.jsonld", %{ - "@language" => "pl" + "@language" => "pl", + "htmlMfm" => "https://w3id.org/fep/c16b#htmlMfm" } ] } diff --git a/test/pleroma/web/common_api_test.exs b/test/pleroma/web/common_api_test.exs index 017fac696..ea1795c0b 100644 --- a/test/pleroma/web/common_api_test.exs +++ b/test/pleroma/web/common_api_test.exs @@ -709,6 +709,47 @@ defmodule Pleroma.Web.CommonAPITest do assert object.data["source"]["content"] == post end + test "it renders MFM posts and marks their ActivityPub representation" do + user = insert(:user) + + post = "
$[spin.speed=1s 13:37]
" + + {:ok, activity} = + CommonAPI.post(user, %{ + status: post, + content_type: "text/x.misskeymarkdown" + }) + + object = Object.normalize(activity, fetch: false) + + assert object.data["htmlMfm"] == true + + assert object.data["source"] == %{ + "content" => post, + "mediaType" => "text/x.misskeymarkdown" + } + + assert object.data["content"] =~ ~s(class="mfm-spin") + assert object.data["content"] =~ ~s(data-mfm-speed="1s") + assert object.data["content"] =~ "13:37" + refute object.data["content"] =~ "scrub-this" + end + + test "it falls back safely for malformed MFM" do + user = insert(:user) + + {:ok, activity} = + CommonAPI.post(user, %{ + status: "$[spin.speed=1s=boom malformed]", + content_type: "text/x.misskeymarkdown" + }) + + object = Object.normalize(activity, fetch: false) + + refute object.data["content"] =~ ~s(class="mfm-spin") + assert object.data["content"] =~ "malformed" + end + test "it does not allow replies to direct messages that are not direct messages themselves" do user = insert(:user)