Merge branch 'post-languages' into 'develop'

Allow to specify post language

See merge request pleroma/pleroma!3940

lib/pleroma/web/activity_pub/object_validator.ex

@@ -26,6 +26,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
   alias Pleroma.Web.ActivityPub.ObjectValidators.AudioImageVideoValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.BlockValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.ChatMessageValidator
+  alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes
   alias Pleroma.Web.ActivityPub.ObjectValidators.CreateChatMessageValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.DeleteValidator
@@ -115,7 +116,10 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
         meta
       )
       when objtype in ~w[Question Answer Audio Video Image Event Article Note Page] do
-    with {:ok, object_data} <- cast_and_apply_and_stringify_with_history(object),
+    with {:ok, object_data} <-
+           object
+           |> CommonFixes.maybe_add_language_from_activity(create_activity)
+           |> cast_and_apply_and_stringify_with_history(),
          meta = Keyword.put(meta, :object_data, object_data),
          {:ok, create_activity} <-
            create_activity
@@ -165,7 +169,11 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
       )
       when objtype in ~w[Question Answer Audio Video Event Article Note Page] do
     with {_, false} <- {:local, Access.get(meta, :local, false)},
-         {_, {:ok, object_data, _}} <- {:object_validation, validate(object, meta)},
+         {_, {:ok, object_data, _}} <-
+           {:object_validation,
+            object
+            |> CommonFixes.maybe_add_language_from_activity(update_activity)
+            |> validate(meta)},
          meta = Keyword.put(meta, :object_data, object_data),
          {:ok, update_activity} <-
            update_activity

lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex

@@ -30,7 +30,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidator do
 
   def cast_and_apply(data) do
     data
-    |> cast_data
+    |> cast_data()
     |> apply_action(:insert)
   end
 
@@ -88,6 +88,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidator do
     |> CommonFixes.fix_likes()
     |> Transmogrifier.fix_emoji()
     |> Transmogrifier.fix_content_map()
+    |> CommonFixes.maybe_add_language()
+    |> CommonFixes.maybe_add_content_map()
   end
 
   def changeset(struct, data) do

lib/pleroma/web/activity_pub/object_validators/common_fields.ex

@@ -31,6 +31,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFields do
   defmacro object_fields do
     quote bind_quoted: binding() do
       field(:content, :string)
+      field(:contentMap, ObjectValidators.ContentLanguageMap)
 
       field(:published, ObjectValidators.DateTime)
       field(:updated, ObjectValidators.DateTime)
@@ -58,6 +59,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFields do
       field(:like_count, :integer, default: 0)
       field(:announcement_count, :integer, default: 0)
       field(:quotes_count, :integer, default: 0)
+      field(:language, ObjectValidators.LanguageCode)
       field(:inReplyTo, ObjectValidators.ObjectID)
       field(:quoteUrl, ObjectValidators.ObjectID)
       field(:url, ObjectValidators.BareUri)

lib/pleroma/web/activity_pub/object_validators/common_fixes.ex

@@ -11,6 +11,11 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes do
   alias Pleroma.Web.ActivityPub.Transmogrifier
   alias Pleroma.Web.ActivityPub.Utils
 
+  import Pleroma.EctoType.ActivityPub.ObjectValidators.LanguageCode,
+    only: [good_locale_code?: 1]
+
+  import Pleroma.Web.Utils.Guards, only: [not_empty_string: 1]
+
   require Pleroma.Constants
 
   def cast_and_filter_recipients(message, field, follower_collection, field_fallback \\ []) do
@@ -132,4 +137,60 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes do
   end
 
   def object_link_tag?(_), do: false
+
+  def maybe_add_language_from_activity(object, activity) do
+    language = get_language_from_context(activity)
+
+    if language do
+      Map.put(object, "language", language)
+    else
+      object
+    end
+  end
+
+  def maybe_add_language(object) do
+    language =
+      [
+        get_language_from_context(object),
+        get_language_from_content_map(object)
+      ]
+      |> Enum.find(&good_locale_code?(&1))
+
+    if language do
+      Map.put(object, "language", language)
+    else
+      object
+    end
+  end
+
+  defp get_language_from_context(%{"@context" => context}) when is_list(context) do
+    case context
+         |> Enum.find(fn
+           %{"@language" => language} -> language != "und"
+           _ -> nil
+         end) do
+      %{"@language" => language} -> language
+      _ -> nil
+    end
+  end
+
+  defp get_language_from_context(_), do: nil
+
+  defp get_language_from_content_map(%{"contentMap" => content_map, "content" => source_content}) do
+    content_groups = Map.to_list(content_map)
+
+    case Enum.find(content_groups, fn {_, content} -> content == source_content end) do
+      {language, _} -> language
+      _ -> nil
+    end
+  end
+
+  defp get_language_from_content_map(_), do: nil
+
+  def maybe_add_content_map(%{"language" => language, "content" => content} = object)
+      when not_empty_string(language) do
+    Map.put(object, "contentMap", Map.put(%{}, language, content))
+  end
+
+  def maybe_add_content_map(object), do: object
 end

lib/pleroma/web/activity_pub/object_validators/event_validator.ex

@@ -28,7 +28,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.EventValidator do
 
   def cast_and_apply(data) do
     data
-    |> cast_data
+    |> cast_data()
     |> apply_action(:insert)
   end
 
@@ -38,6 +38,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.EventValidator do
     |> validate_data()
   end
 
+  @spec cast_data(map()) :: map()
   def cast_data(data) do
     %__MODULE__{}
     |> changeset(data)
@@ -49,6 +50,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.EventValidator do
     |> CommonFixes.fix_object_defaults()
     |> CommonFixes.fix_likes()
     |> Transmogrifier.fix_emoji()
+    |> CommonFixes.maybe_add_language()
+    |> CommonFixes.maybe_add_content_map()
   end
 
   def changeset(struct, data) do

lib/pleroma/web/activity_pub/transmogrifier.ex

@@ -16,12 +16,14 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.ActivityPub.Builder
   alias Pleroma.Web.ActivityPub.ObjectValidator
+  alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes
   alias Pleroma.Web.ActivityPub.Pipeline
   alias Pleroma.Web.ActivityPub.Utils
   alias Pleroma.Web.ActivityPub.Visibility
   alias Pleroma.Web.Federator
 
   import Ecto.Query
+  import Pleroma.Web.Utils.Guards, only: [not_empty_string: 1]
 
   require Pleroma.Constants
 
@@ -166,7 +168,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
 
   def fix_quote_url_and_maybe_fetch(object, options \\ []) do
     quote_url =
-      case Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes.fix_quote_url(object) do
+      case CommonFixes.fix_quote_url(object) do
         %{"quoteUrl" => quote_url} -> quote_url
         _ -> nil
       end
@@ -336,6 +338,9 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
 
   def fix_tag(object), do: object
 
+  # prefer content over contentMap
+  def fix_content_map(%{"content" => content} = object) when not_empty_string(content), do: object
+
   # content map usually only has one language so this will do for now.
   def fix_content_map(%{"contentMap" => content_map} = object) do
     content_groups = Map.to_list(content_map)
@@ -716,6 +721,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
     |> set_reply_to_uri
     |> set_quote_url
     |> set_replies
+    |> CommonFixes.maybe_add_content_map()
     |> strip_internal_fields
     |> strip_internal_tags
     |> set_type
@@ -750,12 +756,11 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
       object_id
       |> Object.normalize(fetch: false)
       |> Map.get(:data)
-      |> prepare_object
 
     data =
       data
-      |> Map.put("object", object)
-      |> Map.merge(Utils.make_json_ld_header())
+      |> Map.put("object", prepare_object(object))
+      |> Map.merge(Utils.make_json_ld_header(object))
       |> Map.delete("bcc")
 
     {:ok, data}
@@ -763,14 +768,10 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
 
   def prepare_outgoing(%{"type" => "Update", "object" => %{"type" => objtype} = object} = data)
       when objtype in Pleroma.Constants.updatable_object_types() do
-    object =
-      object
-      |> prepare_object
-
     data =
       data
-      |> Map.put("object", object)
-      |> Map.merge(Utils.make_json_ld_header())
+      |> Map.put("object", prepare_object(object))
+      |> Map.merge(Utils.make_json_ld_header(object))
       |> Map.delete("bcc")
 
     {:ok, data}
@@ -840,7 +841,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
       data
       |> strip_internal_fields
       |> maybe_fix_object_url
-      |> Map.merge(Utils.make_json_ld_header())
+      |> Map.merge(Utils.make_json_ld_header(data))
 
     {:ok, data}
   end

lib/pleroma/web/activity_pub/utils.ex

@@ -20,6 +20,7 @@ defmodule Pleroma.Web.ActivityPub.Utils do
   alias Pleroma.Web.Router.Helpers
 
   import Ecto.Query
+  import Pleroma.Web.Utils.Guards, only: [not_empty_string: 1]
 
   require Logger
   require Pleroma.Constants
@@ -109,18 +110,24 @@ defmodule Pleroma.Web.ActivityPub.Utils do
     end
   end
 
-  def make_json_ld_header do
+  def make_json_ld_header(data \\ %{}) do
     %{
       "@context" => [
         "https://www.w3.org/ns/activitystreams",
         "#{Endpoint.url()}/schemas/litepub-0.1.jsonld",
         %{
-          "@language" => "und"
+          "@language" => get_language(data)
         }
       ]
     }
   end
 
+  defp get_language(%{"language" => language}) when not_empty_string(language) do
+    language
+  end
+
+  defp get_language(_), do: "und"
+
   def make_date do
     DateTime.utc_now() |> DateTime.to_iso8601()
   end

lib/pleroma/web/activity_pub/views/object_view.ex

@@ -9,7 +9,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectView do
   alias Pleroma.Web.ActivityPub.Transmogrifier
 
   def render("object.json", %{object: %Object{} = object}) do
-    base = Pleroma.Web.ActivityPub.Utils.make_json_ld_header()
+    base = Pleroma.Web.ActivityPub.Utils.make_json_ld_header(object.data)
 
     additional = Transmogrifier.prepare_object(object.data)
     Map.merge(base, additional)
@@ -17,7 +17,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectView do
 
   def render("object.json", %{object: %Activity{data: %{"type" => activity_type}} = activity})
       when activity_type in ["Create", "Listen"] do
-    base = Pleroma.Web.ActivityPub.Utils.make_json_ld_header()
+    base = Pleroma.Web.ActivityPub.Utils.make_json_ld_header(activity.data)
     object = Object.normalize(activity, fetch: false)
 
     additional =
@@ -28,7 +28,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectView do
   end
 
   def render("object.json", %{object: %Activity{} = activity}) do
-    base = Pleroma.Web.ActivityPub.Utils.make_json_ld_header()
+    base = Pleroma.Web.ActivityPub.Utils.make_json_ld_header(activity.data)
     object_id = Object.normalize(activity, id_only: true)
 
     additional =

lib/pleroma/web/common_api/activity_draft.ex

@@ -11,6 +11,9 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do
   alias Pleroma.Web.CommonAPI
   alias Pleroma.Web.CommonAPI.Utils
 
+  import Pleroma.EctoType.ActivityPub.ObjectValidators.LanguageCode,
+    only: [good_locale_code?: 1]
+
   import Pleroma.Web.Gettext
   import Pleroma.Web.Utils.Guards, only: [not_empty_string: 1]
 
@@ -38,6 +41,7 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do
             cc: [],
             context: nil,
             sensitive: false,
+            language: nil,
             object: nil,
             preview?: false,
             changes: %{}
@@ -64,6 +68,7 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do
     |> content()
     |> with_valid(&to_and_cc/1)
     |> with_valid(&context/1)
+    |> with_valid(&language/1)
     |> sensitive()
     |> with_valid(&object/1)
     |> preview?()
@@ -249,6 +254,16 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do
     %__MODULE__{draft | sensitive: sensitive}
   end
 
+  defp language(draft) do
+    language = draft.params[:language]
+
+    if good_locale_code?(language) do
+      %__MODULE__{draft | language: language}
+    else
+      draft
+    end
+  end
+
   defp object(draft) do
     emoji = Map.merge(Pleroma.Emoji.Formatter.get_emoji_map(draft.full_payload), draft.emoji)
 
@@ -288,6 +303,7 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do
         "mediaType" => Utils.get_content_type(draft.params[:content_type])
       })
       |> Map.put("generator", draft.params[:generator])
+      |> Map.put("language", draft.language)
 
     %__MODULE__{draft | object: object}
   end

lib/pleroma/web/mastodon_api/views/status_view.ex

@@ -227,7 +227,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
       mentions: mentions,
       tags: reblogged[:tags] || [],
       application: build_application(object.data["generator"]),
-      language: nil,
+      language: get_language(object),
       emojis: [],
       pleroma: %{
         local: activity.local,
@@ -445,7 +445,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
       mentions: mentions,
       tags: build_tags(tags),
       application: build_application(object.data["generator"]),
-      language: nil,
+      language: get_language(object),
       emojis: build_emojis(object.data["emoji"]),
       pleroma: %{
         local: activity.local,
@@ -829,6 +829,10 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
     Utils.get_content_type(nil)
   end
 
+  defp get_language(%{data: %{"language" => "und"}}), do: nil
+
+  defp get_language(object), do: object.data["language"]
+
   defp proxied_url(url, page_url_data) do
     if is_binary(url) do
       build_image_url(URI.parse(url), page_url_data) |> MediaProxy.url()