Merge branch 'translate-posts' into 'develop'

Support translation providers (DeepL, LibreTranslate)

See merge request pleroma/pleroma!4102

lib/pleroma/application.ex

@@ -172,7 +172,8 @@ defmodule Pleroma.Application do
         limit: 500_000
       ),
       build_cachex("rel_me", limit: 2500),
-      build_cachex("host_meta", default_ttl: :timer.minutes(120), limit: 5000)
+      build_cachex("host_meta", default_ttl: :timer.minutes(120), limit: 5_000),
+      build_cachex("translations", default_ttl: :timer.hours(24), limit: 5_000)
     ]
   end
 

lib/pleroma/application_requirements.ex

@@ -202,11 +202,24 @@ defmodule Pleroma.ApplicationRequirements do
         false
       end
 
+    translation_commands_status =
+      if Pleroma.Language.Translation.missing_dependencies() == [] do
+        true
+      else
+        Logger.error(
+          "The following dependencies required by the currently enabled " <>
+            "translation provider are not installed: " <>
+            inspect(Pleroma.Language.Translation.missing_dependencies())
+        )
+
+        false
+      end
+
     if Enum.all?(
          [
            preview_proxy_commands_status,
-           language_detector_commands_status
-           | filter_commands_statuses
+           language_detector_commands_status,
+           translation_commands_status | filter_commands_statuses
          ],
          & &1
        ) do

lib/pleroma/language/translation.ex

@@ -0,0 +1,127 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Language.Translation do
+  @cachex Pleroma.Config.get([:cachex, :provider], Cachex)
+
+  def configured? do
+    provider = get_provider()
+
+    !!provider and provider.configured?
+  end
+
+  def missing_dependencies do
+    provider = get_provider()
+
+    if provider do
+      provider.missing_dependencies()
+    else
+      []
+    end
+  end
+
+  def translate(text, source_language, target_language) do
+    cache_key = get_cache_key(text, source_language, target_language)
+
+    case @cachex.get(:translations_cache, cache_key) do
+      {:ok, nil} ->
+        provider = get_provider()
+
+        result =
+          if !configured?() do
+            {:error, :not_found}
+          else
+            provider.translate(text, source_language, target_language)
+            |> scrub_html()
+          end
+
+        store_result(result, cache_key)
+
+        result
+
+      {:ok, result} ->
+        {:ok, result}
+
+      {:error, error} ->
+        {:error, error}
+    end
+  end
+
+  def supported_languages(type) when type in [:source, :target] do
+    provider = get_provider()
+
+    cache_key = "#{type}_languages/#{provider.name()}"
+
+    case @cachex.get(:translations_cache, cache_key) do
+      {:ok, nil} ->
+        result =
+          if !configured?() do
+            {:error, :not_found}
+          else
+            provider.supported_languages(type)
+          end
+
+        store_result(result, cache_key)
+
+        result
+
+      {:ok, result} ->
+        {:ok, result}
+
+      {:error, error} ->
+        {:error, error}
+    end
+  end
+
+  def languages_matrix do
+    provider = get_provider()
+
+    cache_key = "languages_matrix/#{provider.name()}"
+
+    case @cachex.get(:translations_cache, cache_key) do
+      {:ok, nil} ->
+        result =
+          if !configured?() do
+            {:error, :not_found}
+          else
+            provider.languages_matrix()
+          end
+
+        store_result(result, cache_key)
+
+        result
+
+      {:ok, result} ->
+        {:ok, result}
+
+      {:error, error} ->
+        {:error, error}
+    end
+  end
+
+  defp get_provider, do: Pleroma.Config.get([__MODULE__, :provider])
+
+  defp get_cache_key(text, source_language, target_language) do
+    "#{source_language}/#{target_language}/#{content_hash(text)}"
+  end
+
+  defp store_result({:ok, result}, cache_key) do
+    @cachex.put(:translations_cache, cache_key, result)
+  end
+
+  defp store_result(_, _), do: nil
+
+  defp content_hash(text), do: :crypto.hash(:sha256, text) |> Base.encode64()
+
+  defp scrub_html({:ok, %{content: content} = result}) when is_binary(content) do
+    scrubbers = Pleroma.Config.get([:markup, :scrub_policy])
+
+    content
+    |> Pleroma.HTML.filter_tags(scrubbers)
+
+    {:ok, %{result | content: content}}
+  end
+
+  defp scrub_html(result), do: result
+end

lib/pleroma/language/translation/deepl.ex

@@ -0,0 +1,123 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Language.Translation.Deepl do
+  import Pleroma.Web.Utils.Guards, only: [not_empty_string: 1]
+
+  alias Pleroma.Language.Translation.Provider
+
+  use Provider
+
+  @behaviour Provider
+
+  @name "DeepL"
+
+  @impl Provider
+  def configured?, do: not_empty_string(base_url()) and not_empty_string(api_key())
+
+  @impl Provider
+  def translate(content, source_language, target_language) do
+    endpoint =
+      base_url()
+      |> URI.merge("/v2/translate")
+      |> URI.to_string()
+
+    case Pleroma.HTTP.post(
+           endpoint <>
+             "?" <>
+             URI.encode_query(%{
+               text: content,
+               source_lang: source_language |> String.upcase(),
+               target_lang: target_language,
+               tag_handling: "html"
+             }),
+           "",
+           [
+             {"Content-Type", "application/x-www-form-urlencoded"},
+             {"Authorization", "DeepL-Auth-Key #{api_key()}"}
+           ]
+         ) do
+      {:ok, %{status: 429}} ->
+        {:error, :too_many_requests}
+
+      {:ok, %{status: 456}} ->
+        {:error, :quota_exceeded}
+
+      {:ok, %{status: 200} = res} ->
+        %{
+          "translations" => [
+            %{"text" => content, "detected_source_language" => detected_source_language}
+          ]
+        } = Jason.decode!(res.body)
+
+        {:ok,
+         %{
+           content: content,
+           detected_source_language: detected_source_language,
+           provider: @name
+         }}
+
+      _ ->
+        {:error, :internal_server_error}
+    end
+  end
+
+  @impl Provider
+  def supported_languages(type) when type in [:source, :target] do
+    endpoint =
+      base_url()
+      |> URI.merge("/v2/languages")
+      |> URI.to_string()
+
+    case Pleroma.HTTP.post(
+           endpoint <> "?" <> URI.encode_query(%{type: type}),
+           "",
+           [
+             {"Content-Type", "application/x-www-form-urlencoded"},
+             {"Authorization", "DeepL-Auth-Key #{api_key()}"}
+           ]
+         ) do
+      {:ok, %{status: 200} = res} ->
+        languages =
+          Jason.decode!(res.body)
+          |> Enum.map(fn %{"language" => language} -> language |> String.downcase() end)
+          |> Enum.map(fn language ->
+            if String.contains?(language, "-") do
+              [language, language |> String.split("-") |> Enum.at(0)]
+            else
+              language
+            end
+          end)
+          |> List.flatten()
+          |> Enum.uniq()
+
+        {:ok, languages}
+
+      _ ->
+        {:error, :internal_server_error}
+    end
+  end
+
+  @impl Provider
+  def languages_matrix do
+    with {:ok, source_languages} <- supported_languages(:source),
+         {:ok, target_languages} <- supported_languages(:target) do
+      {:ok,
+       Map.new(source_languages, fn language -> {language, target_languages -- [language]} end)}
+    else
+      {:error, error} -> {:error, error}
+    end
+  end
+
+  @impl Provider
+  def name, do: @name
+
+  defp base_url do
+    Pleroma.Config.get([__MODULE__, :base_url])
+  end
+
+  defp api_key do
+    Pleroma.Config.get([__MODULE__, :api_key])
+  end
+end

lib/pleroma/language/translation/libretranslate.ex

@@ -0,0 +1,93 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Language.Translation.Libretranslate do
+  import Pleroma.Web.Utils.Guards, only: [not_empty_string: 1]
+
+  alias Pleroma.Language.Translation.Provider
+
+  use Provider
+
+  @behaviour Provider
+
+  @name "LibreTranslate"
+
+  @impl Provider
+  def configured?, do: not_empty_string(base_url()) and not_empty_string(api_key())
+
+  @impl Provider
+  def translate(content, source_language, target_language) do
+    case Pleroma.HTTP.post(
+           base_url() <> "/translate",
+           Jason.encode!(%{
+             q: content,
+             source: source_language |> String.upcase(),
+             target: target_language,
+             format: "html",
+             api_key: api_key()
+           }),
+           [
+             {"Content-Type", "application/json"}
+           ]
+         ) do
+      {:ok, %{status: 429}} ->
+        {:error, :too_many_requests}
+
+      {:ok, %{status: 403}} ->
+        {:error, :quota_exceeded}
+
+      {:ok, %{status: 200} = res} ->
+        %{
+          "translatedText" => content
+        } = Jason.decode!(res.body)
+
+        {:ok,
+         %{
+           content: content,
+           detected_source_language: source_language,
+           provider: @name
+         }}
+
+      _ ->
+        {:error, :internal_server_error}
+    end
+  end
+
+  @impl Provider
+  def supported_languages(_) do
+    case Pleroma.HTTP.get(base_url() <> "/languages") do
+      {:ok, %{status: 200} = res} ->
+        languages =
+          Jason.decode!(res.body)
+          |> Enum.map(fn %{"code" => code} -> code end)
+
+        {:ok, languages}
+
+      _ ->
+        {:error, :internal_server_error}
+    end
+  end
+
+  @impl Provider
+  def languages_matrix do
+    with {:ok, source_languages} <- supported_languages(:source),
+         {:ok, target_languages} <- supported_languages(:target) do
+      {:ok,
+       Map.new(source_languages, fn language -> {language, target_languages -- [language]} end)}
+    else
+      {:error, error} -> {:error, error}
+    end
+  end
+
+  @impl Provider
+  def name, do: @name
+
+  defp base_url do
+    Pleroma.Config.get([__MODULE__, :base_url])
+  end
+
+  defp api_key do
+    Pleroma.Config.get([__MODULE__, :api_key], "")
+  end
+end

lib/pleroma/language/translation/provider.ex

@@ -0,0 +1,40 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Language.Translation.Provider do
+  alias Pleroma.Language.Translation.Provider
+
+  @callback missing_dependencies() :: [String.t()]
+
+  @callback configured?() :: boolean()
+
+  @callback translate(
+              content :: String.t(),
+              source_language :: String.t(),
+              target_language :: String.t()
+            ) ::
+              {:ok,
+               %{
+                 content: String.t(),
+                 detected_source_language: String.t(),
+                 provider: String.t()
+               }}
+              | {:error, atom()}
+
+  @callback supported_languages(type :: :string | :target) ::
+              {:ok, [String.t()]} | {:error, atom()}
+
+  @callback languages_matrix() :: {:ok, Map.t()} | {:error, atom()}
+
+  @callback name() :: String.t()
+
+  defmacro __using__(_opts) do
+    quote do
+      @impl Provider
+      def missing_dependencies, do: []
+
+      defoverridable missing_dependencies: 0
+    end
+  end
+end

lib/pleroma/web/api_spec/operations/instance_operation.ex

@@ -57,6 +57,29 @@ defmodule Pleroma.Web.ApiSpec.InstanceOperation do
     }
   end
 
+  def translation_languages_operation do
+    %Operation{
+      tags: ["Instance misc"],
+      summary: "Retrieve supported languages matrix",
+      operationId: "InstanceController.translation_languages",
+      responses: %{
+        200 =>
+          Operation.response(
+            "Translation languages matrix",
+            "application/json",
+            %Schema{
+              type: :object,
+              additionalProperties: %Schema{
+                type: :array,
+                items: %Schema{type: :string},
+                description: "Supported target languages for a source language"
+              }
+            }
+          )
+      }
+    }
+  end
+
   defp instance do
     %Schema{
       type: :object,

lib/pleroma/web/api_spec/operations/status_operation.ex

@@ -427,6 +427,38 @@ defmodule Pleroma.Web.ApiSpec.StatusOperation do
     }
   end
 
+  def translate_operation do
+    %Operation{
+      tags: ["Retrieve status information"],
+      summary: "Translate status",
+      description: "Translate status with an external API",
+      operationId: "StatusController.translate",
+      security: [%{"oAuth" => ["read:statuses"]}],
+      parameters: [id_param()],
+      requestBody:
+        request_body(
+          "Parameters",
+          %Schema{
+            type: :object,
+            properties: %{
+              lang: %Schema{
+                type: :string,
+                nullable: true,
+                description: "Translation target language."
+              }
+            }
+          },
+          required: false
+        ),
+      responses: %{
+        200 => Operation.response("Translation", "application/json", translation()),
+        400 => Operation.response("Error", "application/json", ApiError),
+        404 => Operation.response("Error", "application/json", ApiError),
+        503 => Operation.response("Error", "application/json", ApiError)
+      }
+    }
+  end
+
   def favourites_operation do
     %Operation{
       tags: ["Timelines"],
@@ -819,4 +851,32 @@ defmodule Pleroma.Web.ApiSpec.StatusOperation do
       }
     }
   end
+
+  defp translation do
+    %Schema{
+      title: "StatusTranslation",
+      description: "Represents status translation with related information.",
+      type: :object,
+      required: [:content, :detected_source_language, :provider],
+      properties: %{
+        content: %Schema{
+          type: :string,
+          description: "Translated status content"
+        },
+        detected_source_language: %Schema{
+          type: :string,
+          description: "Detected source language"
+        },
+        provider: %Schema{
+          type: :string,
+          description: "Translation provider service name"
+        }
+      },
+      example: %{
+        "content" => "Software für die nächste Generation der sozialen Medien.",
+        "detected_source_language" => "en",
+        "provider" => "Deepl"
+      }
+    }
+  end
 end

lib/pleroma/web/mastodon_api/controllers/instance_controller.ex

@@ -30,4 +30,9 @@ defmodule Pleroma.Web.MastodonAPI.InstanceController do
   def rules(conn, _params) do
     render(conn, "rules.json")
   end
+
+  @doc "GET /api/v1/instance/translation_languages"
+  def translation_languages(conn, _params) do
+    render(conn, "translation_languages.json")
+  end
 end

lib/pleroma/web/mastodon_api/controllers/status_controller.ex

@@ -13,6 +13,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do
   alias Pleroma.Activity
   alias Pleroma.Bookmark
   alias Pleroma.BookmarkFolder
+  alias Pleroma.Language.Translation
   alias Pleroma.Object
   alias Pleroma.Repo
   alias Pleroma.ScheduledActivity
@@ -44,6 +45,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do
          ]
   )
 
+  plug(OAuthScopesPlug, %{scopes: ["read:statuses"]} when action == :translate)
+
   plug(
     OAuthScopesPlug,
     %{scopes: ["write:statuses"]}
@@ -85,7 +88,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do
     %{scopes: ["write:bookmarks"]} when action in [:bookmark, :unbookmark]
   )
 
-  @rate_limited_status_actions ~w(reblog unreblog favourite unfavourite create delete)a
+  @rate_limited_status_actions ~w(reblog unreblog favourite unfavourite create delete translate)a
 
   plug(
     RateLimiter,
@@ -549,6 +552,41 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do
     end
   end
 
+  @doc "POST /api/v1/statuses/:id/translate"
+  def translate(
+        %{
+          assigns: %{user: user},
+          private: %{open_api_spex: %{body_params: params, params: %{id: status_id}}}
+        } = conn,
+        _
+      ) do
+    with %Activity{object: object} <- Activity.get_by_id_with_object(status_id),
+         {:visibility, visibility} when visibility in ["public", "unlisted"] <-
+           {:visibility, Visibility.get_visibility(object)},
+         {:language, language} when is_binary(language) <-
+           {:language, Map.get(params, :lang) || user.language},
+         {:ok, result} <-
+           Translation.translate(
+             object.data["content"],
+             object.data["language"],
+             language
+           ) do
+      render(conn, "translation.json", result)
+    else
+      {:language, nil} ->
+        render_error(conn, :bad_request, "Language not specified")
+
+      {:visibility, _} ->
+        render_error(conn, :not_found, "Record not found")
+
+      {:error, :not_found} ->
+        render_error(conn, :not_found, "Translation service not configured")
+
+      {:error, error} when error in [:unexpected_response, :quota_exceeded, :too_many_requests] ->
+        render_error(conn, :service_unavailable, "Translation service not available")
+    end
+  end
+
   @doc "GET /api/v1/favourites"
   def favourites(
         %{assigns: %{user: %User{} = user}, private: %{open_api_spex: %{params: params}}} = conn,

lib/pleroma/web/mastodon_api/views/instance_view.ex

@@ -90,6 +90,15 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do
     }
   end
 
+  def render("translation_languages.json", _) do
+    with true <- Pleroma.Language.Translation.configured?(),
+         {:ok, languages} <- Pleroma.Language.Translation.languages_matrix() do
+      languages
+    else
+      _ -> %{}
+    end
+  end
+
   defp common_information(instance) do
     %{
       languages: Keyword.get(instance, :languages, ["en"]),
@@ -246,7 +255,8 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do
       },
       vapid: %{
         public_key: Keyword.get(Pleroma.Web.Push.vapid_config(), :public_key)
-      }
+      },
+      translation: %{enabled: Pleroma.Language.Translation.configured?()}
     })
   end
 
@@ -259,7 +269,8 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do
         fields_limits: fields_limits(),
         post_formats: Config.get([:instance, :allowed_post_formats]),
         birthday_required: Config.get([:instance, :birthday_required]),
-        birthday_min_age: Config.get([:instance, :birthday_min_age])
+        birthday_min_age: Config.get([:instance, :birthday_min_age]),
+        translation: supported_languages()
       },
       stats: %{mau: Pleroma.User.active_user_count()},
       vapid_public_key: Keyword.get(Pleroma.Web.Push.vapid_config(), :public_key)
@@ -285,4 +296,29 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do
         })
     })
   end
+
+  defp supported_languages do
+    enabled = Pleroma.Language.Translation.configured?()
+
+    source_languages =
+      with true <- enabled,
+           {:ok, languages} <- Pleroma.Language.Translation.supported_languages(:source) do
+        languages
+      else
+        _ -> nil
+      end
+
+    target_languages =
+      with true <- enabled,
+           {:ok, languages} <- Pleroma.Language.Translation.supported_languages(:target) do
+        languages
+      else
+        _ -> nil
+      end
+
+    %{
+      source_languages: source_languages,
+      target_languages: target_languages
+    }
+  end
 end

lib/pleroma/web/mastodon_api/views/status_view.ex

@@ -681,6 +681,14 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
     }
   end
 
+  def render("translation.json", %{
+        content: content,
+        detected_source_language: detected_source_language,
+        provider: provider
+      }) do
+    %{content: content, detected_source_language: detected_source_language, provider: provider}
+  end
+
   def get_reply_to(activity, %{replied_to_activities: replied_to_activities}) do
     object = Object.normalize(activity, fetch: false)
 

lib/pleroma/web/router.ex

@@ -740,6 +740,7 @@ defmodule Pleroma.Web.Router do
     post("/statuses/:id/unbookmark", StatusController, :unbookmark)
     post("/statuses/:id/mute", StatusController, :mute_conversation)
     post("/statuses/:id/unmute", StatusController, :unmute_conversation)
+    post("/statuses/:id/translate", StatusController, :translate)
 
     post("/push/subscription", SubscriptionController, :create)
     get("/push/subscription", SubscriptionController, :show)
@@ -787,6 +788,7 @@ defmodule Pleroma.Web.Router do
     get("/instance", InstanceController, :show)
     get("/instance/peers", InstanceController, :peers)
     get("/instance/rules", InstanceController, :rules)
+    get("/instance/translation_languages", InstanceController, :translation_languages)
 
     get("/statuses", StatusController, :index)
     get("/statuses/:id", StatusController, :show)