Merge remote-tracking branch 'origin/develop' into translate-posts

Signed-off-by: mkljczk <git@mkljczk.pl>

lib/pleroma/language/language_detector.ex

@@ -0,0 +1,59 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Language.LanguageDetector do
+  import Pleroma.EctoType.ActivityPub.ObjectValidators.LanguageCode,
+    only: [good_locale_code?: 1]
+
+  @words_threshold 4
+  @config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config)
+
+  def configured? do
+    provider = get_provider()
+
+    !!provider and provider.configured?
+  end
+
+  def missing_dependencies do
+    provider = get_provider()
+
+    if provider do
+      provider.missing_dependencies()
+    else
+      []
+    end
+  end
+
+  # Strip tags from text, etc.
+  defp prepare_text(text) do
+    text
+    |> Floki.parse_fragment!()
+    |> Floki.filter_out(
+      ".h-card, .mention, .hashtag, .u-url, .quote-inline, .recipients-inline, code, pre"
+    )
+    |> Floki.text()
+  end
+
+  def detect(text) do
+    provider = get_provider()
+
+    text = prepare_text(text)
+    word_count = text |> String.split(~r/\s+/) |> Enum.count()
+
+    if word_count < @words_threshold or !provider or !provider.configured? do
+      nil
+    else
+      with language <- provider.detect(text),
+           true <- good_locale_code?(language) do
+        language
+      else
+        _ -> nil
+      end
+    end
+  end
+
+  defp get_provider do
+    @config_impl.get([__MODULE__, :provider])
+  end
+end

lib/pleroma/language/language_detector/fasttext.ex

@@ -0,0 +1,47 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Language.LanguageDetector.Fasttext do
+  import Pleroma.Web.Utils.Guards, only: [not_empty_string: 1]
+
+  alias Pleroma.Language.LanguageDetector.Provider
+
+  @behaviour Provider
+
+  @impl Provider
+  def missing_dependencies do
+    if Pleroma.Utils.command_available?("fasttext") do
+      []
+    else
+      ["fasttext"]
+    end
+  end
+
+  @impl Provider
+  def configured?, do: not_empty_string(get_model())
+
+  @impl Provider
+  def detect(text) do
+    text_path = Path.join(System.tmp_dir!(), "fasttext-#{Ecto.UUID.generate()}")
+
+    File.write(text_path, text |> String.replace(~r/\s+/, " "))
+
+    detected_language =
+      case System.cmd("fasttext", ["predict", get_model(), text_path]) do
+        {"__label__" <> language, _} ->
+          language |> String.trim()
+
+        _ ->
+          nil
+      end
+
+    File.rm(text_path)
+
+    detected_language
+  end
+
+  defp get_model do
+    Pleroma.Config.get([__MODULE__, :model])
+  end
+end

lib/pleroma/language/language_detector/provider.ex

@@ -0,0 +1,11 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Language.LanguageDetector.Provider do
+  @callback missing_dependencies() :: [String.t()]
+
+  @callback configured?() :: boolean()
+
+  @callback detect(text :: String.t()) :: String.t() | nil
+end