config editing through database

lib/pleroma/config/transfer_task.ex

@@ -4,56 +4,59 @@
 
 defmodule Pleroma.Config.TransferTask do
   use Task
+
+  require Logger
+
+  alias Pleroma.Repo
   alias Pleroma.Web.AdminAPI.Config
 
   def start_link(_) do
     load_and_update_env()
-    if Pleroma.Config.get(:env) == :test, do: Ecto.Adapters.SQL.Sandbox.checkin(Pleroma.Repo)
+    if Pleroma.Config.get(:env) == :test, do: Ecto.Adapters.SQL.Sandbox.checkin(Repo)
     :ignore
   end
 
   def load_and_update_env do
-    if Pleroma.Config.get([:instance, :dynamic_configuration]) and
-         Ecto.Adapters.SQL.table_exists?(Pleroma.Repo, "config") do
-      for_restart =
-        Pleroma.Repo.all(Config)
-        |> Enum.map(&update_env(&1))
-
+    with true <- Pleroma.Config.get([:instance, :dynamic_configuration]),
+         true <- Ecto.Adapters.SQL.table_exists?(Repo, "config"),
+         started_applications <- Application.started_applications() do
       # We need to restart applications for loaded settings take effect
-      for_restart
-      |> Enum.reject(&(&1 in [:pleroma, :ok]))
-      |> Enum.each(fn app ->
-        Application.stop(app)
-        :ok = Application.start(app)
-      end)
+      Config
+      |> Repo.all()
+      |> Enum.map(&update_env(&1))
+      |> Enum.uniq()
+      # TODO: some problem with prometheus after restart!
+      |> Enum.reject(&(&1 in [:pleroma, nil, :prometheus]))
+      |> Enum.each(&restart(started_applications, &1))
     end
   end
 
   defp update_env(setting) do
     try do
-      key =
-        if String.starts_with?(setting.key, "Pleroma.") do
-          "Elixir." <> setting.key
-        else
-          String.trim_leading(setting.key, ":")
-        end
+      key = Config.from_string(setting.key)
+      group = Config.from_string(setting.group)
+      value = Config.from_binary(setting.value)
 
-      group = String.to_existing_atom(setting.group)
-
-      Application.put_env(
-        group,
-        String.to_existing_atom(key),
-        Config.from_binary(setting.value)
-      )
+      :ok = Application.put_env(group, key, value)
 
       group
     rescue
       e ->
-        require Logger
-
         Logger.warn(
           "updating env causes error, key: #{inspect(setting.key)}, error: #{inspect(e)}"
         )
+
+        nil
+    end
+  end
+
+  defp restart(started_applications, app) do
+    with {^app, _, _} <- List.keyfind(started_applications, app, 0),
+         :ok <- Application.stop(app) do
+      :ok = Application.start(app)
+    else
+      nil -> Logger.warn("#{app} is not started.")
+      error -> Logger.warn(inspect(error))
     end
   end
 end

lib/pleroma/docs/generator.ex

@@ -6,68 +6,108 @@ defmodule Pleroma.Docs.Generator do
     implementation.process(descriptions)
   end
 
-  @spec uploaders_list() :: [module()]
-  def uploaders_list do
-    {:ok, modules} = :application.get_key(:pleroma, :modules)
-
-    Enum.filter(modules, fn module ->
-      name_as_list = Module.split(module)
-
-      List.starts_with?(name_as_list, ["Pleroma", "Uploaders"]) and
-        List.last(name_as_list) != "Uploader"
-    end)
+  @spec list_modules_in_dir(String.t(), String.t()) :: [module()]
+  def list_modules_in_dir(dir, start) do
+    with {:ok, files} <- File.ls(dir) do
+      files
+      |> Enum.filter(&String.ends_with?(&1, ".ex"))
+      |> Enum.map(fn filename ->
+        module = filename |> String.trim_trailing(".ex") |> Macro.camelize()
+        String.to_existing_atom(start <> module)
+      end)
+    end
   end
 
-  @spec filters_list() :: [module()]
-  def filters_list do
-    {:ok, modules} = :application.get_key(:pleroma, :modules)
-
-    Enum.filter(modules, fn module ->
-      name_as_list = Module.split(module)
-
-      List.starts_with?(name_as_list, ["Pleroma", "Upload", "Filter"])
-    end)
+  @doc """
+  Converts:
+  - atoms to strings with leading `:`
+  - module names to strings, without leading `Elixir.`
+  - add humanized labels to `keys` if label is not defined, e.g. `:instance` -> `Instance`
+  """
+  @spec convert_to_strings([map()]) :: [map()]
+  def convert_to_strings(descriptions) do
+    Enum.map(descriptions, &format_entity(&1))
   end
 
-  @spec mrf_list() :: [module()]
-  def mrf_list do
-    {:ok, modules} = :application.get_key(:pleroma, :modules)
-
-    Enum.filter(modules, fn module ->
-      name_as_list = Module.split(module)
-
-      List.starts_with?(name_as_list, ["Pleroma", "Web", "ActivityPub", "MRF"]) and
-        length(name_as_list) > 4
-    end)
+  defp format_entity(entity) do
+    entity
+    |> format_key()
+    |> Map.put(:group, atom_to_string(entity[:group]))
+    |> format_children()
   end
 
-  @spec richmedia_parsers() :: [module()]
-  def richmedia_parsers do
-    {:ok, modules} = :application.get_key(:pleroma, :modules)
-
-    Enum.filter(modules, fn module ->
-      name_as_list = Module.split(module)
-
-      List.starts_with?(name_as_list, ["Pleroma", "Web", "RichMedia", "Parsers"]) and
-        length(name_as_list) == 5
-    end)
+  defp format_key(%{key: key} = entity) do
+    entity
+    |> Map.put(:key, atom_to_string(key))
+    |> Map.put(:label, entity[:label] || humanize(key))
   end
+
+  defp format_key(%{group: group} = entity) do
+    Map.put(entity, :label, entity[:label] || humanize(group))
+  end
+
+  defp format_key(entity), do: entity
+
+  defp format_children(%{children: children} = entity) do
+    Map.put(entity, :children, Enum.map(children, &format_child(&1)))
+  end
+
+  defp format_children(entity), do: entity
+
+  defp format_child(%{suggestions: suggestions} = entity) do
+    entity
+    |> Map.put(:suggestions, format_suggestions(suggestions))
+    |> format_key()
+    |> format_children()
+  end
+
+  defp format_child(entity) do
+    entity
+    |> format_key()
+    |> format_children()
+  end
+
+  defp atom_to_string(entity) when is_binary(entity), do: entity
+
+  defp atom_to_string(entity) when is_atom(entity), do: inspect(entity)
+
+  defp humanize(entity) do
+    string = inspect(entity)
+
+    if String.starts_with?(string, ":"),
+      do: Phoenix.Naming.humanize(entity),
+      else: string
+  end
+
+  defp format_suggestions([]), do: []
+
+  defp format_suggestions([suggestion | tail]) do
+    [format_suggestion(suggestion) | format_suggestions(tail)]
+  end
+
+  defp format_suggestion(entity) when is_atom(entity) do
+    atom_to_string(entity)
+  end
+
+  defp format_suggestion([head | tail] = entity) when is_list(entity) do
+    [format_suggestion(head) | format_suggestions(tail)]
+  end
+
+  defp format_suggestion(entity) when is_tuple(entity) do
+    format_suggestions(Tuple.to_list(entity)) |> List.to_tuple()
+  end
+
+  defp format_suggestion(entity), do: entity
 end
 
 defimpl Jason.Encoder, for: Tuple do
-  def encode(tuple, opts) do
-    Jason.Encode.list(Tuple.to_list(tuple), opts)
-  end
+  def encode(tuple, opts), do: Jason.Encode.list(Tuple.to_list(tuple), opts)
 end
 
 defimpl Jason.Encoder, for: [Regex, Function] do
-  def encode(term, opts) do
-    Jason.Encode.string(inspect(term), opts)
-  end
+  def encode(term, opts), do: Jason.Encode.string(inspect(term), opts)
 end
 
 defimpl String.Chars, for: Regex do
-  def to_string(term) do
-    inspect(term)
-  end
+  def to_string(term), do: inspect(term)
 end

lib/pleroma/docs/json.ex

@@ -3,18 +3,22 @@ defmodule Pleroma.Docs.JSON do
 
   @spec process(keyword()) :: {:ok, String.t()}
   def process(descriptions) do
-    config_path = "docs/generate_config.json"
-
-    with {:ok, file} <- File.open(config_path, [:write, :utf8]),
-         json <- generate_json(descriptions),
+    with path <- "docs/generated_config.json",
+         {:ok, file} <- File.open(path, [:write, :utf8]),
+         formatted_descriptions <-
+           Pleroma.Docs.Generator.convert_to_strings(descriptions),
+         json <- Jason.encode!(formatted_descriptions),
          :ok <- IO.write(file, json),
          :ok <- File.close(file) do
-      {:ok, config_path}
+      {:ok, path}
     end
   end
 
-  @spec generate_json([keyword()]) :: String.t()
-  def generate_json(descriptions) do
-    Jason.encode!(descriptions)
+  def compile do
+    with {config, _paths} <- Mix.Config.eval!("config/description.exs") do
+      config[:pleroma][:config_description]
+      |> Pleroma.Docs.Generator.convert_to_strings()
+      |> Jason.encode!()
+    end
   end
 end

lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex

@@ -19,8 +19,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.VocabularyPolicy do
   def filter(%{"type" => message_type} = message) do
     with accepted_vocabulary <- Pleroma.Config.get([:mrf_vocabulary, :accept]),
          rejected_vocabulary <- Pleroma.Config.get([:mrf_vocabulary, :reject]),
-         true <-
-           length(accepted_vocabulary) == 0 || Enum.member?(accepted_vocabulary, message_type),
+         true <- accepted_vocabulary == [] || Enum.member?(accepted_vocabulary, message_type),
          false <-
            length(rejected_vocabulary) > 0 && Enum.member?(rejected_vocabulary, message_type),
          {:ok, _} <- filter(message["object"]) do

lib/pleroma/web/admin_api/admin_api_controller.ex

@@ -4,6 +4,9 @@
 
 defmodule Pleroma.Web.AdminAPI.AdminAPIController do
   use Pleroma.Web, :controller
+
+  import Pleroma.Web.ControllerHelper, only: [json_response: 3]
+
   alias Pleroma.Activity
   alias Pleroma.ModerationLog
   alias Pleroma.Plugs.OAuthScopesPlug
@@ -25,10 +28,11 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
   alias Pleroma.Web.MastodonAPI.StatusView
   alias Pleroma.Web.Router
 
-  import Pleroma.Web.ControllerHelper, only: [json_response: 3]
-
   require Logger
 
+  @descriptions_json Pleroma.Docs.JSON.compile()
+  @users_page_size 50
+
   plug(
     OAuthScopesPlug,
     %{scopes: ["read:accounts"], admin: true}
@@ -93,8 +97,6 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
     when action in [:relay_follow, :relay_unfollow, :config_update]
   )
 
-  @users_page_size 50
-
   action_fallback(:errors)
 
   def user_delete(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname}) do
@@ -782,10 +784,22 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
   end
 
   def migrate_from_db(conn, _params) do
-    Mix.Tasks.Pleroma.Config.run(["migrate_from_db", Pleroma.Config.get(:env), "true"])
+    Mix.Tasks.Pleroma.Config.run([
+      "migrate_from_db",
+      "--env",
+      to_string(Pleroma.Config.get(:env)),
+      "-d"
+    ])
+
     json(conn, %{})
   end
 
+  def config_descriptions(conn, _params) do
+    conn
+    |> Plug.Conn.put_resp_content_type("application/json")
+    |> Plug.Conn.send_resp(200, @descriptions_json)
+  end
+
   def config_show(conn, _params) do
     configs = Pleroma.Repo.all(Config)
 
@@ -800,17 +814,27 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
         updated =
           Enum.map(configs, fn
             %{"group" => group, "key" => key, "delete" => "true"} = params ->
-              {:ok, config} = Config.delete(%{group: group, key: key, subkeys: params["subkeys"]})
-              config
+              with {:ok, config} <-
+                     Config.delete(%{group: group, key: key, subkeys: params["subkeys"]}) do
+                config
+              end
 
             %{"group" => group, "key" => key, "value" => value} ->
-              {:ok, config} = Config.update_or_create(%{group: group, key: key, value: value})
-              config
+              with {:ok, config} <-
+                     Config.update_or_create(%{group: group, key: key, value: value}) do
+                config
+              end
           end)
           |> Enum.reject(&is_nil(&1))
 
         Pleroma.Config.TransferTask.load_and_update_env()
-        Mix.Tasks.Pleroma.Config.run(["migrate_from_db", Pleroma.Config.get(:env), "false"])
+
+        Mix.Tasks.Pleroma.Config.run([
+          "migrate_from_db",
+          "--env",
+          to_string(Pleroma.Config.get(:env))
+        ])
+
         updated
       else
         []

lib/pleroma/web/admin_api/config.ex

@@ -24,6 +24,8 @@ defmodule Pleroma.Web.AdminAPI.Config do
 
   @spec changeset(Config.t(), map()) :: Changeset.t()
   def changeset(config, params \\ %{}) do
+    params = Map.put(params, :value, transform(params[:value]))
+
     config
     |> cast(params, [:key, :group, :value])
     |> validate_required([:key, :group, :value])
@@ -33,42 +35,43 @@ defmodule Pleroma.Web.AdminAPI.Config do
   @spec create(map()) :: {:ok, Config.t()} | {:error, Changeset.t()}
   def create(params) do
     %Config{}
-    |> changeset(Map.put(params, :value, transform(params[:value])))
+    |> changeset(params)
     |> Repo.insert()
   end
 
   @spec update(Config.t(), map()) :: {:ok, Config} | {:error, Changeset.t()}
   def update(%Config{} = config, %{value: value}) do
     config
-    |> change(value: transform(value))
+    |> changeset(%{value: value})
     |> Repo.update()
   end
 
   @spec update_or_create(map()) :: {:ok, Config.t()} | {:error, Changeset.t()}
   def update_or_create(params) do
-    with %Config{} = config <- Config.get_by_params(Map.take(params, [:group, :key])) do
+    search_opts = Map.take(params, [:group, :key])
+
+    with %Config{} = config <- Config.get_by_params(search_opts) do
       Config.update(config, params)
     else
       nil -> Config.create(params)
     end
   end
 
-  @spec delete(map()) :: {:ok, Config.t()} | {:error, Changeset.t()}
+  @spec delete(map()) :: {:ok, Config.t()} | {:error, Changeset.t()} | {:ok, nil}
   def delete(params) do
-    with %Config{} = config <- Config.get_by_params(Map.delete(params, :subkeys)) do
-      if params[:subkeys] do
-        updated_value =
-          Keyword.drop(
-            :erlang.binary_to_term(config.value),
-            Enum.map(params[:subkeys], &do_transform_string(&1))
-          )
+    search_opts = Map.delete(params, :subkeys)
 
-        Config.update(config, %{value: updated_value})
-      else
+    with %Config{} = config <- Config.get_by_params(search_opts),
+         {config, sub_keys} when is_list(sub_keys) <- {config, params[:subkeys]},
+         old_value <- :erlang.binary_to_term(config.value),
+         keys <- Enum.map(sub_keys, &do_transform_string(&1)),
+         new_value <- Keyword.drop(old_value, keys) do
+      Config.update(config, %{value: new_value})
+    else
+      {config, nil} ->
         Repo.delete(config)
         {:ok, nil}
-      end
-    else
+
       nil ->
         err =
           dgettext("errors", "Config with params %{params} not found", params: inspect(params))
@@ -82,10 +85,22 @@ defmodule Pleroma.Web.AdminAPI.Config do
 
   @spec from_binary_with_convert(binary()) :: any()
   def from_binary_with_convert(binary) do
-    from_binary(binary)
+    binary
+    |> from_binary()
     |> do_convert()
   end
 
+  @spec from_string(String.t()) :: atom() | no_return()
+  def from_string(":" <> entity), do: String.to_existing_atom(entity)
+
+  def from_string(entity) when is_binary(entity) do
+    if is_module_name?(entity) do
+      String.to_existing_atom("Elixir.#{entity}")
+    else
+      entity
+    end
+  end
+
   defp do_convert(entity) when is_list(entity) do
     for v <- entity, into: [], do: do_convert(v)
   end
@@ -97,6 +112,7 @@ defmodule Pleroma.Web.AdminAPI.Config do
   end
 
   defp do_convert({:dispatch, [entity]}), do: %{"tuple" => [":dispatch", [inspect(entity)]]}
+  # TODO: will become useless after removing hackney
   defp do_convert({:partial_chain, entity}), do: %{"tuple" => [":partial_chain", inspect(entity)]}
 
   defp do_convert(entity) when is_tuple(entity),
@@ -105,21 +121,15 @@ defmodule Pleroma.Web.AdminAPI.Config do
   defp do_convert(entity) when is_boolean(entity) or is_number(entity) or is_nil(entity),
     do: entity
 
-  defp do_convert(entity) when is_atom(entity) do
-    string = to_string(entity)
-
-    if String.starts_with?(string, "Elixir."),
-      do: do_convert(string),
-      else: ":" <> string
-  end
-
-  defp do_convert("Elixir." <> module_name), do: module_name
+  defp do_convert(entity) when is_atom(entity), do: inspect(entity)
 
   defp do_convert(entity) when is_binary(entity), do: entity
 
-  @spec transform(any()) :: binary()
+  @spec transform(any()) :: binary() | no_return()
   def transform(entity) when is_binary(entity) or is_map(entity) or is_list(entity) do
-    :erlang.term_to_binary(do_transform(entity))
+    entity
+    |> do_transform()
+    |> :erlang.term_to_binary()
   end
 
   def transform(entity), do: :erlang.term_to_binary(entity)
@@ -131,6 +141,7 @@ defmodule Pleroma.Web.AdminAPI.Config do
     {:dispatch, [dispatch_settings]}
   end
 
+  # TODO: will become useless after removing hackney
   defp do_transform(%{"tuple" => [":partial_chain", entity]}) do
     {partial_chain, []} = do_eval(entity)
     {:partial_chain, partial_chain}
@@ -149,34 +160,63 @@ defmodule Pleroma.Web.AdminAPI.Config do
   end
 
   defp do_transform(entity) when is_binary(entity) do
-    String.trim(entity)
+    entity
+    |> String.trim()
     |> do_transform_string()
   end
 
   defp do_transform(entity), do: entity
 
-  defp do_transform_string("~r/" <> pattern) do
-    modificator = String.split(pattern, "/") |> List.last()
-    pattern = String.trim_trailing(pattern, "/" <> modificator)
+  @delimiters ["/", "|", "\"", "'", {"(", ")"}, {"[", "]"}, {"{", "}"}, {"<", ">"}]
 
-    case modificator do
-      "" -> ~r/#{pattern}/
-      "i" -> ~r/#{pattern}/i
-      "u" -> ~r/#{pattern}/u
-      "s" -> ~r/#{pattern}/s
+  defp find_valid_delimiter([], _string, _),
+    do: raise(ArgumentError, message: "valid delimiter for Regex expression not found")
+
+  defp find_valid_delimiter([{leading, closing} = delimiter | others], pattern, regex_delimiter)
+       when is_tuple(delimiter) do
+    if String.contains?(pattern, closing) do
+      find_valid_delimiter(others, pattern, regex_delimiter)
+    else
+      {:ok, {leading, closing}}
+    end
+  end
+
+  defp find_valid_delimiter([delimiter | others], pattern, regex_delimiter) do
+    if String.contains?(pattern, delimiter) do
+      find_valid_delimiter(others, pattern, regex_delimiter)
+    else
+      {:ok, {delimiter, delimiter}}
+    end
+  end
+
+  @regex_parts ~r/^~r(?'delimiter'[\/|"'([{<]{1})(?'pattern'.+)[\/|"')\]}>]{1}(?'modifier'[uismxfU]*)/u
+
+  defp do_transform_string("~r" <> _pattern = regex) do
+    with %{"modifier" => modifier, "pattern" => pattern, "delimiter" => regex_delimiter} <-
+           Regex.named_captures(@regex_parts, regex),
+         {:ok, {leading, closing}} <- find_valid_delimiter(@delimiters, pattern, regex_delimiter),
+         {result, _} <- Code.eval_string("~r#{leading}#{pattern}#{closing}#{modifier}") do
+      result
     end
   end
 
   defp do_transform_string(":" <> atom), do: String.to_atom(atom)
 
   defp do_transform_string(value) do
-    if String.starts_with?(value, "Pleroma") or String.starts_with?(value, "Phoenix"),
-      do: String.to_existing_atom("Elixir." <> value),
-      else: value
+    if is_module_name?(value) do
+      String.to_existing_atom("Elixir." <> value)
+    else
+      value
+    end
+  end
+
+  @spec is_module_name?(String.t()) :: boolean()
+  def is_module_name?(string) do
+    Regex.match?(~r/^(Pleroma|Phoenix|Tesla)\./, string) or string in ["Oban", "Ueberauth"]
   end
 
   defp do_eval(entity) do
     cleaned_string = String.replace(entity, ~r/[^\w|^{:,[|^,|^[|^\]^}|^\/|^\.|^"]^\s/, "")
-    Code.eval_string(cleaned_string, [], requires: [], macros: [])
+    Code.eval_string(cleaned_string)
   end
 end

lib/pleroma/web/router.ex

@@ -195,6 +195,7 @@ defmodule Pleroma.Web.Router do
 
     get("/config", AdminAPIController, :config_show)
     post("/config", AdminAPIController, :config_update)
+    get("/config/descriptions", AdminAPIController, :config_descriptions)
     get("/config/migrate_to_db", AdminAPIController, :migrate_to_db)
     get("/config/migrate_from_db", AdminAPIController, :migrate_from_db)