saving to DB only added by user settings

lib/mix/tasks/pleroma/config.ex

@@ -4,42 +4,25 @@
 
 defmodule Mix.Tasks.Pleroma.Config do
   use Mix.Task
+
   import Mix.Pleroma
+
   alias Pleroma.ConfigDB
   alias Pleroma.Repo
+
+  require Logger
+
   @shortdoc "Manages the location of the config"
   @moduledoc File.read!("docs/administration/CLI_tasks/config.md")
 
-  @groups [
-    :pleroma,
-    :logger,
-    :quack,
-    :mime,
-    :tesla,
-    :phoenix,
-    :cors_plug,
-    :auto_linker,
-    :esshd,
-    :ueberauth,
-    :http_signatures,
-    :web_push_encryption,
-    :joken
-  ]
-
   def run(["migrate_to_db"]) do
     # we want to save original logger level
     start_pleroma(false)
-
-    if Pleroma.Config.get([:configurable_from_database]) do
-      Enum.each(@groups, &load_and_create(&1))
-    else
-      Mix.shell().info(
-        "Migration is not allowed by config. You can change this behavior in instance settings."
-      )
-    end
+    migrate_to_db()
   end
 
   def run(["migrate_from_db" | options]) do
+    # TODO: add support for releases
     start_pleroma()
 
     {opts, _} =
@@ -72,10 +55,36 @@ defmodule Mix.Tasks.Pleroma.Config do
     end
   end
 
-  defp load_and_create(group) do
-    group
-    |> Application.get_all_env()
-    |> Enum.reject(fn {k, _v} ->
+  @spec migrate_to_db(Path.t() | nil) :: any()
+  def migrate_to_db(file_path \\ nil) do
+    if Pleroma.Config.get([:configurable_from_database]) do
+      # TODO: add support for releases
+      config_file = file_path || "config/#{Pleroma.Config.get(:env)}.secret.exs"
+      do_migrate_to_db(config_file)
+    else
+      Mix.shell().info(
+        "migration is not allowed by config. You can change this behavior in instance settings."
+      )
+    end
+  end
+
+  defp do_migrate_to_db(config_file) do
+    if File.exists?(config_file) do
+      {custom_config, _paths} =
+        if Code.ensure_loaded?(Config.Reader),
+          do: Config.Reader.read_imports!(config_file),
+          else: Mix.Config.eval!(config_file)
+
+      custom_config
+      |> Keyword.keys()
+      |> Enum.each(&create(&1, custom_config[&1]))
+    else
+      Logger.warn("to migrate settings, you must define custom settings in #{config_file}")
+    end
+  end
+
+  defp create(group, settings) do
+    Enum.reject(settings, fn {k, _v} ->
       k in [Pleroma.Repo, Pleroma.Web.Endpoint, :env, :configurable_from_database] or
         (group == :phoenix and k == :serve_endpoints)
     end)

lib/pleroma/config/config_db.ex

@@ -5,6 +5,7 @@
 defmodule Pleroma.ConfigDB do
   use Ecto.Schema
   import Ecto.Changeset
+  import Ecto.Query
   import Pleroma.Web.Gettext
   alias __MODULE__
   alias Pleroma.Repo
@@ -19,6 +20,20 @@ defmodule Pleroma.ConfigDB do
     timestamps()
   end
 
+  @spec get_all_as_keyword() :: keyword()
+  def get_all_as_keyword do
+    ConfigDB
+    |> select([c], {c.group, c.key, c.value})
+    |> Repo.all()
+    |> Enum.reduce([], fn {group, key, value}, acc ->
+      group = ConfigDB.from_string(group)
+      key = ConfigDB.from_string(key)
+      value = from_binary(value)
+
+      Keyword.update(acc, group, [{key, value}], &Keyword.merge(&1, [{key, value}]))
+    end)
+  end
+
   @spec get_by_params(map()) :: ConfigDB.t() | nil
   def get_by_params(params), do: Repo.get_by(ConfigDB, params)
 
@@ -137,6 +152,9 @@ defmodule Pleroma.ConfigDB do
     end
   end
 
+  @spec convert(any()) :: any()
+  def convert(entity), do: do_convert(entity)
+
   defp do_convert(entity) when is_list(entity) do
     for v <- entity, into: [], do: do_convert(v)
   end

lib/pleroma/config/holder.ex

@@ -0,0 +1,51 @@
+defmodule Pleroma.Config.Loader do
+  # TODO: add support for releases
+  if Code.ensure_loaded?(Config.Reader) do
+    @spec load() :: map()
+    def load do
+      config = Config.Reader.read!("config/config.exs")
+      env_config = Config.Reader.read!("config/#{Mix.env()}.exs")
+
+      Config.Reader.merge(config, env_config)
+    end
+  else
+    # support for Elixir less than 1.9
+    @spec load() :: map()
+    def load do
+      config = Mix.Config.eval!("config/config.exs")
+      env_config = Mix.Config.eval!("config/#{Mix.env()}.exs")
+
+      Mix.Config.merge(config, env_config)
+    end
+  end
+end
+
+defmodule Pleroma.Config.Holder do
+  @config Pleroma.Config.Loader.load()
+
+  @spec config() :: keyword()
+  def config do
+    @config
+    |> Keyword.keys()
+    |> Enum.map(&filter(&1, config(&1)))
+    |> List.flatten()
+  end
+
+  @spec config(atom()) :: any()
+  def config(group), do: @config[group]
+
+  @spec config(atom(), atom()) :: any()
+  def config(group, key), do: @config[group][key]
+
+  defp filter(group, settings) when group not in [:swarm] do
+    filtered =
+      Enum.reject(settings, fn {k, _v} ->
+        k in [Pleroma.Repo, Pleroma.Web.Endpoint, :env, :configurable_from_database] or
+          (group == :phoenix and k == :serve_endpoints)
+      end)
+
+    {group, filtered}
+  end
+
+  defp filter(_, _), do: []
+end

lib/pleroma/config/transfer_task.ex

@@ -38,10 +38,18 @@ defmodule Pleroma.Config.TransferTask do
       value = ConfigDB.from_binary(setting.value)
 
       if group != :phoenix and key != :serve_endpoints do
-        :ok = Application.put_env(group, key, value)
-      end
+        default = Pleroma.Config.Holder.config(group, key)
 
-      group
+        merged_value =
+          if can_be_merged?(default, value) do
+            DeepMerge.deep_merge(default, value)
+          else
+            value
+          end
+
+        :ok = Application.put_env(group, key, merged_value)
+        group
+      end
     rescue
       e ->
         Logger.warn(
@@ -61,4 +69,12 @@ defmodule Pleroma.Config.TransferTask do
       error -> Logger.warn(inspect(error))
     end
   end
+
+  defp can_be_merged?(val1, val2) when is_map(val1) and is_map(val2), do: true
+
+  defp can_be_merged?(val1, val2) when is_list(val1) and is_list(val2) do
+    Keyword.keyword?(val1) and Keyword.keyword?(val2)
+  end
+
+  defp can_be_merged?(_val1, _val2), do: false
 end

lib/pleroma/web/admin_api/admin_api_controller.ex

@@ -797,7 +797,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
     end
   end
 
-  def config_show(conn, _params) do
+  def config_show(conn, %{"only_db" => true}) do
     with :ok <- configurable_from_database(conn) do
       configs = Pleroma.Repo.all(ConfigDB)
 
@@ -814,6 +814,46 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
     end
   end
 
+  def config_show(conn, _params) do
+    with :ok <- configurable_from_database(conn) do
+      configs = ConfigDB.get_all_as_keyword()
+
+      if configs == [] do
+        errors(
+          conn,
+          {:error, "To use configuration from database migrate your settings to database."}
+        )
+      else
+        merged =
+          Pleroma.Config.Holder.config()
+          |> DeepMerge.deep_merge(configs)
+          |> Enum.map(fn {group, value} ->
+            Enum.map(value, fn {key, value} ->
+              db =
+                if configs[group][key] do
+                  if Keyword.keyword?(value) do
+                    Keyword.keys(value) |> Enum.map(fn key -> ConfigDB.convert(key) end)
+                  else
+                    ConfigDB.convert(key)
+                  end
+                end
+
+              setting = %{
+                group: ConfigDB.convert(group),
+                key: ConfigDB.convert(key),
+                value: ConfigDB.convert(value)
+              }
+
+              if db, do: Map.put(setting, :db, db), else: setting
+            end)
+          end)
+          |> List.flatten()
+
+        json(conn, %{configs: merged})
+      end
+    end
+  end
+
   def config_update(conn, %{"configs" => configs}) do
     with :ok <- configurable_from_database(conn) do
       updated =

lib/pleroma/web/router.ex

@@ -196,7 +196,6 @@ defmodule Pleroma.Web.Router do
     get("/config", AdminAPIController, :config_show)
     post("/config", AdminAPIController, :config_update)
     get("/config/descriptions", AdminAPIController, :config_descriptions)
-    get("/config/migrate_to_db", AdminAPIController, :migrate_to_db)
     get("/config/migrate_from_db", AdminAPIController, :migrate_from_db)
 
     get("/moderation_log", AdminAPIController, :list_log)