Merge branch 'develop' into feature/addressable-lists

lib/healthcheck.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Healthcheck do
   @moduledoc """
   Module collects metrics about app and assign healthy status.

lib/jason_types.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 Postgrex.Types.define(
   Pleroma.PostgresTypes,
   [] ++ Ecto.Adapters.Postgres.extensions(),

lib/mix/pleroma.ex

@@ -0,0 +1,67 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Mix.Pleroma do
+  @doc "Common functions to be reused in mix tasks"
+  def start_pleroma do
+    Application.put_env(:phoenix, :serve_endpoints, false, persistent: true)
+    {:ok, _} = Application.ensure_all_started(:pleroma)
+  end
+
+  def load_pleroma do
+    Application.load(:pleroma)
+  end
+
+  def get_option(options, opt, prompt, defval \\ nil, defname \\ nil) do
+    Keyword.get(options, opt) || shell_prompt(prompt, defval, defname)
+  end
+
+  def shell_prompt(prompt, defval \\ nil, defname \\ nil) do
+    prompt_message = "#{prompt} [#{defname || defval}] "
+
+    input =
+      if mix_shell?(),
+        do: Mix.shell().prompt(prompt_message),
+        else: :io.get_line(prompt_message)
+
+    case input do
+      "\n" ->
+        case defval do
+          nil ->
+            shell_prompt(prompt, defval, defname)
+
+          defval ->
+            defval
+        end
+
+      input ->
+        String.trim(input)
+    end
+  end
+
+  def shell_yes?(message) do
+    if mix_shell?(),
+      do: Mix.shell().yes?("Continue?"),
+      else: shell_prompt(message, "Continue?") in ~w(Yn Y y)
+  end
+
+  def shell_info(message) do
+    if mix_shell?(),
+      do: Mix.shell().info(message),
+      else: IO.puts(message)
+  end
+
+  def shell_error(message) do
+    if mix_shell?(),
+      do: Mix.shell().error(message),
+      else: IO.puts(:stderr, message)
+  end
+
+  @doc "Performs a safe check whether `Mix.shell/0` is available (does not raise if Mix is not loaded)"
+  def mix_shell?, do: :erlang.function_exported(Mix, :shell, 0)
+
+  def escape_sh_path(path) do
+    ~S(') <> String.replace(path, ~S('), ~S(\')) <> ~S(')
+  end
+end

lib/mix/tasks/pleroma/benchmark.ex

@@ -1,19 +1,23 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Mix.Tasks.Pleroma.Benchmark do
+  import Mix.Pleroma
   use Mix.Task
-  alias Mix.Tasks.Pleroma.Common
 
   def run(["search"]) do
-    Common.start_pleroma()
+    start_pleroma()
 
     Benchee.run(%{
       "search" => fn ->
-        Pleroma.Web.MastodonAPI.MastodonAPIController.status_search(nil, "cofe")
+        Pleroma.Activity.search(nil, "cofe")
       end
     })
   end
 
   def run(["tag"]) do
-    Common.start_pleroma()
+    start_pleroma()
 
     Benchee.run(%{
       "tag" => fn ->

lib/mix/tasks/pleroma/common.ex

@@ -1,28 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Mix.Tasks.Pleroma.Common do
-  @doc "Common functions to be reused in mix tasks"
-  def start_pleroma do
-    Mix.Task.run("app.start")
-  end
-
-  def get_option(options, opt, prompt, defval \\ nil, defname \\ nil) do
-    Keyword.get(options, opt) ||
-      case Mix.shell().prompt("#{prompt} [#{defname || defval}]") do
-        "\n" ->
-          case defval do
-            nil -> get_option(options, opt, prompt, defval)
-            defval -> defval
-          end
-
-        opt ->
-          opt |> String.trim()
-      end
-  end
-
-  def escape_sh_path(path) do
-    ~S(') <> String.replace(path, ~S('), ~S(\')) <> ~S(')
-  end
-end

lib/mix/tasks/pleroma/config.ex

@@ -0,0 +1,83 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Mix.Tasks.Pleroma.Config do
+  use Mix.Task
+  import Mix.Pleroma
+  alias Pleroma.Repo
+  alias Pleroma.Web.AdminAPI.Config
+  @shortdoc "Manages the location of the config"
+  @moduledoc """
+  Manages the location of the config.
+
+  ## Transfers config from file to DB.
+
+      mix pleroma.config migrate_to_db
+
+  ## Transfers config from DB to file.
+
+      mix pleroma.config migrate_from_db ENV
+  """
+
+  def run(["migrate_to_db"]) do
+    start_pleroma()
+
+    if Pleroma.Config.get([:instance, :dynamic_configuration]) do
+      Application.get_all_env(:pleroma)
+      |> Enum.reject(fn {k, _v} -> k in [Pleroma.Repo, :env] end)
+      |> Enum.each(fn {k, v} ->
+        key = to_string(k) |> String.replace("Elixir.", "")
+        {:ok, _} = Config.update_or_create(%{group: "pleroma", key: key, value: v})
+        Mix.shell().info("#{key} is migrated.")
+      end)
+
+      Mix.shell().info("Settings migrated.")
+    else
+      Mix.shell().info(
+        "Migration is not allowed by config. You can change this behavior in instance settings."
+      )
+    end
+  end
+
+  def run(["migrate_from_db", env, delete?]) do
+    start_pleroma()
+
+    delete? = if delete? == "true", do: true, else: false
+
+    if Pleroma.Config.get([:instance, :dynamic_configuration]) do
+      config_path = "config/#{env}.exported_from_db.secret.exs"
+
+      {:ok, file} = File.open(config_path, [:write])
+      IO.write(file, "use Mix.Config\r\n")
+
+      Repo.all(Config)
+      |> Enum.each(fn config ->
+        mark =
+          if String.starts_with?(config.key, "Pleroma.") or
+               String.starts_with?(config.key, "Ueberauth"),
+             do: ",",
+             else: ":"
+
+        IO.write(
+          file,
+          "config :#{config.group}, #{config.key}#{mark} #{
+            inspect(Config.from_binary(config.value))
+          }\r\n"
+        )
+
+        if delete? do
+          {:ok, _} = Repo.delete(config)
+          Mix.shell().info("#{config.key} deleted from DB.")
+        end
+      end)
+
+      File.close(file)
+      System.cmd("mix", ["format", config_path])
+    else
+      Mix.shell().info(
+        "Migration is not allowed by config. You can change this behavior in instance settings."
+      )
+    end
+  end
+end

lib/mix/tasks/pleroma/database.ex

@@ -3,12 +3,12 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Mix.Tasks.Pleroma.Database do
-  alias Mix.Tasks.Pleroma.Common
   alias Pleroma.Conversation
   alias Pleroma.Object
   alias Pleroma.Repo
   alias Pleroma.User
   require Logger
+  import Mix.Pleroma
   use Mix.Task
 
   @shortdoc "A collection of database related tasks"
@@ -45,7 +45,7 @@ defmodule Mix.Tasks.Pleroma.Database do
         ]
       )
 
-    Common.start_pleroma()
+    start_pleroma()
     Logger.info("Removing embedded objects")
 
     Repo.query!(
@@ -66,12 +66,12 @@ defmodule Mix.Tasks.Pleroma.Database do
   end
 
   def run(["bump_all_conversations"]) do
-    Common.start_pleroma()
+    start_pleroma()
     Conversation.bump_for_all_activities()
   end
 
   def run(["update_users_following_followers_counts"]) do
-    Common.start_pleroma()
+    start_pleroma()
 
     users = Repo.all(User)
     Enum.each(users, &User.remove_duplicated_following/1)
@@ -89,7 +89,7 @@ defmodule Mix.Tasks.Pleroma.Database do
         ]
       )
 
-    Common.start_pleroma()
+    start_pleroma()
 
     deadline = Pleroma.Config.get([:instance, :remote_post_retention_days])
 

lib/mix/tasks/pleroma/ecto/ecto.ex

@@ -0,0 +1,50 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-onl
+
+defmodule Mix.Tasks.Pleroma.Ecto do
+  @doc """
+  Ensures the given repository's migrations path exists on the file system.
+  """
+  @spec ensure_migrations_path(Ecto.Repo.t(), Keyword.t()) :: String.t()
+  def ensure_migrations_path(repo, opts) do
+    path = opts[:migrations_path] || Path.join(source_repo_priv(repo), "migrations")
+
+    path =
+      case Path.type(path) do
+        :relative ->
+          Path.join(Application.app_dir(:pleroma), path)
+
+        :absolute ->
+          path
+      end
+
+    if not File.dir?(path) do
+      raise_missing_migrations(Path.relative_to_cwd(path), repo)
+    end
+
+    path
+  end
+
+  @doc """
+  Returns the private repository path relative to the source.
+  """
+  def source_repo_priv(repo) do
+    config = repo.config()
+    priv = config[:priv] || "priv/#{repo |> Module.split() |> List.last() |> Macro.underscore()}"
+    Path.join(Application.app_dir(:pleroma), priv)
+  end
+
+  defp raise_missing_migrations(path, repo) do
+    raise("""
+    Could not find migrations directory #{inspect(path)}
+    for repo #{inspect(repo)}.
+    This may be because you are in a new project and the
+    migration directory has not been created yet. Creating an
+    empty directory at the path above will fix this error.
+    If you expected existing migrations to be found, please
+    make sure your repository has been properly configured
+    and the configured path exists.
+    """)
+  end
+end

lib/mix/tasks/pleroma/ecto/migrate.ex

@@ -0,0 +1,63 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-onl
+
+defmodule Mix.Tasks.Pleroma.Ecto.Migrate do
+  use Mix.Task
+  import Mix.Pleroma
+  require Logger
+
+  @shortdoc "Wrapper on `ecto.migrate` task."
+
+  @aliases [
+    n: :step,
+    v: :to
+  ]
+
+  @switches [
+    all: :boolean,
+    step: :integer,
+    to: :integer,
+    quiet: :boolean,
+    log_sql: :boolean,
+    strict_version_order: :boolean,
+    migrations_path: :string
+  ]
+
+  @moduledoc """
+  Changes `Logger` level to `:info` before start migration.
+  Changes level back when migration ends.
+
+  ## Start migration
+
+      mix pleroma.ecto.migrate [OPTIONS]
+
+  Options:
+    - see https://hexdocs.pm/ecto/2.0.0/Mix.Tasks.Ecto.Migrate.html
+  """
+
+  @impl true
+  def run(args \\ []) do
+    load_pleroma()
+    {opts, _} = OptionParser.parse!(args, strict: @switches, aliases: @aliases)
+
+    opts =
+      if opts[:to] || opts[:step] || opts[:all],
+        do: opts,
+        else: Keyword.put(opts, :all, true)
+
+    opts =
+      if opts[:quiet],
+        do: Keyword.merge(opts, log: false, log_sql: false),
+        else: opts
+
+    path = Mix.Tasks.Pleroma.Ecto.ensure_migrations_path(Pleroma.Repo, opts)
+
+    level = Logger.level()
+    Logger.configure(level: :info)
+
+    {:ok, _, _} = Ecto.Migrator.with_repo(Pleroma.Repo, &Ecto.Migrator.run(&1, path, :up, opts))
+
+    Logger.configure(level: level)
+  end
+end

lib/mix/tasks/pleroma/ecto/rollback.ex

@@ -0,0 +1,67 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-onl
+
+defmodule Mix.Tasks.Pleroma.Ecto.Rollback do
+  use Mix.Task
+  import Mix.Pleroma
+  require Logger
+  @shortdoc "Wrapper on `ecto.rollback` task"
+
+  @aliases [
+    n: :step,
+    v: :to
+  ]
+
+  @switches [
+    all: :boolean,
+    step: :integer,
+    to: :integer,
+    start: :boolean,
+    quiet: :boolean,
+    log_sql: :boolean,
+    migrations_path: :string
+  ]
+
+  @moduledoc """
+  Changes `Logger` level to `:info` before start rollback.
+  Changes level back when rollback ends.
+
+  ## Start rollback
+
+      mix pleroma.ecto.rollback
+
+  Options:
+    - see https://hexdocs.pm/ecto/2.0.0/Mix.Tasks.Ecto.Rollback.html
+  """
+
+  @impl true
+  def run(args \\ []) do
+    load_pleroma()
+    {opts, _} = OptionParser.parse!(args, strict: @switches, aliases: @aliases)
+
+    opts =
+      if opts[:to] || opts[:step] || opts[:all],
+        do: opts,
+        else: Keyword.put(opts, :step, 1)
+
+    opts =
+      if opts[:quiet],
+        do: Keyword.merge(opts, log: false, log_sql: false),
+        else: opts
+
+    path = Mix.Tasks.Pleroma.Ecto.ensure_migrations_path(Pleroma.Repo, opts)
+
+    level = Logger.level()
+    Logger.configure(level: :info)
+
+    if Pleroma.Config.get(:env) == :test do
+      Logger.info("Rollback succesfully")
+    else
+      {:ok, _, _} =
+        Ecto.Migrator.with_repo(Pleroma.Repo, &Ecto.Migrator.run(&1, path, :down, opts))
+    end
+
+    Logger.configure(level: level)
+  end
+end

lib/mix/tasks/pleroma/emoji.ex

@@ -55,15 +55,13 @@ defmodule Mix.Tasks.Pleroma.Emoji do
   are extracted).
   """
 
-  @default_manifest Pleroma.Config.get!([:emoji, :default_manifest])
-
   def run(["ls-packs" | args]) do
     Application.ensure_all_started(:hackney)
 
     {options, [], []} = parse_global_opts(args)
 
     manifest =
-      fetch_manifest(if options[:manifest], do: options[:manifest], else: @default_manifest)
+      fetch_manifest(if options[:manifest], do: options[:manifest], else: default_manifest())
 
     Enum.each(manifest, fn {name, info} ->
       to_print = [
@@ -88,7 +86,7 @@ defmodule Mix.Tasks.Pleroma.Emoji do
 
     {options, pack_names, []} = parse_global_opts(args)
 
-    manifest_url = if options[:manifest], do: options[:manifest], else: @default_manifest
+    manifest_url = if options[:manifest], do: options[:manifest], else: default_manifest()
 
     manifest = fetch_manifest(manifest_url)
 
@@ -298,4 +296,6 @@ defmodule Mix.Tasks.Pleroma.Emoji do
 
     Tesla.client(middleware)
   end
+
+  defp default_manifest, do: Pleroma.Config.get!([:emoji, :default_manifest])
 end

lib/mix/tasks/pleroma/instance.ex

@@ -4,7 +4,7 @@
 
 defmodule Mix.Tasks.Pleroma.Instance do
   use Mix.Task
-  alias Mix.Tasks.Pleroma.Common
+  import Mix.Pleroma
 
   @shortdoc "Manages Pleroma instance"
   @moduledoc """
@@ -29,7 +29,13 @@ defmodule Mix.Tasks.Pleroma.Instance do
   - `--dbname DBNAME` - the name of the database to use
   - `--dbuser DBUSER` - the user (aka role) to use for the database connection
   - `--dbpass DBPASS` - the password to use for the database connection
+  - `--rum Y/N` - Whether to enable RUM indexes
   - `--indexable Y/N` - Allow/disallow indexing site by search engines
+  - `--db-configurable Y/N` - Allow/disallow configuring instance from admin part
+  - `--uploads-dir` - the directory uploads go in when using a local uploader
+  - `--static-dir` - the directory custom public files should be read from (custom emojis, frontend bundle overrides, robots.txt, etc.)
+  - `--listen-ip` - the ip the app should listen to, defaults to 127.0.0.1
+  - `--listen-port` - the port the app should listen to, defaults to 4000
   """
 
   def run(["gen" | rest]) do
@@ -48,7 +54,13 @@ defmodule Mix.Tasks.Pleroma.Instance do
           dbname: :string,
           dbuser: :string,
           dbpass: :string,
-          indexable: :string
+          rum: :string,
+          indexable: :string,
+          db_configurable: :string,
+          uploads_dir: :string,
+          static_dir: :string,
+          listen_ip: :string,
+          listen_port: :string
         ],
         aliases: [
           o: :output,
@@ -68,7 +80,7 @@ defmodule Mix.Tasks.Pleroma.Instance do
     if proceed? do
       [domain, port | _] =
         String.split(
-          Common.get_option(
+          get_option(
             options,
             :domain,
             "What domain will your instance use? (e.g pleroma.soykaf.com)"
@@ -77,16 +89,16 @@ defmodule Mix.Tasks.Pleroma.Instance do
         ) ++ [443]
 
       name =
-        Common.get_option(
+        get_option(
           options,
           :instance_name,
           "What is the name of your instance? (e.g. Pleroma/Soykaf)"
         )
 
-      email = Common.get_option(options, :admin_email, "What is your admin email address?")
+      email = get_option(options, :admin_email, "What is your admin email address?")
 
       notify_email =
-        Common.get_option(
+        get_option(
           options,
           :notify_email,
           "What email address do you want to use for sending email notifications?",
@@ -94,21 +106,27 @@ defmodule Mix.Tasks.Pleroma.Instance do
         )
 
       indexable =
-        Common.get_option(
+        get_option(
           options,
           :indexable,
           "Do you want search engines to index your site? (y/n)",
           "y"
         ) === "y"
 
-      dbhost =
-        Common.get_option(options, :dbhost, "What is the hostname of your database?", "localhost")
+      db_configurable? =
+        get_option(
+          options,
+          :db_configurable,
+          "Do you want to store the configuration in the database (allows controlling it from admin-fe)? (y/n)",
+          "n"
+        ) === "y"
 
-      dbname =
-        Common.get_option(options, :dbname, "What is the name of your database?", "pleroma_dev")
+      dbhost = get_option(options, :dbhost, "What is the hostname of your database?", "localhost")
+
+      dbname = get_option(options, :dbname, "What is the name of your database?", "pleroma")
 
       dbuser =
-        Common.get_option(
+        get_option(
           options,
           :dbuser,
           "What is the user used to connect to your database?",
@@ -116,7 +134,7 @@ defmodule Mix.Tasks.Pleroma.Instance do
         )
 
       dbpass =
-        Common.get_option(
+        get_option(
           options,
           :dbpass,
           "What is the password used to connect to your database?",
@@ -124,13 +142,54 @@ defmodule Mix.Tasks.Pleroma.Instance do
           "autogenerated"
         )
 
+      rum_enabled =
+        get_option(
+          options,
+          :rum,
+          "Would you like to use RUM indices?",
+          "n"
+        ) === "y"
+
+      listen_port =
+        get_option(
+          options,
+          :listen_port,
+          "What port will the app listen to (leave it if you are using the default setup with nginx)?",
+          4000
+        )
+
+      listen_ip =
+        get_option(
+          options,
+          :listen_ip,
+          "What ip will the app listen to (leave it if you are using the default setup with nginx)?",
+          "127.0.0.1"
+        )
+
+      uploads_dir =
+        get_option(
+          options,
+          :uploads_dir,
+          "What directory should media uploads go in (when using the local uploader)?",
+          Pleroma.Config.get([Pleroma.Uploaders.Local, :uploads])
+        )
+
+      static_dir =
+        get_option(
+          options,
+          :static_dir,
+          "What directory should custom public files be read from (custom emojis, frontend bundle overrides, robots.txt, etc.)?",
+          Pleroma.Config.get([:instance, :static_dir])
+        )
+
       secret = :crypto.strong_rand_bytes(64) |> Base.encode64() |> binary_part(0, 64)
       signing_salt = :crypto.strong_rand_bytes(8) |> Base.encode64() |> binary_part(0, 8)
       {web_push_public_key, web_push_private_key} = :crypto.generate_key(:ecdh, :prime256v1)
+      template_dir = Application.app_dir(:pleroma, "priv") <> "/templates"
 
       result_config =
         EEx.eval_file(
-          "sample_config.eex" |> Path.expand(__DIR__),
+          template_dir <> "/sample_config.eex",
           domain: domain,
           port: port,
           email: email,
@@ -140,46 +199,40 @@ defmodule Mix.Tasks.Pleroma.Instance do
           dbname: dbname,
           dbuser: dbuser,
           dbpass: dbpass,
-          version: Pleroma.Mixfile.project() |> Keyword.get(:version),
           secret: secret,
           signing_salt: signing_salt,
           web_push_public_key: Base.url_encode64(web_push_public_key, padding: false),
-          web_push_private_key: Base.url_encode64(web_push_private_key, padding: false)
+          web_push_private_key: Base.url_encode64(web_push_private_key, padding: false),
+          db_configurable?: db_configurable?,
+          static_dir: static_dir,
+          uploads_dir: uploads_dir,
+          rum_enabled: rum_enabled,
+          listen_ip: listen_ip,
+          listen_port: listen_port
         )
 
       result_psql =
         EEx.eval_file(
-          "sample_psql.eex" |> Path.expand(__DIR__),
+          template_dir <> "/sample_psql.eex",
           dbname: dbname,
           dbuser: dbuser,
-          dbpass: dbpass
+          dbpass: dbpass,
+          rum_enabled: rum_enabled
         )
 
-      Mix.shell().info(
-        "Writing config to #{config_path}. You should rename it to config/prod.secret.exs or config/dev.secret.exs."
-      )
+      shell_info("Writing config to #{config_path}.")
 
       File.write(config_path, result_config)
-      Mix.shell().info("Writing #{psql_path}.")
+      shell_info("Writing the postgres script to #{psql_path}.")
       File.write(psql_path, result_psql)
 
-      write_robots_txt(indexable)
+      write_robots_txt(indexable, template_dir)
 
-      Mix.shell().info(
-        "\n" <>
-          """
-          To get started:
-          1. Verify the contents of the generated files.
-          2. Run `sudo -u postgres psql -f #{Common.escape_sh_path(psql_path)}`.
-          """ <>
-          if config_path in ["config/dev.secret.exs", "config/prod.secret.exs"] do
-            ""
-          else
-            "3. Run `mv #{Common.escape_sh_path(config_path)} 'config/prod.secret.exs'`."
-          end
+      shell_info(
+        "\n All files successfully written! Refer to the installation instructions for your platform for next steps"
       )
     else
-      Mix.shell().error(
+      shell_error(
         "The task would have overwritten the following files:\n" <>
           (Enum.map(paths, &"- #{&1}\n") |> Enum.join("")) <>
           "Rerun with `--force` to overwrite them."
@@ -187,10 +240,10 @@ defmodule Mix.Tasks.Pleroma.Instance do
     end
   end
 
-  defp write_robots_txt(indexable) do
+  defp write_robots_txt(indexable, template_dir) do
     robots_txt =
       EEx.eval_file(
-        Path.expand("robots_txt.eex", __DIR__),
+        template_dir <> "/robots_txt.eex",
         indexable: indexable
       )
 
@@ -204,10 +257,10 @@ defmodule Mix.Tasks.Pleroma.Instance do
 
     if File.exists?(robots_txt_path) do
       File.cp!(robots_txt_path, "#{robots_txt_path}.bak")
-      Mix.shell().info("Backing up existing robots.txt to #{robots_txt_path}.bak")
+      shell_info("Backing up existing robots.txt to #{robots_txt_path}.bak")
     end
 
     File.write(robots_txt_path, robots_txt)
-    Mix.shell().info("Writing #{robots_txt_path}.")
+    shell_info("Writing #{robots_txt_path}.")
   end
 end

lib/mix/tasks/pleroma/relay.ex

@@ -4,7 +4,7 @@
 
 defmodule Mix.Tasks.Pleroma.Relay do
   use Mix.Task
-  alias Mix.Tasks.Pleroma.Common
+  import Mix.Pleroma
   alias Pleroma.Web.ActivityPub.Relay
 
   @shortdoc "Manages remote relays"
@@ -24,24 +24,24 @@ defmodule Mix.Tasks.Pleroma.Relay do
   Example: ``mix pleroma.relay unfollow https://example.org/relay``
   """
   def run(["follow", target]) do
-    Common.start_pleroma()
+    start_pleroma()
 
     with {:ok, _activity} <- Relay.follow(target) do
       # put this task to sleep to allow the genserver to push out the messages
       :timer.sleep(500)
     else
-      {:error, e} -> Mix.shell().error("Error while following #{target}: #{inspect(e)}")
+      {:error, e} -> shell_error("Error while following #{target}: #{inspect(e)}")
     end
   end
 
   def run(["unfollow", target]) do
-    Common.start_pleroma()
+    start_pleroma()
 
     with {:ok, _activity} <- Relay.unfollow(target) do
       # put this task to sleep to allow the genserver to push out the messages
       :timer.sleep(500)
     else
-      {:error, e} -> Mix.shell().error("Error while following #{target}: #{inspect(e)}")
+      {:error, e} -> shell_error("Error while following #{target}: #{inspect(e)}")
     end
   end
 end

lib/mix/tasks/pleroma/robots_txt.eex

@@ -1,2 +0,0 @@
-User-Agent: *
-Disallow: <%= if indexable, do: "", else: "/" %>

lib/mix/tasks/pleroma/sample_config.eex

@@ -1,78 +0,0 @@
-# Pleroma instance configuration
-
-# NOTE: This file should not be committed to a repo or otherwise made public
-# without removing sensitive information.
-
-use Mix.Config
-
-config :pleroma, Pleroma.Web.Endpoint,
-   url: [host: "<%= domain %>", scheme: "https", port: <%= port %>],
-   secret_key_base: "<%= secret %>",
-   signing_salt: "<%= signing_salt %>"
-
-config :pleroma, :instance,
-  name: "<%= name %>",
-  email: "<%= email %>",
-  notify_email: "<%= notify_email %>",
-  limit: 5000,
-  registrations_open: true,
-  dedupe_media: false
-
-config :pleroma, :media_proxy,
-  enabled: false,
-  redirect_on_failure: true
-  #base_url: "https://cache.pleroma.social"
-
-config :pleroma, Pleroma.Repo,
-  adapter: Ecto.Adapters.Postgres,
-  username: "<%= dbuser %>",
-  password: "<%= dbpass %>",
-  database: "<%= dbname %>",
-  hostname: "<%= dbhost %>",
-  pool_size: 10
-
-# Configure web push notifications
-config :web_push_encryption, :vapid_details,
-  subject: "mailto:<%= email %>",
-  public_key: "<%= web_push_public_key %>",
-  private_key: "<%= web_push_private_key %>"
-
-# Enable Strict-Transport-Security once SSL is working:
-# config :pleroma, :http_security,
-#   sts: true
-
-# Configure S3 support if desired.
-# The public S3 endpoint is different depending on region and provider,
-# consult your S3 provider's documentation for details on what to use.
-#
-# config :pleroma, Pleroma.Uploaders.S3,
-#   bucket: "some-bucket",
-#   public_endpoint: "https://s3.amazonaws.com"
-#
-# Configure S3 credentials:
-# config :ex_aws, :s3,
-#   access_key_id: "xxxxxxxxxxxxx",
-#   secret_access_key: "yyyyyyyyyyyy",
-#   region: "us-east-1",
-#   scheme: "https://"
-#
-# For using third-party S3 clones like wasabi, also do:
-# config :ex_aws, :s3,
-#   host: "s3.wasabisys.com"
-
-
-# Configure Openstack Swift support if desired.
-#
-# Many openstack deployments are different, so config is left very open with
-# no assumptions made on which provider you're using. This should allow very
-# wide support without needing separate handlers for OVH, Rackspace, etc.
-#
-# config :pleroma, Pleroma.Uploaders.Swift,
-#  container: "some-container",
-#  username: "api-username-yyyy",
-#  password: "api-key-xxxx",
-#  tenant_id: "<openstack-project/tenant-id>",
-#  auth_url: "https://keystone-endpoint.provider.com",
-#  storage_url: "https://swift-endpoint.prodider.com/v1/AUTH_<tenant>/<container>",
-#  object_url: "https://cdn-endpoint.provider.com/<container>"
-#

lib/mix/tasks/pleroma/sample_psql.eex

@@ -1,7 +0,0 @@
-CREATE USER <%= dbuser %> WITH ENCRYPTED PASSWORD '<%= dbpass %>';
-CREATE DATABASE <%= dbname %> OWNER <%= dbuser %>;
-\c <%= dbname %>;
---Extensions made by ecto.migrate that need superuser access
-CREATE EXTENSION IF NOT EXISTS citext;
-CREATE EXTENSION IF NOT EXISTS pg_trgm;
-CREATE EXTENSION IF NOT EXISTS "uuid-ossp";

lib/mix/tasks/pleroma/uploads.ex

@@ -4,7 +4,7 @@
 
 defmodule Mix.Tasks.Pleroma.Uploads do
   use Mix.Task
-  alias Mix.Tasks.Pleroma.Common
+  import Mix.Pleroma
   alias Pleroma.Upload
   alias Pleroma.Uploaders.Local
   require Logger
@@ -24,7 +24,7 @@ defmodule Mix.Tasks.Pleroma.Uploads do
   """
   def run(["migrate_local", target_uploader | args]) do
     delete? = Enum.member?(args, "--delete")
-    Common.start_pleroma()
+    start_pleroma()
     local_path = Pleroma.Config.get!([Local, :uploads])
     uploader = Module.concat(Pleroma.Uploaders, target_uploader)
 
@@ -38,10 +38,10 @@ defmodule Mix.Tasks.Pleroma.Uploads do
       Pleroma.Config.put([Upload, :uploader], uploader)
     end
 
-    Mix.shell().info("Migrating files from local #{local_path} to #{to_string(uploader)}")
+    shell_info("Migrating files from local #{local_path} to #{to_string(uploader)}")
 
     if delete? do
-      Mix.shell().info(
+      shell_info(
         "Attention: uploaded files will be deleted, hope you have backups! (--delete ; cancel with ^C)"
       )
 
@@ -78,7 +78,7 @@ defmodule Mix.Tasks.Pleroma.Uploads do
       |> Enum.filter(& &1)
 
     total_count = length(uploads)
-    Mix.shell().info("Found #{total_count} uploads")
+    shell_info("Found #{total_count} uploads")
 
     uploads
     |> Task.async_stream(
@@ -90,7 +90,7 @@ defmodule Mix.Tasks.Pleroma.Uploads do
             :ok
 
           error ->
-            Mix.shell().error("failed to upload #{inspect(upload.path)}: #{inspect(error)}")
+            shell_error("failed to upload #{inspect(upload.path)}: #{inspect(error)}")
         end
       end,
       timeout: 150_000
@@ -99,10 +99,10 @@ defmodule Mix.Tasks.Pleroma.Uploads do
     # credo:disable-for-next-line Credo.Check.Warning.UnusedEnumOperation
     |> Enum.reduce(0, fn done, count ->
       count = count + length(done)
-      Mix.shell().info("Uploaded #{count}/#{total_count} files")
+      shell_info("Uploaded #{count}/#{total_count} files")
       count
     end)
 
-    Mix.shell().info("Done!")
+    shell_info("Done!")
   end
 end

lib/mix/tasks/pleroma/user.ex

@@ -5,9 +5,10 @@
 defmodule Mix.Tasks.Pleroma.User do
   use Mix.Task
   import Ecto.Changeset
-  alias Mix.Tasks.Pleroma.Common
+  import Mix.Pleroma
   alias Pleroma.User
   alias Pleroma.UserInviteToken
+  alias Pleroma.Web.OAuth
 
   @shortdoc "Manages Pleroma users"
   @moduledoc """
@@ -49,6 +50,10 @@ defmodule Mix.Tasks.Pleroma.User do
 
       mix pleroma.user delete_activities NICKNAME
 
+  ## Sign user out from all applications (delete user's OAuth tokens and authorizations).
+
+      mix pleroma.user sign_out NICKNAME
+
   ## Deactivate or activate the user's account.
 
       mix pleroma.user toggle_activated NICKNAME
@@ -115,7 +120,7 @@ defmodule Mix.Tasks.Pleroma.User do
     admin? = Keyword.get(options, :admin, false)
     assume_yes? = Keyword.get(options, :assume_yes, false)
 
-    Mix.shell().info("""
+    shell_info("""
     A user will be created with the following information:
       - nickname: #{nickname}
       - email: #{email}
@@ -128,10 +133,10 @@ defmodule Mix.Tasks.Pleroma.User do
       - admin: #{if(admin?, do: "true", else: "false")}
     """)
 
-    proceed? = assume_yes? or Mix.shell().yes?("Continue?")
+    proceed? = assume_yes? or shell_yes?("Continue?")
 
     if proceed? do
-      Common.start_pleroma()
+      start_pleroma()
 
       params = %{
         nickname: nickname,
@@ -145,7 +150,7 @@ defmodule Mix.Tasks.Pleroma.User do
       changeset = User.register_changeset(%User{}, params, need_confirmation: false)
       {:ok, _user} = User.register(changeset)
 
-      Mix.shell().info("User #{nickname} created")
+      shell_info("User #{nickname} created")
 
       if moderator? do
         run(["set", nickname, "--moderator"])
@@ -159,64 +164,64 @@ defmodule Mix.Tasks.Pleroma.User do
         run(["reset_password", nickname])
       end
     else
-      Mix.shell().info("User will not be created.")
+      shell_info("User will not be created.")
     end
   end
 
   def run(["rm", nickname]) do
-    Common.start_pleroma()
+    start_pleroma()
 
     with %User{local: true} = user <- User.get_cached_by_nickname(nickname) do
       User.perform(:delete, user)
-      Mix.shell().info("User #{nickname} deleted.")
+      shell_info("User #{nickname} deleted.")
     else
       _ ->
-        Mix.shell().error("No local user #{nickname}")
+        shell_error("No local user #{nickname}")
     end
   end
 
   def run(["toggle_activated", nickname]) do
-    Common.start_pleroma()
+    start_pleroma()
 
     with %User{} = user <- User.get_cached_by_nickname(nickname) do
       {:ok, user} = User.deactivate(user, !user.info.deactivated)
 
-      Mix.shell().info(
+      shell_info(
         "Activation status of #{nickname}: #{if(user.info.deactivated, do: "de", else: "")}activated"
       )
     else
       _ ->
-        Mix.shell().error("No user #{nickname}")
+        shell_error("No user #{nickname}")
     end
   end
 
   def run(["reset_password", nickname]) do
-    Common.start_pleroma()
+    start_pleroma()
 
     with %User{local: true} = user <- User.get_cached_by_nickname(nickname),
          {:ok, token} <- Pleroma.PasswordResetToken.create_token(user) do
-      Mix.shell().info("Generated password reset token for #{user.nickname}")
+      shell_info("Generated password reset token for #{user.nickname}")
 
       IO.puts(
         "URL: #{
-          Pleroma.Web.Router.Helpers.util_url(
+          Pleroma.Web.Router.Helpers.reset_password_url(
             Pleroma.Web.Endpoint,
-            :show_password_reset,
+            :reset,
             token.token
           )
         }"
       )
     else
       _ ->
-        Mix.shell().error("No local user #{nickname}")
+        shell_error("No local user #{nickname}")
     end
   end
 
   def run(["unsubscribe", nickname]) do
-    Common.start_pleroma()
+    start_pleroma()
 
     with %User{} = user <- User.get_cached_by_nickname(nickname) do
-      Mix.shell().info("Deactivating #{user.nickname}")
+      shell_info("Deactivating #{user.nickname}")
       User.deactivate(user)
 
       {:ok, friends} = User.get_friends(user)
@@ -224,7 +229,7 @@ defmodule Mix.Tasks.Pleroma.User do
       Enum.each(friends, fn friend ->
         user = User.get_cached_by_id(user.id)
 
-        Mix.shell().info("Unsubscribing #{friend.nickname} from #{user.nickname}")
+        shell_info("Unsubscribing #{friend.nickname} from #{user.nickname}")
         User.unfollow(user, friend)
       end)
 
@@ -233,16 +238,16 @@ defmodule Mix.Tasks.Pleroma.User do
       user = User.get_cached_by_id(user.id)
 
       if Enum.empty?(user.following) do
-        Mix.shell().info("Successfully unsubscribed all followers from #{user.nickname}")
+        shell_info("Successfully unsubscribed all followers from #{user.nickname}")
       end
     else
       _ ->
-        Mix.shell().error("No user #{nickname}")
+        shell_error("No user #{nickname}")
     end
   end
 
   def run(["set", nickname | rest]) do
-    Common.start_pleroma()
+    start_pleroma()
 
     {options, [], []} =
       OptionParser.parse(
@@ -274,33 +279,33 @@ defmodule Mix.Tasks.Pleroma.User do
         end
     else
       _ ->
-        Mix.shell().error("No local user #{nickname}")
+        shell_error("No local user #{nickname}")
     end
   end
 
   def run(["tag", nickname | tags]) do
-    Common.start_pleroma()
+    start_pleroma()
 
     with %User{} = user <- User.get_cached_by_nickname(nickname) do
       user = user |> User.tag(tags)
 
-      Mix.shell().info("Tags of #{user.nickname}: #{inspect(tags)}")
+      shell_info("Tags of #{user.nickname}: #{inspect(tags)}")
     else
       _ ->
-        Mix.shell().error("Could not change user tags for #{nickname}")
+        shell_error("Could not change user tags for #{nickname}")
     end
   end
 
   def run(["untag", nickname | tags]) do
-    Common.start_pleroma()
+    start_pleroma()
 
     with %User{} = user <- User.get_cached_by_nickname(nickname) do
       user = user |> User.untag(tags)
 
-      Mix.shell().info("Tags of #{user.nickname}: #{inspect(tags)}")
+      shell_info("Tags of #{user.nickname}: #{inspect(tags)}")
     else
       _ ->
-        Mix.shell().error("Could not change user tags for #{nickname}")
+        shell_error("Could not change user tags for #{nickname}")
     end
   end
 
@@ -321,14 +326,12 @@ defmodule Mix.Tasks.Pleroma.User do
       end)
       |> Enum.into(%{})
 
-    Common.start_pleroma()
+    start_pleroma()
 
     with {:ok, val} <- options[:expires_at],
          options = Map.put(options, :expires_at, val),
          {:ok, invite} <- UserInviteToken.create_invite(options) do
-      Mix.shell().info(
-        "Generated user invite token " <> String.replace(invite.invite_type, "_", " ")
-      )
+      shell_info("Generated user invite token " <> String.replace(invite.invite_type, "_", " "))
 
       url =
         Pleroma.Web.Router.Helpers.redirect_url(
@@ -340,14 +343,14 @@ defmodule Mix.Tasks.Pleroma.User do
       IO.puts(url)
     else
       error ->
-        Mix.shell().error("Could not create invite token: #{inspect(error)}")
+        shell_error("Could not create invite token: #{inspect(error)}")
     end
   end
 
   def run(["invites"]) do
-    Common.start_pleroma()
+    start_pleroma()
 
-    Mix.shell().info("Invites list:")
+    shell_info("Invites list:")
 
     UserInviteToken.list_invites()
     |> Enum.each(fn invite ->
@@ -361,7 +364,7 @@ defmodule Mix.Tasks.Pleroma.User do
           " | Max use: #{max_use}    Left use: #{max_use - invite.uses}"
         end
 
-      Mix.shell().info(
+      shell_info(
         "ID: #{invite.id} | Token: #{invite.token} | Token type: #{invite.invite_type} | Used: #{
           invite.used
         }#{expire_info}#{using_info}"
@@ -370,40 +373,54 @@ defmodule Mix.Tasks.Pleroma.User do
   end
 
   def run(["revoke_invite", token]) do
-    Common.start_pleroma()
+    start_pleroma()
 
     with {:ok, invite} <- UserInviteToken.find_by_token(token),
          {:ok, _} <- UserInviteToken.update_invite(invite, %{used: true}) do
-      Mix.shell().info("Invite for token #{token} was revoked.")
+      shell_info("Invite for token #{token} was revoked.")
     else
-      _ -> Mix.shell().error("No invite found with token #{token}")
+      _ -> shell_error("No invite found with token #{token}")
     end
   end
 
   def run(["delete_activities", nickname]) do
-    Common.start_pleroma()
+    start_pleroma()
 
     with %User{local: true} = user <- User.get_cached_by_nickname(nickname) do
       {:ok, _} = User.delete_user_activities(user)
-      Mix.shell().info("User #{nickname} statuses deleted.")
+      shell_info("User #{nickname} statuses deleted.")
     else
       _ ->
-        Mix.shell().error("No local user #{nickname}")
+        shell_error("No local user #{nickname}")
     end
   end
 
   def run(["toggle_confirmed", nickname]) do
-    Common.start_pleroma()
+    start_pleroma()
 
     with %User{} = user <- User.get_cached_by_nickname(nickname) do
       {:ok, user} = User.toggle_confirmation(user)
 
       message = if user.info.confirmation_pending, do: "needs", else: "doesn't need"
 
-      Mix.shell().info("#{nickname} #{message} confirmation.")
+      shell_info("#{nickname} #{message} confirmation.")
     else
       _ ->
-        Mix.shell().error("No local user #{nickname}")
+        shell_error("No local user #{nickname}")
+    end
+  end
+
+  def run(["sign_out", nickname]) do
+    start_pleroma()
+
+    with %User{local: true} = user <- User.get_cached_by_nickname(nickname) do
+      OAuth.Token.delete_user_tokens(user)
+      OAuth.Authorization.delete_user_authorizations(user)
+
+      shell_info("#{nickname} signed out from all apps.")
+    else
+      _ ->
+        shell_error("No local user #{nickname}")
     end
   end
 
@@ -416,7 +433,7 @@ defmodule Mix.Tasks.Pleroma.User do
 
     {:ok, user} = User.update_and_set_cache(user_cng)
 
-    Mix.shell().info("Moderator status of #{user.nickname}: #{user.info.is_moderator}")
+    shell_info("Moderator status of #{user.nickname}: #{user.info.is_moderator}")
     user
   end
 
@@ -429,7 +446,7 @@ defmodule Mix.Tasks.Pleroma.User do
 
     {:ok, user} = User.update_and_set_cache(user_cng)
 
-    Mix.shell().info("Admin status of #{user.nickname}: #{user.info.is_admin}")
+    shell_info("Admin status of #{user.nickname}: #{user.info.is_admin}")
     user
   end
 
@@ -442,7 +459,7 @@ defmodule Mix.Tasks.Pleroma.User do
 
     {:ok, user} = User.update_and_set_cache(user_cng)
 
-    Mix.shell().info("Locked status of #{user.nickname}: #{user.info.locked}")
+    shell_info("Locked status of #{user.nickname}: #{user.info.locked}")
     user
   end
 end

lib/pleroma/activity.ex

@@ -343,4 +343,6 @@ defmodule Pleroma.Activity do
         )
     )
   end
+
+  defdelegate search(user, query), to: Pleroma.Activity.Search
 end

lib/pleroma/activity/search.ex

@@ -0,0 +1,81 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Activity.Search do
+  alias Pleroma.Activity
+  alias Pleroma.Object.Fetcher
+  alias Pleroma.Repo
+  alias Pleroma.User
+  alias Pleroma.Web.ActivityPub.Visibility
+
+  import Ecto.Query
+
+  def search(user, search_query) do
+    index_type = if Pleroma.Config.get([:database, :rum_enabled]), do: :rum, else: :gin
+
+    Activity
+    |> Activity.with_preloaded_object()
+    |> Activity.restrict_deactivated_users()
+    |> restrict_public()
+    |> query_with(index_type, search_query)
+    |> maybe_restrict_local(user)
+    |> Repo.all()
+    |> maybe_fetch(user, search_query)
+  end
+
+  defp restrict_public(q) do
+    from([a, o] in q,
+      where: fragment("?->>'type' = 'Create'", a.data),
+      where: "https://www.w3.org/ns/activitystreams#Public" in a.recipients,
+      limit: 40
+    )
+  end
+
+  defp query_with(q, :gin, search_query) do
+    from([a, o] in q,
+      where:
+        fragment(
+          "to_tsvector('english', ?->>'content') @@ plainto_tsquery('english', ?)",
+          o.data,
+          ^search_query
+        )
+    )
+  end
+
+  defp query_with(q, :rum, search_query) do
+    from([a, o] in q,
+      where:
+        fragment(
+          "? @@ plainto_tsquery('english', ?)",
+          o.fts_content,
+          ^search_query
+        ),
+      order_by: [fragment("? <=> now()::date", o.inserted_at)]
+    )
+  end
+
+  defp maybe_restrict_local(q, user) do
+    limit = Pleroma.Config.get([:instance, :limit_to_local_content], :unauthenticated)
+
+    case {limit, user} do
+      {:all, _} -> restrict_local(q)
+      {:unauthenticated, %User{}} -> q
+      {:unauthenticated, _} -> restrict_local(q)
+      {false, _} -> q
+    end
+  end
+
+  defp restrict_local(q), do: where(q, local: true)
+
+  defp maybe_fetch(activities, user, search_query) do
+    with true <- Regex.match?(~r/https?:/, search_query),
+         {:ok, object} <- Fetcher.fetch_object_from_id(search_query),
+         %Activity{} = activity <- Activity.get_create_by_object_ap_id(object.data["id"]),
+         true <- Visibility.visible_for_user?(activity, user) do
+      activities ++ [activity]
+    else
+      _ -> activities
+    end
+  end
+end

lib/pleroma/application.ex

@@ -4,7 +4,6 @@
 
 defmodule Pleroma.Application do
   use Application
-  import Supervisor.Spec
 
   @name Mix.Project.config()[:name]
   @version Mix.Project.config()[:version]
@@ -31,96 +30,132 @@ defmodule Pleroma.Application do
     children =
       [
         # Start the Ecto repository
-        supervisor(Pleroma.Repo, []),
-        worker(Pleroma.Emoji, []),
-        worker(Pleroma.Captcha, []),
-        worker(
-          Cachex,
-          [
-            :used_captcha_cache,
-            [
-              ttl_interval: :timer.seconds(Pleroma.Config.get!([Pleroma.Captcha, :seconds_valid]))
-            ]
-          ],
-          id: :cachex_used_captcha_cache
-        ),
-        worker(
-          Cachex,
-          [
-            :user_cache,
-            [
-              default_ttl: 25_000,
-              ttl_interval: 1000,
-              limit: 2500
-            ]
-          ],
-          id: :cachex_user
-        ),
-        worker(
-          Cachex,
-          [
-            :object_cache,
-            [
-              default_ttl: 25_000,
-              ttl_interval: 1000,
-              limit: 2500
-            ]
-          ],
-          id: :cachex_object
-        ),
-        worker(
-          Cachex,
-          [
-            :rich_media_cache,
-            [
-              default_ttl: :timer.minutes(120),
-              limit: 5000
-            ]
-          ],
-          id: :cachex_rich_media
-        ),
-        worker(
-          Cachex,
-          [
-            :scrubber_cache,
-            [
-              limit: 2500
-            ]
-          ],
-          id: :cachex_scrubber
-        ),
-        worker(
-          Cachex,
-          [
-            :idempotency_cache,
-            [
-              expiration:
-                expiration(
-                  default: :timer.seconds(6 * 60 * 60),
-                  interval: :timer.seconds(60)
-                ),
-              limit: 2500
-            ]
-          ],
-          id: :cachex_idem
-        ),
-        worker(Pleroma.FlakeId, []),
-        worker(Pleroma.ScheduledActivityWorker, [])
+        %{id: Pleroma.Repo, start: {Pleroma.Repo, :start_link, []}, type: :supervisor},
+        %{id: Pleroma.Config.TransferTask, start: {Pleroma.Config.TransferTask, :start_link, []}},
+        %{id: Pleroma.Emoji, start: {Pleroma.Emoji, :start_link, []}},
+        %{id: Pleroma.Captcha, start: {Pleroma.Captcha, :start_link, []}},
+        %{
+          id: :cachex_used_captcha_cache,
+          start:
+            {Cachex, :start_link,
+             [
+               :used_captcha_cache,
+               [
+                 ttl_interval:
+                   :timer.seconds(Pleroma.Config.get!([Pleroma.Captcha, :seconds_valid]))
+               ]
+             ]}
+        },
+        %{
+          id: :cachex_user,
+          start:
+            {Cachex, :start_link,
+             [
+               :user_cache,
+               [
+                 default_ttl: 25_000,
+                 ttl_interval: 1000,
+                 limit: 2500
+               ]
+             ]}
+        },
+        %{
+          id: :cachex_object,
+          start:
+            {Cachex, :start_link,
+             [
+               :object_cache,
+               [
+                 default_ttl: 25_000,
+                 ttl_interval: 1000,
+                 limit: 2500
+               ]
+             ]}
+        },
+        %{
+          id: :cachex_rich_media,
+          start:
+            {Cachex, :start_link,
+             [
+               :rich_media_cache,
+               [
+                 default_ttl: :timer.minutes(120),
+                 limit: 5000
+               ]
+             ]}
+        },
+        %{
+          id: :cachex_scrubber,
+          start:
+            {Cachex, :start_link,
+             [
+               :scrubber_cache,
+               [
+                 limit: 2500
+               ]
+             ]}
+        },
+        %{
+          id: :cachex_idem,
+          start:
+            {Cachex, :start_link,
+             [
+               :idempotency_cache,
+               [
+                 expiration:
+                   expiration(
+                     default: :timer.seconds(6 * 60 * 60),
+                     interval: :timer.seconds(60)
+                   ),
+                 limit: 2500
+               ]
+             ]}
+        },
+        %{id: Pleroma.FlakeId, start: {Pleroma.FlakeId, :start_link, []}},
+        %{
+          id: Pleroma.ScheduledActivityWorker,
+          start: {Pleroma.ScheduledActivityWorker, :start_link, []}
+        }
       ] ++
         hackney_pool_children() ++
         [
-          worker(Pleroma.Web.Federator.RetryQueue, []),
-          worker(Pleroma.Web.OAuth.Token.CleanWorker, []),
-          worker(Pleroma.Stats, []),
-          worker(Task, [&Pleroma.Web.Push.init/0], restart: :temporary, id: :web_push_init),
-          worker(Task, [&Pleroma.Web.Federator.init/0], restart: :temporary, id: :federator_init)
+          %{
+            id: Pleroma.Web.Federator.RetryQueue,
+            start: {Pleroma.Web.Federator.RetryQueue, :start_link, []}
+          },
+          %{
+            id: Pleroma.Web.OAuth.Token.CleanWorker,
+            start: {Pleroma.Web.OAuth.Token.CleanWorker, :start_link, []}
+          },
+          %{
+            id: Pleroma.Stats,
+            start: {Pleroma.Stats, :start_link, []}
+          },
+          %{
+            id: :web_push_init,
+            start: {Task, :start_link, [&Pleroma.Web.Push.init/0]},
+            restart: :temporary
+          },
+          %{
+            id: :federator_init,
+            start: {Task, :start_link, [&Pleroma.Web.Federator.init/0]},
+            restart: :temporary
+          }
         ] ++
         streamer_child() ++
         chat_child() ++
         [
           # Start the endpoint when the application starts
-          supervisor(Pleroma.Web.Endpoint, []),
-          worker(Pleroma.Gopher.Server, [])
+          %{
+            id: Pleroma.Web.Endpoint,
+            start: {Pleroma.Web.Endpoint, :start_link, []},
+            type: :supervisor
+          },
+          %{id: Pleroma.Gopher.Server, start: {Pleroma.Gopher.Server, :start_link, []}},
+          %{
+            id: Pleroma.User.SynchronizationWorker,
+            start: {Pleroma.User.SynchronizationWorker, :start_link, []}
+          }
         ]
 
     # See http://elixir-lang.org/docs/stable/elixir/Supervisor.html
@@ -144,7 +179,6 @@ defmodule Pleroma.Application do
       Pleroma.Repo.Instrumenter.setup()
     end
 
-    Prometheus.Registry.register_collector(:prometheus_process_collector)
     Pleroma.Web.Endpoint.MetricsExporter.setup()
     Pleroma.Web.Endpoint.PipelineInstrumenter.setup()
     Pleroma.Web.Endpoint.Instrumenter.setup()
@@ -157,24 +191,29 @@ defmodule Pleroma.Application do
       else
         []
       end ++
-      if Pleroma.Config.get([Pleroma.Uploader, :proxy_remote]) do
+      if Pleroma.Config.get([Pleroma.Upload, :proxy_remote]) do
         [:upload]
       else
         []
       end
   end
 
-  if Mix.env() == :test do
+  if Pleroma.Config.get(:env) == :test do
     defp streamer_child, do: []
     defp chat_child, do: []
   else
     defp streamer_child do
-      [worker(Pleroma.Web.Streamer, [])]
+      [%{id: Pleroma.Web.Streamer, start: {Pleroma.Web.Streamer, :start_link, []}}]
     end
 
     defp chat_child do
       if Pleroma.Config.get([:chat, :enabled]) do
-        [worker(Pleroma.Web.ChatChannel.ChatChannelState, [])]
+        [
+          %{
+            id: Pleroma.Web.ChatChannel.ChatChannelState,
+            start: {Pleroma.Web.ChatChannel.ChatChannelState, :start_link, []}
+          }
+        ]
       else
         []
       end

lib/pleroma/bbs/authenticator.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.BBS.Authenticator do
   use Sshd.PasswordAuthenticator
   alias Comeonin.Pbkdf2

lib/pleroma/bbs/handler.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.BBS.Handler do
   use Sshd.ShellHandler
   alias Pleroma.Activity

lib/pleroma/bookmark.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Bookmark do
   use Ecto.Schema
 

lib/pleroma/captcha/captcha.ex

@@ -3,6 +3,8 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Captcha do
+  import Pleroma.Web.Gettext
+
   alias Calendar.DateTime
   alias Plug.Crypto.KeyGenerator
   alias Plug.Crypto.MessageEncryptor
@@ -83,10 +85,11 @@ defmodule Pleroma.Captcha do
       with {:ok, data} <- MessageEncryptor.decrypt(answer_data, secret, sign_secret),
            %{at: at, answer_data: answer_md5} <- :erlang.binary_to_term(data) do
         try do
-          if DateTime.before?(at, valid_if_after), do: throw({:error, "CAPTCHA expired"})
+          if DateTime.before?(at, valid_if_after),
+            do: throw({:error, dgettext("errors", "CAPTCHA expired")})
 
           if not is_nil(Cachex.get!(:used_captcha_cache, token)),
-            do: throw({:error, "CAPTCHA already used"})
+            do: throw({:error, dgettext("errors", "CAPTCHA already used")})
 
           res = method().validate(token, captcha, answer_md5)
           # Throw if an error occurs
@@ -101,7 +104,7 @@ defmodule Pleroma.Captcha do
           :throw, e -> e
         end
       else
-        _ -> {:error, "Invalid answer data"}
+        _ -> {:error, dgettext("errors", "Invalid answer data")}
       end
 
     {:reply, result, state}

lib/pleroma/captcha/kocaptcha.ex

@@ -3,6 +3,7 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Captcha.Kocaptcha do
+  import Pleroma.Web.Gettext
   alias Pleroma.Captcha.Service
   @behaviour Service
 
@@ -12,7 +13,7 @@ defmodule Pleroma.Captcha.Kocaptcha do
 
     case Tesla.get(endpoint <> "/new") do
       {:error, _} ->
-        %{error: "Kocaptcha service unavailable"}
+        %{error: dgettext("errors", "Kocaptcha service unavailable")}
 
       {:ok, res} ->
         json_resp = Jason.decode!(res.body)
@@ -32,6 +33,6 @@ defmodule Pleroma.Captcha.Kocaptcha do
     if not is_nil(captcha) and
          :crypto.hash(:md5, captcha) |> Base.encode16() == String.upcase(answer_data),
        do: :ok,
-       else: {:error, "Invalid CAPTCHA"}
+       else: {:error, dgettext("errors", "Invalid CAPTCHA")}
   end
 end

lib/pleroma/config.ex

@@ -38,7 +38,7 @@ defmodule Pleroma.Config do
 
   def put([parent_key | keys], value) do
     parent =
-      Application.get_env(:pleroma, parent_key)
+      Application.get_env(:pleroma, parent_key, [])
       |> put_in(keys, value)
 
     Application.put_env(:pleroma, parent_key, parent)

lib/pleroma/config/transfer_task.ex

@@ -0,0 +1,59 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Config.TransferTask do
+  use Task
+  alias Pleroma.Web.AdminAPI.Config
+
+  def start_link do
+    load_and_update_env()
+    if Pleroma.Config.get(:env) == :test, do: Ecto.Adapters.SQL.Sandbox.checkin(Pleroma.Repo)
+    :ignore
+  end
+
+  def load_and_update_env do
+    if Pleroma.Config.get([:instance, :dynamic_configuration]) and
+         Ecto.Adapters.SQL.table_exists?(Pleroma.Repo, "config") do
+      for_restart =
+        Pleroma.Repo.all(Config)
+        |> Enum.map(&update_env(&1))
+
+      # We need to restart applications for loaded settings take effect
+      for_restart
+      |> Enum.reject(&(&1 in [:pleroma, :ok]))
+      |> Enum.each(fn app ->
+        Application.stop(app)
+        :ok = Application.start(app)
+      end)
+    end
+  end
+
+  defp update_env(setting) do
+    try do
+      key =
+        if String.starts_with?(setting.key, "Pleroma.") do
+          "Elixir." <> setting.key
+        else
+          setting.key
+        end
+
+      group = String.to_existing_atom(setting.group)
+
+      Application.put_env(
+        group,
+        String.to_existing_atom(key),
+        Config.from_binary(setting.value)
+      )
+
+      group
+    rescue
+      e ->
+        require Logger
+
+        Logger.warn(
+          "updating env causes error, key: #{inspect(setting.key)}, error: #{inspect(e)}"
+        )
+    end
+  end
+end

lib/pleroma/conversation/participation.ex

@@ -59,10 +59,10 @@ defmodule Pleroma.Conversation.Participation do
   def for_user(user, params \\ %{}) do
     from(p in __MODULE__,
       where: p.user_id == ^user.id,
-      order_by: [desc: p.updated_at]
+      order_by: [desc: p.updated_at],
+      preload: [conversation: [:users]]
     )
     |> Pleroma.Pagination.fetch_paginated(params)
-    |> Repo.preload(conversation: [:users])
   end
 
   def for_user_with_last_activity_id(user, params \\ %{}) do

lib/pleroma/emails/mailer.ex

@@ -3,11 +3,58 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Emails.Mailer do
-  use Swoosh.Mailer, otp_app: :pleroma
+  @moduledoc """
+  Defines the Pleroma mailer.
 
+  The module contains functions to delivery email using Swoosh.Mailer.
+  """
+
+  alias Swoosh.DeliveryError
+
+  @otp_app :pleroma
+  @mailer_config [otp: :pleroma]
+
+  @spec enabled?() :: boolean()
+  def enabled?, do: Pleroma.Config.get([__MODULE__, :enabled])
+
+  @doc "add email to queue"
   def deliver_async(email, config \\ []) do
     PleromaJobQueue.enqueue(:mailer, __MODULE__, [:deliver_async, email, config])
   end
 
+  @doc "callback to perform send email from queue"
   def perform(:deliver_async, email, config), do: deliver(email, config)
+
+  @spec deliver(Swoosh.Email.t(), Keyword.t()) :: {:ok, term} | {:error, term}
+  def deliver(email, config \\ [])
+
+  def deliver(email, config) do
+    case enabled?() do
+      true -> Swoosh.Mailer.deliver(email, parse_config(config))
+      false -> {:error, :deliveries_disabled}
+    end
+  end
+
+  @spec deliver!(Swoosh.Email.t(), Keyword.t()) :: term | no_return
+  def deliver!(email, config \\ [])
+
+  def deliver!(email, config) do
+    case deliver(email, config) do
+      {:ok, result} -> result
+      {:error, reason} -> raise DeliveryError, reason: reason
+    end
+  end
+
+  @on_load :validate_dependency
+
+  @doc false
+  def validate_dependency do
+    parse_config([])
+    |> Keyword.get(:adapter)
+    |> Swoosh.Mailer.validate_dependency()
+  end
+
+  defp parse_config(config) do
+    Swoosh.Mailer.parse_config(@otp_app, __MODULE__, @mailer_config, config)
+  end
 end

lib/pleroma/emails/user_email.ex

@@ -23,13 +23,8 @@ defmodule Pleroma.Emails.UserEmail do
   defp recipient(email, name), do: {name, email}
   defp recipient(%Pleroma.User{} = user), do: recipient(user.email, user.name)
 
-  def password_reset_email(user, password_reset_token) when is_binary(password_reset_token) do
-    password_reset_url =
-      Router.Helpers.util_url(
-        Endpoint,
-        :show_password_reset,
-        password_reset_token
-      )
+  def password_reset_email(user, token) when is_binary(token) do
+    password_reset_url = Router.Helpers.reset_password_url(Endpoint, :reset, token)
 
     html_body = """
     <h3>Reset your password at #{instance_name()}</h3>

lib/pleroma/emoji.ex

@@ -22,7 +22,6 @@ defmodule Pleroma.Emoji do
 
   @ets __MODULE__.Ets
   @ets_options [:ordered_set, :protected, :named_table, {:read_concurrency, true}]
-  @groups Pleroma.Config.get([:emoji, :groups])
 
   @doc false
   def start_link do
@@ -87,6 +86,8 @@ defmodule Pleroma.Emoji do
         "emoji"
       )
 
+    emoji_groups = Pleroma.Config.get([:emoji, :groups])
+
     case File.ls(emoji_dir_path) do
       {:error, :enoent} ->
         # The custom emoji directory doesn't exist,
@@ -98,7 +99,9 @@ defmodule Pleroma.Emoji do
         Logger.error("Could not access the custom emoji directory #{emoji_dir_path}: #{e}")
 
       {:ok, results} ->
-        grouped = Enum.group_by(results, &File.dir?/1)
+        grouped =
+          Enum.group_by(results, fn file -> File.dir?(Path.join(emoji_dir_path, file)) end)
+
         packs = grouped[true] || []
         files = grouped[false] || []
 
@@ -116,7 +119,7 @@ defmodule Pleroma.Emoji do
         emojis =
           Enum.flat_map(
             packs,
-            fn pack -> load_pack(Path.join(emoji_dir_path, pack)) end
+            fn pack -> load_pack(Path.join(emoji_dir_path, pack), emoji_groups) end
           )
 
         true = :ets.insert(@ets, emojis)
@@ -127,9 +130,9 @@ defmodule Pleroma.Emoji do
     shortcode_globs = Pleroma.Config.get([:emoji, :shortcode_globs], [])
 
     emojis =
-      (load_from_file("config/emoji.txt") ++
-         load_from_file("config/custom_emoji.txt") ++
-         load_from_globs(shortcode_globs))
+      (load_from_file("config/emoji.txt", emoji_groups) ++
+         load_from_file("config/custom_emoji.txt", emoji_groups) ++
+         load_from_globs(shortcode_globs, emoji_groups))
       |> Enum.reject(fn value -> value == nil end)
 
     true = :ets.insert(@ets, emojis)
@@ -137,23 +140,25 @@ defmodule Pleroma.Emoji do
     :ok
   end
 
-  defp load_pack(pack_dir) do
+  defp load_pack(pack_dir, emoji_groups) do
     pack_name = Path.basename(pack_dir)
 
     emoji_txt = Path.join(pack_dir, "emoji.txt")
 
     if File.exists?(emoji_txt) do
-      load_from_file(emoji_txt)
+      load_from_file(emoji_txt, emoji_groups)
     else
+      extensions = Pleroma.Config.get([:emoji, :pack_extensions])
+
       Logger.info(
-        "No emoji.txt found for pack \"#{pack_name}\", assuming all .png files are emoji"
+        "No emoji.txt found for pack \"#{pack_name}\", assuming all #{Enum.join(extensions, ", ")} files are emoji"
       )
 
-      make_shortcode_to_file_map(pack_dir, [".png"])
+      make_shortcode_to_file_map(pack_dir, extensions)
       |> Enum.map(fn {shortcode, rel_file} ->
         filename = Path.join("/emoji/#{pack_name}", rel_file)
 
-        {shortcode, filename, [to_string(match_extra(@groups, filename))]}
+        {shortcode, filename, [to_string(match_extra(emoji_groups, filename))]}
       end)
     end
   end
@@ -182,21 +187,21 @@ defmodule Pleroma.Emoji do
     |> Enum.filter(fn f -> Path.extname(f) in exts end)
   end
 
-  defp load_from_file(file) do
+  defp load_from_file(file, emoji_groups) do
     if File.exists?(file) do
-      load_from_file_stream(File.stream!(file))
+      load_from_file_stream(File.stream!(file), emoji_groups)
     else
       []
     end
   end
 
-  defp load_from_file_stream(stream) do
+  defp load_from_file_stream(stream, emoji_groups) do
     stream
     |> Stream.map(&String.trim/1)
     |> Stream.map(fn line ->
       case String.split(line, ~r/,\s*/) do
         [name, file] ->
-          {name, file, [to_string(match_extra(@groups, file))]}
+          {name, file, [to_string(match_extra(emoji_groups, file))]}
 
         [name, file | tags] ->
           {name, file, tags}
@@ -208,7 +213,7 @@ defmodule Pleroma.Emoji do
     |> Enum.to_list()
   end
 
-  defp load_from_globs(globs) do
+  defp load_from_globs(globs, emoji_groups) do
     static_path = Path.join(:code.priv_dir(:pleroma), "static")
 
     paths =
@@ -219,7 +224,7 @@ defmodule Pleroma.Emoji do
       |> Enum.concat()
 
     Enum.map(paths, fn path ->
-      tag = match_extra(@groups, Path.join("/", Path.relative_to(path, static_path)))
+      tag = match_extra(emoji_groups, Path.join("/", Path.relative_to(path, static_path)))
       shortcode = Path.basename(path, Path.extname(path))
       external_path = Path.join("/", Path.relative_to(path, static_path))
       {shortcode, external_path, [to_string(tag)]}

lib/pleroma/helpers/uri_helper.ex

@@ -0,0 +1,27 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Helpers.UriHelper do
+  def append_uri_params(uri, appended_params) do
+    uri = URI.parse(uri)
+    appended_params = for {k, v} <- appended_params, into: %{}, do: {to_string(k), v}
+    existing_params = URI.query_decoder(uri.query || "") |> Enum.into(%{})
+    updated_params_keys = Enum.uniq(Map.keys(existing_params) ++ Map.keys(appended_params))
+
+    updated_params =
+      for k <- updated_params_keys, do: {k, appended_params[k] || existing_params[k]}
+
+    uri
+    |> Map.put(:query, URI.encode_query(updated_params))
+    |> URI.to_string()
+  end
+
+  def append_param_if_present(%{} = params, param_name, param_value) do
+    if param_value do
+      Map.put(params, param_name, param_value)
+    else
+      params
+    end
+  end
+end

lib/pleroma/html.ex

@@ -89,7 +89,7 @@ defmodule Pleroma.HTML do
     Cachex.fetch!(:scrubber_cache, key, fn _key ->
       result =
         content
-        |> Floki.filter_out("a.mention")
+        |> Floki.filter_out("a.mention,a.hashtag,a[rel~=\"tag\"]")
         |> Floki.attribute("a", "href")
         |> Enum.at(0)
 

lib/pleroma/instances.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Instances do
   @moduledoc "Instances context."
 
@@ -13,7 +17,7 @@ defmodule Pleroma.Instances do
 
   def reachability_datetime_threshold do
     federation_reachability_timeout_days =
-      Pleroma.Config.get(:instance)[:federation_reachability_timeout_days] || 0
+      Pleroma.Config.get([:instance, :federation_reachability_timeout_days], 0)
 
     if federation_reachability_timeout_days > 0 do
       NaiveDateTime.add(

lib/pleroma/instances/instance.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Instances.Instance do
   @moduledoc "Instance."
 

lib/pleroma/notification.ex

@@ -13,6 +13,8 @@ defmodule Pleroma.Notification do
   alias Pleroma.User
   alias Pleroma.Web.CommonAPI
   alias Pleroma.Web.CommonAPI.Utils
+  alias Pleroma.Web.Push
+  alias Pleroma.Web.Streamer
 
   import Ecto.Query
   import Ecto.Changeset
@@ -125,8 +127,7 @@ defmodule Pleroma.Notification do
     end
   end
 
-  def create_notifications(%Activity{data: %{"to" => _, "type" => type}} = activity)
-      when type in ["Create", "Like", "Announce", "Follow"] do
+  def create_notifications(%Activity{data: %{"to" => _, "type" => "Create"}} = activity) do
     object = Object.normalize(activity)
 
     unless object && object.data["type"] == "Answer" do
@@ -138,6 +139,13 @@ defmodule Pleroma.Notification do
     end
   end
 
+  def create_notifications(%Activity{data: %{"to" => _, "type" => type}} = activity)
+      when type in ["Like", "Announce", "Follow"] do
+    users = get_notified_from_activity(activity)
+    notifications = Enum.map(users, fn user -> create_notification(activity, user) end)
+    {:ok, notifications}
+  end
+
   def create_notifications(_), do: {:ok, []}
 
   # TODO move to sql, too.
@@ -145,8 +153,9 @@ defmodule Pleroma.Notification do
     unless skip?(activity, user) do
       notification = %Notification{user_id: user.id, activity: activity}
       {:ok, notification} = Repo.insert(notification)
-      Pleroma.Web.Streamer.stream("user", notification)
-      Pleroma.Web.Push.send(notification)
+      Streamer.stream("user", notification)
+      Streamer.stream("user:notification", notification)
+      Push.send(notification)
       notification
     end
   end

lib/pleroma/object.ex

@@ -44,44 +44,46 @@ defmodule Pleroma.Object do
     Repo.one(from(object in Object, where: fragment("(?)->>'id' = ?", object.data, ^ap_id)))
   end
 
-  def normalize(_, fetch_remote \\ true)
+  defp warn_on_no_object_preloaded(ap_id) do
+    "Object.normalize() called without preloaded object (#{ap_id}). Consider preloading the object"
+    |> Logger.debug()
+
+    Logger.debug("Backtrace: #{inspect(Process.info(:erlang.self(), :current_stacktrace))}")
+  end
+
+  def normalize(_, fetch_remote \\ true, options \\ [])
+
   # If we pass an Activity to Object.normalize(), we can try to use the preloaded object.
   # Use this whenever possible, especially when walking graphs in an O(N) loop!
-  def normalize(%Object{} = object, _), do: object
-  def normalize(%Activity{object: %Object{} = object}, _), do: object
+  def normalize(%Object{} = object, _, _), do: object
+  def normalize(%Activity{object: %Object{} = object}, _, _), do: object
 
   # A hack for fake activities
-  def normalize(%Activity{data: %{"object" => %{"fake" => true} = data}}, _) do
+  def normalize(%Activity{data: %{"object" => %{"fake" => true} = data}}, _, _) do
     %Object{id: "pleroma:fake_object_id", data: data}
   end
 
-  # Catch and log Object.normalize() calls where the Activity's child object is not
-  # preloaded.
-  def normalize(%Activity{data: %{"object" => %{"id" => ap_id}}}, fetch_remote) do
-    Logger.debug(
-      "Object.normalize() called without preloaded object (#{ap_id}).  Consider preloading the object!"
-    )
-
-    Logger.debug("Backtrace: #{inspect(Process.info(:erlang.self(), :current_stacktrace))}")
-
+  # No preloaded object
+  def normalize(%Activity{data: %{"object" => %{"id" => ap_id}}}, fetch_remote, _) do
+    warn_on_no_object_preloaded(ap_id)
     normalize(ap_id, fetch_remote)
   end
 
-  def normalize(%Activity{data: %{"object" => ap_id}}, fetch_remote) do
-    Logger.debug(
-      "Object.normalize() called without preloaded object (#{ap_id}).  Consider preloading the object!"
-    )
-
-    Logger.debug("Backtrace: #{inspect(Process.info(:erlang.self(), :current_stacktrace))}")
-
+  # No preloaded object
+  def normalize(%Activity{data: %{"object" => ap_id}}, fetch_remote, _) do
+    warn_on_no_object_preloaded(ap_id)
     normalize(ap_id, fetch_remote)
   end
 
   # Old way, try fetching the object through cache.
-  def normalize(%{"id" => ap_id}, fetch_remote), do: normalize(ap_id, fetch_remote)
-  def normalize(ap_id, false) when is_binary(ap_id), do: get_cached_by_ap_id(ap_id)
-  def normalize(ap_id, true) when is_binary(ap_id), do: Fetcher.fetch_object_from_id!(ap_id)
-  def normalize(_, _), do: nil
+  def normalize(%{"id" => ap_id}, fetch_remote, _), do: normalize(ap_id, fetch_remote)
+  def normalize(ap_id, false, _) when is_binary(ap_id), do: get_cached_by_ap_id(ap_id)
+
+  def normalize(ap_id, true, options) when is_binary(ap_id) do
+    Fetcher.fetch_object_from_id!(ap_id, options)
+  end
+
+  def normalize(_, _, _), do: nil
 
   # Owned objects can only be mutated by their owner
   def authorize_mutation(%Object{data: %{"actor" => actor}}, %User{ap_id: ap_id}),

lib/pleroma/object/containment.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Object.Containment do
   @moduledoc """
   This module contains some useful functions for containing objects to specific

lib/pleroma/object/fetcher.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Object.Fetcher do
   alias Pleroma.HTTP
   alias Pleroma.Object
@@ -22,7 +26,7 @@ defmodule Pleroma.Object.Fetcher do
 
   # TODO:
   # This will create a Create activity, which we need internally at the moment.
-  def fetch_object_from_id(id) do
+  def fetch_object_from_id(id, options \\ []) do
     if object = Object.get_cached_by_ap_id(id) do
       {:ok, object}
     else
@@ -38,7 +42,7 @@ defmodule Pleroma.Object.Fetcher do
              "object" => data
            },
            :ok <- Containment.contain_origin(id, params),
-           {:ok, activity} <- Transmogrifier.handle_incoming(params),
+           {:ok, activity} <- Transmogrifier.handle_incoming(params, options),
            {:object, _data, %Object{} = object} <-
              {:object, data, Object.normalize(activity, false)} do
         {:ok, object}
@@ -63,8 +67,8 @@ defmodule Pleroma.Object.Fetcher do
     end
   end
 
-  def fetch_object_from_id!(id) do
-    with {:ok, object} <- fetch_object_from_id(id) do
+  def fetch_object_from_id!(id, options \\ []) do
+    with {:ok, object} <- fetch_object_from_id(id, options) do
       object
     else
       _e ->
@@ -85,6 +89,9 @@ defmodule Pleroma.Object.Fetcher do
          :ok <- Containment.contain_origin_from_id(id, data) do
       {:ok, data}
     else
+      {:ok, %{status: code}} when code in [404, 410] ->
+        {:error, "Object has been deleted"}
+
       e ->
         {:error, e}
     end

lib/pleroma/object_tombstone.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.ObjectTombstone do
   @enforce_keys [:id, :formerType, :deleted]
   defstruct [:id, :formerType, :deleted, type: "Tombstone"]

lib/pleroma/pagination.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Pagination do
   @moduledoc """
   Implements Mastodon-compatible pagination.

lib/pleroma/password_reset_token.ex

@@ -37,6 +37,7 @@ defmodule Pleroma.PasswordResetToken do
     |> put_change(:used, true)
   end
 
+  @spec reset_password(binary(), map()) :: {:ok, User.t()} | {:error, binary()}
   def reset_password(token, data) do
     with %{used: false} = token <- Repo.get_by(PasswordResetToken, %{token: token}),
          %User{} = user <- User.get_cached_by_id(token.user_id),

lib/pleroma/plugs/ensure_authenticated_plug.ex

@@ -4,6 +4,7 @@
 
 defmodule Pleroma.Plugs.EnsureAuthenticatedPlug do
   import Plug.Conn
+  import Pleroma.Web.TranslationHelpers
   alias Pleroma.User
 
   def init(options) do
@@ -16,8 +17,7 @@ defmodule Pleroma.Plugs.EnsureAuthenticatedPlug do
 
   def call(conn, _) do
     conn
-    |> put_resp_content_type("application/json")
-    |> send_resp(403, Jason.encode!(%{error: "Invalid credentials."}))
+    |> render_error(:forbidden, "Invalid credentials.")
     |> halt
   end
 end

lib/pleroma/plugs/ensure_public_or_authenticated_plug.ex

@@ -3,6 +3,7 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Plugs.EnsurePublicOrAuthenticatedPlug do
+  import Pleroma.Web.TranslationHelpers
   import Plug.Conn
   alias Pleroma.Config
   alias Pleroma.User
@@ -23,8 +24,7 @@ defmodule Pleroma.Plugs.EnsurePublicOrAuthenticatedPlug do
 
       {false, _} ->
         conn
-        |> put_resp_content_type("application/json")
-        |> send_resp(403, Jason.encode!(%{error: "This resource requires authentication."}))
+        |> render_error(:forbidden, "This resource requires authentication.")
         |> halt
     end
   end

lib/pleroma/plugs/http_security_plug.ex

@@ -56,14 +56,14 @@ defmodule Pleroma.Plugs.HTTPSecurityPlug do
     connect_src = "connect-src 'self' #{static_url} #{websocket_url}"
 
     connect_src =
-      if Mix.env() == :dev do
+      if Pleroma.Config.get(:env) == :dev do
         connect_src <> " http://localhost:3035/"
       else
         connect_src
       end
 
     script_src =
-      if Mix.env() == :dev do
+      if Pleroma.Config.get(:env) == :dev do
         "script-src 'self' 'unsafe-eval'"
       else
         "script-src 'self'"

lib/pleroma/plugs/idempotency_plug.ex

@@ -0,0 +1,84 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Plugs.IdempotencyPlug do
+  import Phoenix.Controller, only: [json: 2]
+  import Plug.Conn
+
+  @behaviour Plug
+
+  @impl true
+  def init(opts), do: opts
+
+  # Sending idempotency keys in `GET` and `DELETE` requests has no effect
+  # and should be avoided, as these requests are idempotent by definition.
+
+  @impl true
+  def call(%{method: method} = conn, _) when method in ["POST", "PUT", "PATCH"] do
+    case get_req_header(conn, "idempotency-key") do
+      [key] -> process_request(conn, key)
+      _ -> conn
+    end
+  end
+
+  def call(conn, _), do: conn
+
+  def process_request(conn, key) do
+    case Cachex.get(:idempotency_cache, key) do
+      {:ok, nil} ->
+        cache_resposnse(conn, key)
+
+      {:ok, record} ->
+        send_cached(conn, key, record)
+
+      {atom, message} when atom in [:ignore, :error] ->
+        render_error(conn, message)
+    end
+  end
+
+  defp cache_resposnse(conn, key) do
+    register_before_send(conn, fn conn ->
+      [request_id] = get_resp_header(conn, "x-request-id")
+      content_type = get_content_type(conn)
+
+      record = {request_id, content_type, conn.status, conn.resp_body}
+      {:ok, _} = Cachex.put(:idempotency_cache, key, record)
+
+      conn
+      |> put_resp_header("idempotency-key", key)
+      |> put_resp_header("x-original-request-id", request_id)
+    end)
+  end
+
+  defp send_cached(conn, key, record) do
+    {request_id, content_type, status, body} = record
+
+    conn
+    |> put_resp_header("idempotency-key", key)
+    |> put_resp_header("idempotent-replayed", "true")
+    |> put_resp_header("x-original-request-id", request_id)
+    |> put_resp_content_type(content_type)
+    |> send_resp(status, body)
+    |> halt()
+  end
+
+  defp render_error(conn, message) do
+    conn
+    |> put_status(:unprocessable_entity)
+    |> json(%{error: message})
+    |> halt()
+  end
+
+  defp get_content_type(conn) do
+    [content_type] = get_resp_header(conn, "content-type")
+
+    if String.contains?(content_type, ";") do
+      content_type
+      |> String.split(";")
+      |> hd()
+    else
+      content_type
+    end
+  end
+end

lib/pleroma/plugs/oauth_scopes_plug.ex

@@ -4,6 +4,7 @@
 
 defmodule Pleroma.Plugs.OAuthScopesPlug do
   import Plug.Conn
+  import Pleroma.Web.Gettext
 
   @behaviour Plug
 
@@ -30,11 +31,14 @@ defmodule Pleroma.Plugs.OAuthScopesPlug do
 
       true ->
         missing_scopes = scopes -- token.scopes
-        error_message = "Insufficient permissions: #{Enum.join(missing_scopes, " #{op} ")}."
+        permissions = Enum.join(missing_scopes, " #{op} ")
+
+        error_message =
+          dgettext("errors", "Insufficient permissions: %{permissions}.", permissions: permissions)
 
         conn
         |> put_resp_content_type("application/json")
-        |> send_resp(403, Jason.encode!(%{error: error_message}))
+        |> send_resp(:forbidden, Jason.encode!(%{error: error_message}))
         |> halt()
     end
   end

lib/pleroma/plugs/rate_limit_plug.ex

@@ -1,36 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Plugs.RateLimitPlug do
-  import Phoenix.Controller, only: [json: 2]
-  import Plug.Conn
-
-  def init(opts), do: opts
-
-  def call(conn, opts) do
-    enabled? = Pleroma.Config.get([:app_account_creation, :enabled])
-
-    case check_rate(conn, Map.put(opts, :enabled, enabled?)) do
-      {:ok, _count} -> conn
-      {:error, _count} -> render_error(conn)
-      %Plug.Conn{} = conn -> conn
-    end
-  end
-
-  defp check_rate(conn, %{enabled: true} = opts) do
-    max_requests = opts[:max_requests]
-    bucket_name = conn.remote_ip |> Tuple.to_list() |> Enum.join(".")
-
-    ExRated.check_rate(bucket_name, opts[:interval] * 1000, max_requests)
-  end
-
-  defp check_rate(conn, _), do: conn
-
-  defp render_error(conn) do
-    conn
-    |> put_status(:forbidden)
-    |> json(%{error: "Rate limit exceeded."})
-    |> halt()
-  end
-end

lib/pleroma/plugs/rate_limiter.ex

@@ -0,0 +1,92 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Plugs.RateLimiter do
+  @moduledoc """
+
+  ## Configuration
+
+  A keyword list of rate limiters where a key is a limiter name and value is the limiter configuration. The basic configuration is a tuple where:
+
+  * The first element: `scale` (Integer). The time scale in milliseconds.
+  * The second element: `limit` (Integer). How many requests to limit in the time scale provided.
+
+  It is also possible to have different limits for unauthenticated and authenticated users: the keyword value must be a list of two tuples where the first one is a config for unauthenticated users and the second one is for authenticated.
+
+  To disable a limiter set its value to `nil`.
+
+  ### Example
+
+      config :pleroma, :rate_limit,
+        one: {1000, 10},
+        two: [{10_000, 10}, {10_000, 50}],
+        foobar: nil
+
+  Here we have three limiters:
+
+  * `one` which is not over 10req/1s
+  * `two` which has two limits: 10req/10s for unauthenticated users and 50req/10s for authenticated users
+  * `foobar` which is disabled
+
+  ## Usage
+
+  Inside a controller:
+
+      plug(Pleroma.Plugs.RateLimiter, :one when action == :one)
+      plug(Pleroma.Plugs.RateLimiter, :two when action in [:two, :three])
+
+  or inside a router pipiline:
+
+      pipeline :api do
+        ...
+        plug(Pleroma.Plugs.RateLimiter, :one)
+        ...
+      end
+  """
+  import Pleroma.Web.TranslationHelpers
+  import Plug.Conn
+
+  alias Pleroma.User
+
+  def init(limiter_name) do
+    case Pleroma.Config.get([:rate_limit, limiter_name]) do
+      nil -> nil
+      config -> {limiter_name, config}
+    end
+  end
+
+  # do not limit if there is no limiter configuration
+  def call(conn, nil), do: conn
+
+  def call(conn, opts) do
+    case check_rate(conn, opts) do
+      {:ok, _count} -> conn
+      {:error, _count} -> render_throttled_error(conn)
+    end
+  end
+
+  defp check_rate(%{assigns: %{user: %User{id: user_id}}}, {limiter_name, [_, {scale, limit}]}) do
+    ExRated.check_rate("#{limiter_name}:#{user_id}", scale, limit)
+  end
+
+  defp check_rate(conn, {limiter_name, [{scale, limit} | _]}) do
+    ExRated.check_rate("#{limiter_name}:#{ip(conn)}", scale, limit)
+  end
+
+  defp check_rate(conn, {limiter_name, {scale, limit}}) do
+    check_rate(conn, {limiter_name, [{scale, limit}]})
+  end
+
+  def ip(%{remote_ip: remote_ip}) do
+    remote_ip
+    |> Tuple.to_list()
+    |> Enum.join(".")
+  end
+
+  defp render_throttled_error(conn) do
+    conn
+    |> render_error(:too_many_requests, "Throttled")
+    |> halt()
+  end
+end

lib/pleroma/plugs/set_locale_plug.ex

@@ -0,0 +1,63 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+# NOTE: this module is based on https://github.com/smeevil/set_locale
+defmodule Pleroma.Plugs.SetLocalePlug do
+  import Plug.Conn, only: [get_req_header: 2, assign: 3]
+
+  def init(_), do: nil
+
+  def call(conn, _) do
+    locale = get_locale_from_header(conn) || Gettext.get_locale()
+    Gettext.put_locale(locale)
+    assign(conn, :locale, locale)
+  end
+
+  defp get_locale_from_header(conn) do
+    conn
+    |> extract_accept_language()
+    |> Enum.find(&supported_locale?/1)
+  end
+
+  defp extract_accept_language(conn) do
+    case get_req_header(conn, "accept-language") do
+      [value | _] ->
+        value
+        |> String.split(",")
+        |> Enum.map(&parse_language_option/1)
+        |> Enum.sort(&(&1.quality > &2.quality))
+        |> Enum.map(& &1.tag)
+        |> Enum.reject(&is_nil/1)
+        |> ensure_language_fallbacks()
+
+      _ ->
+        []
+    end
+  end
+
+  defp supported_locale?(locale) do
+    Pleroma.Web.Gettext
+    |> Gettext.known_locales()
+    |> Enum.member?(locale)
+  end
+
+  defp parse_language_option(string) do
+    captures = Regex.named_captures(~r/^\s?(?<tag>[\w\-]+)(?:;q=(?<quality>[\d\.]+))?$/i, string)
+
+    quality =
+      case Float.parse(captures["quality"] || "1.0") do
+        {val, _} -> val
+        :error -> 1.0
+      end
+
+    %{tag: captures["tag"], quality: quality}
+  end
+
+  defp ensure_language_fallbacks(tags) do
+    Enum.flat_map(tags, fn tag ->
+      [language | _] = String.split(tag, "-")
+      if Enum.member?(tags, language), do: [tag], else: [tag, language]
+    end)
+  end
+end

lib/pleroma/plugs/uploaded_media.ex

@@ -7,6 +7,7 @@ defmodule Pleroma.Plugs.UploadedMedia do
   """
 
   import Plug.Conn
+  import Pleroma.Web.Gettext
   require Logger
 
   @behaviour Plug
@@ -36,7 +37,7 @@ defmodule Pleroma.Plugs.UploadedMedia do
           conn
       end
 
-    config = Pleroma.Config.get([Pleroma.Upload])
+    config = Pleroma.Config.get(Pleroma.Upload)
 
     with uploader <- Keyword.fetch!(config, :uploader),
          proxy_remote = Keyword.get(config, :proxy_remote, false),
@@ -45,7 +46,7 @@ defmodule Pleroma.Plugs.UploadedMedia do
     else
       _ ->
         conn
-        |> send_resp(500, "Failed")
+        |> send_resp(:internal_server_error, dgettext("errors", "Failed"))
         |> halt()
     end
   end
@@ -64,7 +65,7 @@ defmodule Pleroma.Plugs.UploadedMedia do
       conn
     else
       conn
-      |> send_resp(404, "Not found")
+      |> send_resp(:not_found, dgettext("errors", "Not found"))
       |> halt()
     end
   end
@@ -84,7 +85,7 @@ defmodule Pleroma.Plugs.UploadedMedia do
     Logger.error("#{__MODULE__}: Unknown get startegy: #{inspect(unknown)}")
 
     conn
-    |> send_resp(500, "Internal Error")
+    |> send_resp(:internal_server_error, dgettext("errors", "Internal Error"))
     |> halt()
   end
 end

lib/pleroma/plugs/user_is_admin_plug.ex

@@ -3,6 +3,7 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Plugs.UserIsAdminPlug do
+  import Pleroma.Web.TranslationHelpers
   import Plug.Conn
   alias Pleroma.User
 
@@ -16,8 +17,7 @@ defmodule Pleroma.Plugs.UserIsAdminPlug do
 
   def call(conn, _) do
     conn
-    |> put_resp_content_type("application/json")
-    |> send_resp(403, Jason.encode!(%{error: "User is not admin."}))
+    |> render_error(:forbidden, "User is not admin.")
     |> halt
   end
 end

lib/pleroma/release_tasks.ex

@@ -0,0 +1,66 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.ReleaseTasks do
+  @repo Pleroma.Repo
+
+  def run(args) do
+    [task | args] = String.split(args)
+
+    case task do
+      "migrate" -> migrate(args)
+      "create" -> create()
+      "rollback" -> rollback(args)
+      task -> mix_task(task, args)
+    end
+  end
+
+  defp mix_task(task, args) do
+    Application.load(:pleroma)
+    {:ok, modules} = :application.get_key(:pleroma, :modules)
+
+    module =
+      Enum.find(modules, fn module ->
+        module = Module.split(module)
+
+        match?(["Mix", "Tasks", "Pleroma" | _], module) and
+          String.downcase(List.last(module)) == task
+      end)
+
+    if module do
+      module.run(args)
+    else
+      IO.puts("The task #{task} does not exist")
+    end
+  end
+
+  def migrate(args) do
+    Mix.Tasks.Pleroma.Ecto.Migrate.run(args)
+  end
+
+  def rollback(args) do
+    Mix.Tasks.Pleroma.Ecto.Rollback.run(args)
+  end
+
+  def create do
+    Application.load(:pleroma)
+
+    case @repo.__adapter__.storage_up(@repo.config) do
+      :ok ->
+        IO.puts("The database for #{inspect(@repo)} has been created")
+
+      {:error, :already_up} ->
+        IO.puts("The database for #{inspect(@repo)} has already been created")
+
+      {:error, term} when is_binary(term) ->
+        IO.puts(:stderr, "The database for #{inspect(@repo)} couldn't be created: #{term}")
+
+      {:error, term} ->
+        IO.puts(
+          :stderr,
+          "The database for #{inspect(@repo)} couldn't be created: #{inspect(term)}"
+        )
+    end
+  end
+end

lib/pleroma/repo_streamer.ex

@@ -0,0 +1,34 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.RepoStreamer do
+  alias Pleroma.Repo
+  import Ecto.Query
+
+  def chunk_stream(query, chunk_size) do
+    Stream.unfold(0, fn
+      :halt ->
+        {[], :halt}
+
+      last_id ->
+        query
+        |> order_by(asc: :id)
+        |> where([r], r.id > ^last_id)
+        |> limit(^chunk_size)
+        |> Repo.all()
+        |> case do
+          [] ->
+            {[], :halt}
+
+          records ->
+            last_id = List.last(records).id
+            {records, last_id}
+        end
+    end)
+    |> Stream.take_while(fn
+      [] -> false
+      _ -> true
+    end)
+  end
+end

lib/pleroma/reverse_proxy/client.ex

@@ -0,0 +1,28 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.ReverseProxy.Client do
+  @callback request(atom(), String.t(), [tuple()], String.t(), list()) ::
+              {:ok, pos_integer(), [tuple()], reference() | map()}
+              | {:ok, pos_integer(), [tuple()]}
+              | {:ok, reference()}
+              | {:error, term()}
+
+  @callback stream_body(reference() | pid() | map()) ::
+              {:ok, binary()} | :done | {:error, String.t()}
+
+  @callback close(reference() | pid() | map()) :: :ok
+
+  def request(method, url, headers, "", opts \\ []) do
+    client().request(method, url, headers, "", opts)
+  end
+
+  def stream_body(ref), do: client().stream_body(ref)
+
+  def close(ref), do: client().close(ref)
+
+  defp client do
+    Pleroma.Config.get([Pleroma.ReverseProxy.Client], :hackney)
+  end
+end

lib/pleroma/reverse_proxy/reverse_proxy.ex

@@ -146,7 +146,7 @@ defmodule Pleroma.ReverseProxy do
     Logger.debug("#{__MODULE__} #{method} #{url} #{inspect(headers)}")
     method = method |> String.downcase() |> String.to_existing_atom()
 
-    case :hackney.request(method, url, headers, "", hackney_opts) do
+    case client().request(method, url, headers, "", hackney_opts) do
       {:ok, code, headers, client} when code in @valid_resp_codes ->
         {:ok, code, downcase_headers(headers), client}
 
@@ -173,7 +173,7 @@ defmodule Pleroma.ReverseProxy do
         halt(conn)
 
       {:error, :closed, conn} ->
-        :hackney.close(client)
+        client().close(client)
         halt(conn)
 
       {:error, error, conn} ->
@@ -181,7 +181,7 @@ defmodule Pleroma.ReverseProxy do
           "#{__MODULE__} request to #{url} failed while reading/chunking: #{inspect(error)}"
         )
 
-        :hackney.close(client)
+        client().close(client)
         halt(conn)
     end
   end
@@ -196,7 +196,7 @@ defmodule Pleroma.ReverseProxy do
              duration,
              Keyword.get(opts, :max_read_duration, @max_read_duration)
            ),
-         {:ok, data} <- :hackney.stream_body(client),
+         {:ok, data} <- client().stream_body(client),
          {:ok, duration} <- increase_read_duration(duration),
          sent_so_far = sent_so_far + byte_size(data),
          :ok <- body_size_constraint(sent_so_far, Keyword.get(opts, :max_body_size)),
@@ -377,4 +377,6 @@ defmodule Pleroma.ReverseProxy do
   defp increase_read_duration(_) do
     {:ok, :no_duration_limit, :no_duration_limit}
   end
+
+  defp client, do: Pleroma.ReverseProxy.Client
 end

lib/pleroma/upload/filter/anonymize_filename.ex

@@ -10,10 +10,19 @@ defmodule Pleroma.Upload.Filter.AnonymizeFilename do
   """
   @behaviour Pleroma.Upload.Filter
 
-  def filter(upload) do
-    extension = List.last(String.split(upload.name, "."))
-    name = Pleroma.Config.get([__MODULE__, :text], random(extension))
-    {:ok, %Pleroma.Upload{upload | name: name}}
+  alias Pleroma.Config
+  alias Pleroma.Upload
+
+  def filter(%Upload{name: name} = upload) do
+    extension = List.last(String.split(name, "."))
+    name = predefined_name(extension) || random(extension)
+    {:ok, %Upload{upload | name: name}}
+  end
+
+  @spec predefined_name(String.t()) :: String.t() | nil
+  defp predefined_name(extension) do
+    with name when not is_nil(name) <- Config.get([__MODULE__, :text]),
+         do: String.replace(name, "{extension}", extension)
   end
 
   defp random(extension) do

lib/pleroma/uploaders/swift/keystone.ex

@@ -1,51 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Uploaders.Swift.Keystone do
-  use HTTPoison.Base
-
-  def process_url(url) do
-    Enum.join(
-      [Pleroma.Config.get!([Pleroma.Uploaders.Swift, :auth_url]), url],
-      "/"
-    )
-  end
-
-  def process_response_body(body) do
-    body
-    |> Jason.decode!()
-  end
-
-  def get_token do
-    settings = Pleroma.Config.get(Pleroma.Uploaders.Swift)
-    username = Keyword.fetch!(settings, :username)
-    password = Keyword.fetch!(settings, :password)
-    tenant_id = Keyword.fetch!(settings, :tenant_id)
-
-    case post(
-           "/tokens",
-           make_auth_body(username, password, tenant_id),
-           ["Content-Type": "application/json"],
-           hackney: [:insecure]
-         ) do
-      {:ok, %Tesla.Env{status: 200, body: body}} ->
-        body["access"]["token"]["id"]
-
-      {:ok, %Tesla.Env{status: _}} ->
-        ""
-    end
-  end
-
-  def make_auth_body(username, password, tenant) do
-    Jason.encode!(%{
-      :auth => %{
-        :passwordCredentials => %{
-          :username => username,
-          :password => password
-        },
-        :tenantId => tenant
-      }
-    })
-  end
-end

lib/pleroma/uploaders/swift/swift.ex

@@ -1,29 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Uploaders.Swift.Client do
-  use HTTPoison.Base
-
-  def process_url(url) do
-    Enum.join(
-      [Pleroma.Config.get!([Pleroma.Uploaders.Swift, :storage_url]), url],
-      "/"
-    )
-  end
-
-  def upload_file(filename, body, content_type) do
-    token = Pleroma.Uploaders.Swift.Keystone.get_token()
-
-    case put("#{filename}", body, "X-Auth-Token": token, "Content-Type": content_type) do
-      {:ok, %Tesla.Env{status: 201}} ->
-        {:ok, {:file, filename}}
-
-      {:ok, %Tesla.Env{status: 401}} ->
-        {:error, "Unauthorized, Bad Token"}
-
-      {:error, _} ->
-        {:error, "Swift Upload Error"}
-    end
-  end
-end

lib/pleroma/uploaders/swift/uploader.ex

@@ -1,19 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Uploaders.Swift do
-  @behaviour Pleroma.Uploaders.Uploader
-
-  def get_file(name) do
-    {:ok, {:url, Path.join([Pleroma.Config.get!([__MODULE__, :object_url]), name])}}
-  end
-
-  def put_file(upload) do
-    Pleroma.Uploaders.Swift.Client.upload_file(
-      upload.path,
-      File.read!(upload.tmpfile),
-      upload.content_type
-    )
-  end
-end

lib/pleroma/uploaders/uploader.ex

@@ -3,6 +3,8 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Uploaders.Uploader do
+  import Pleroma.Web.Gettext
+
   @moduledoc """
   Defines the contract to put and get an uploaded file to any backend.
   """
@@ -66,7 +68,7 @@ defmodule Pleroma.Uploaders.Uploader do
             {:error, error}
         end
     after
-      30_000 -> {:error, "Uploader callback timeout"}
+      30_000 -> {:error, dgettext("errors", "Uploader callback timeout")}
     end
   end
 end

lib/pleroma/user.ex

@@ -9,12 +9,14 @@ defmodule Pleroma.User do
   import Ecto.Query
 
   alias Comeonin.Pbkdf2
+  alias Ecto.Multi
   alias Pleroma.Activity
   alias Pleroma.Keys
   alias Pleroma.Notification
   alias Pleroma.Object
   alias Pleroma.Registration
   alias Pleroma.Repo
+  alias Pleroma.RepoStreamer
   alias Pleroma.User
   alias Pleroma.Web
   alias Pleroma.Web.ActivityPub.ActivityPub
@@ -105,15 +107,25 @@ defmodule Pleroma.User do
   def ap_followers(%User{follower_address: fa}) when is_binary(fa), do: fa
   def ap_followers(%User{} = user), do: "#{ap_id(user)}/followers"
 
-  def user_info(%User{} = user) do
+  def user_info(%User{} = user, args \\ %{}) do
+    following_count =
+      if args[:following_count], do: args[:following_count], else: following_count(user)
+
+    follower_count =
+      if args[:follower_count], do: args[:follower_count], else: user.info.follower_count
+
     %{
-      following_count: following_count(user),
       note_count: user.info.note_count,
-      follower_count: user.info.follower_count,
       locked: user.info.locked,
       confirmation_pending: user.info.confirmation_pending,
       default_scope: user.info.default_scope
     }
+    |> Map.put(:following_count, following_count)
+    |> Map.put(:follower_count, follower_count)
+  end
+
+  def set_info_cache(user, args) do
+    Cachex.put(:user_cache, "user_info:#{user.id}", user_info(user, args))
   end
 
   def restrict_deactivated(query) do
@@ -193,27 +205,24 @@ defmodule Pleroma.User do
   end
 
   def password_update_changeset(struct, params) do
-    changeset =
-      struct
-      |> cast(params, [:password, :password_confirmation])
-      |> validate_required([:password, :password_confirmation])
-      |> validate_confirmation(:password)
-
-    OAuth.Token.delete_user_tokens(struct)
-    OAuth.Authorization.delete_user_authorizations(struct)
-
-    if changeset.valid? do
-      hashed = Pbkdf2.hashpwsalt(changeset.changes[:password])
-
-      changeset
-      |> put_change(:password_hash, hashed)
-    else
-      changeset
-    end
+    struct
+    |> cast(params, [:password, :password_confirmation])
+    |> validate_required([:password, :password_confirmation])
+    |> validate_confirmation(:password)
+    |> put_password_hash
   end
 
-  def reset_password(user, data) do
-    update_and_set_cache(password_update_changeset(user, data))
+  def reset_password(%User{id: user_id} = user, data) do
+    multi =
+      Multi.new()
+      |> Multi.update(:user, password_update_changeset(user, data))
+      |> Multi.delete_all(:tokens, OAuth.Token.Query.get_by_user(user_id))
+      |> Multi.delete_all(:auth, OAuth.Authorization.delete_by_user_query(user))
+
+    case Repo.transaction(multi) do
+      {:ok, %{user: user} = _} -> set_cache(user)
+      {:error, _, changeset, _} -> {:error, changeset}
+    end
   end
 
   def register_changeset(struct, params \\ %{}, opts \\ []) do
@@ -249,12 +258,11 @@ defmodule Pleroma.User do
       end
 
     if changeset.valid? do
-      hashed = Pbkdf2.hashpwsalt(changeset.changes[:password])
       ap_id = User.ap_id(%User{nickname: changeset.changes[:nickname]})
       followers = User.ap_followers(%User{nickname: changeset.changes[:nickname]})
 
       changeset
-      |> put_change(:password_hash, hashed)
+      |> put_password_hash
       |> put_change(:ap_id, ap_id)
       |> unique_constraint(:ap_id)
       |> put_change(:following, [followers])
@@ -324,14 +332,6 @@ defmodule Pleroma.User do
     end
   end
 
-  def maybe_follow(%User{} = follower, %User{info: _info} = followed) do
-    if not following?(follower, followed) do
-      follow(follower, followed)
-    else
-      {:ok, follower}
-    end
-  end
-
   @doc "A mass follow for local users. Respects blocks in both directions but does not create activities."
   @spec follow_all(User.t(), list(User.t())) :: {atom(), User.t()}
   def follow_all(follower, followeds) do
@@ -370,8 +370,8 @@ defmodule Pleroma.User do
     ap_followers = followed.follower_address
 
     cond do
-      following?(follower, followed) or info.deactivated ->
-        {:error, "Could not follow user: #{followed.nickname} is already on your list."}
+      info.deactivated ->
+        {:error, "Could not follow user: You are deactivated."}
 
       deny_follow_blocked and blocks?(followed, follower) ->
         {:error, "Could not follow user: #{followed.nickname} blocked you."}
@@ -735,122 +735,6 @@ defmodule Pleroma.User do
     |> Repo.all()
   end
 
-  def search(query, resolve \\ false, for_user \\ nil) do
-    # Strip the beginning @ off if there is a query
-    query = String.trim_leading(query, "@")
-
-    if resolve, do: get_or_fetch(query)
-
-    {:ok, results} =
-      Repo.transaction(fn ->
-        Ecto.Adapters.SQL.query(Repo, "select set_limit(0.25)", [])
-        Repo.all(search_query(query, for_user))
-      end)
-
-    results
-  end
-
-  def search_query(query, for_user) do
-    fts_subquery = fts_search_subquery(query)
-    trigram_subquery = trigram_search_subquery(query)
-    union_query = from(s in trigram_subquery, union_all: ^fts_subquery)
-    distinct_query = from(s in subquery(union_query), order_by: s.search_type, distinct: s.id)
-
-    from(s in subquery(boost_search_rank_query(distinct_query, for_user)),
-      order_by: [desc: s.search_rank],
-      limit: 40
-    )
-  end
-
-  defp boost_search_rank_query(query, nil), do: query
-
-  defp boost_search_rank_query(query, for_user) do
-    friends_ids = get_friends_ids(for_user)
-    followers_ids = get_followers_ids(for_user)
-
-    from(u in subquery(query),
-      select_merge: %{
-        search_rank:
-          fragment(
-            """
-             CASE WHEN (?) THEN (?) * 1.3
-             WHEN (?) THEN (?) * 1.2
-             WHEN (?) THEN (?) * 1.1
-             ELSE (?) END
-            """,
-            u.id in ^friends_ids and u.id in ^followers_ids,
-            u.search_rank,
-            u.id in ^friends_ids,
-            u.search_rank,
-            u.id in ^followers_ids,
-            u.search_rank,
-            u.search_rank
-          )
-      }
-    )
-  end
-
-  defp fts_search_subquery(term, query \\ User) do
-    processed_query =
-      term
-      |> String.replace(~r/\W+/, " ")
-      |> String.trim()
-      |> String.split()
-      |> Enum.map(&(&1 <> ":*"))
-      |> Enum.join(" | ")
-
-    from(
-      u in query,
-      select_merge: %{
-        search_type: ^0,
-        search_rank:
-          fragment(
-            """
-            ts_rank_cd(
-              setweight(to_tsvector('simple', regexp_replace(?, '\\W', ' ', 'g')), 'A') ||
-              setweight(to_tsvector('simple', regexp_replace(coalesce(?, ''), '\\W', ' ', 'g')), 'B'),
-              to_tsquery('simple', ?),
-              32
-            )
-            """,
-            u.nickname,
-            u.name,
-            ^processed_query
-          )
-      },
-      where:
-        fragment(
-          """
-            (setweight(to_tsvector('simple', regexp_replace(?, '\\W', ' ', 'g')), 'A') ||
-            setweight(to_tsvector('simple', regexp_replace(coalesce(?, ''), '\\W', ' ', 'g')), 'B')) @@ to_tsquery('simple', ?)
-          """,
-          u.nickname,
-          u.name,
-          ^processed_query
-        )
-    )
-    |> restrict_deactivated()
-  end
-
-  defp trigram_search_subquery(term) do
-    from(
-      u in User,
-      select_merge: %{
-        # ^1 gives 'Postgrex expected a binary, got 1' for some weird reason
-        search_type: fragment("?", 1),
-        search_rank:
-          fragment(
-            "similarity(?, trim(? || ' ' || coalesce(?, '')))",
-            ^term,
-            u.nickname,
-            u.name
-          )
-      },
-      where: fragment("trim(? || ' ' || coalesce(?, '')) % ?", u.nickname, u.name, ^term)
-    )
-    |> restrict_deactivated()
-  end
-
   def mute(muter, %User{ap_id: ap_id}) do
     info_cng =
       muter.info
@@ -962,15 +846,12 @@ defmodule Pleroma.User do
   def mutes?(nil, _), do: false
   def mutes?(user, %{ap_id: ap_id}), do: Enum.member?(user.info.mutes, ap_id)
 
-  def blocks?(user, %{ap_id: ap_id}) do
-    blocks = user.info.blocks
-    domain_blocks = user.info.domain_blocks
+  def blocks?(%User{info: info} = _user, %{ap_id: ap_id}) do
+    blocks = info.blocks
+    domain_blocks = info.domain_blocks
     %{host: host} = URI.parse(ap_id)
 
-    Enum.member?(blocks, ap_id) ||
-      Enum.any?(domain_blocks, fn domain ->
-        host == domain
-      end)
+    Enum.member?(blocks, ap_id) || Enum.any?(domain_blocks, &(&1 == host))
   end
 
   def subscribed_to?(user, %{ap_id: ap_id}) do
@@ -1056,18 +937,26 @@ defmodule Pleroma.User do
 
   @spec perform(atom(), User.t()) :: {:ok, User.t()}
   def perform(:delete, %User{} = user) do
-    {:ok, user} = User.deactivate(user)
+    {:ok, _user} = ActivityPub.delete(user)
 
     # Remove all relationships
     {:ok, followers} = User.get_followers(user)
 
-    Enum.each(followers, fn follower -> User.unfollow(follower, user) end)
+    Enum.each(followers, fn follower ->
+      ActivityPub.unfollow(follower, user)
+      User.unfollow(follower, user)
+    end)
 
     {:ok, friends} = User.get_friends(user)
 
-    Enum.each(friends, fn followed -> User.unfollow(user, followed) end)
+    Enum.each(friends, fn followed ->
+      ActivityPub.unfollow(user, followed)
+      User.unfollow(user, followed)
+    end)
 
     delete_user_activities(user)
+    invalidate_cache(user)
+    Repo.delete(user)
   end
 
   @spec perform(atom(), User.t()) :: {:ok, User.t()}
@@ -1123,6 +1012,56 @@ defmodule Pleroma.User do
     )
   end
 
+  @spec sync_follow_counter() :: :ok
+  def sync_follow_counter,
+    do: PleromaJobQueue.enqueue(:background, __MODULE__, [:sync_follow_counters])
+
+  @spec perform(:sync_follow_counters) :: :ok
+  def perform(:sync_follow_counters) do
+    {:ok, _pid} = Agent.start_link(fn -> %{} end, name: :domain_errors)
+    config = Pleroma.Config.get([:instance, :external_user_synchronization])
+
+    :ok = sync_follow_counters(config)
+    Agent.stop(:domain_errors)
+  end
+
+  @spec sync_follow_counters(keyword()) :: :ok
+  def sync_follow_counters(opts \\ []) do
+    users = external_users(opts)
+
+    if length(users) > 0 do
+      errors = Agent.get(:domain_errors, fn state -> state end)
+      {last, updated_errors} = User.Synchronization.call(users, errors, opts)
+      Agent.update(:domain_errors, fn _state -> updated_errors end)
+      sync_follow_counters(max_id: last.id, limit: opts[:limit])
+    else
+      :ok
+    end
+  end
+
+  @spec external_users(keyword()) :: [User.t()]
+  def external_users(opts \\ []) do
+    query =
+      User.Query.build(%{
+        external: true,
+        active: true,
+        order_by: :id,
+        select: [:id, :ap_id, :info]
+      })
+
+    query =
+      if opts[:max_id],
+        do: where(query, [u], u.id > ^opts[:max_id]),
+        else: query
+
+    query =
+      if opts[:limit],
+        do: limit(query, ^opts[:limit]),
+        else: query
+
+    Repo.all(query)
+  end
+
   def blocks_import(%User{} = blocker, blocked_identifiers) when is_list(blocked_identifiers),
     do:
       PleromaJobQueue.enqueue(:background, __MODULE__, [
@@ -1140,18 +1079,35 @@ defmodule Pleroma.User do
       ])
 
   def delete_user_activities(%User{ap_id: ap_id} = user) do
-    stream =
-      ap_id
-      |> Activity.query_by_actor()
-      |> Repo.stream()
-
-    Repo.transaction(fn -> Enum.each(stream, &delete_activity(&1)) end, timeout: :infinity)
+    ap_id
+    |> Activity.query_by_actor()
+    |> RepoStreamer.chunk_stream(50)
+    |> Stream.each(fn activities ->
+      Enum.each(activities, &delete_activity(&1))
+    end)
+    |> Stream.run()
 
     {:ok, user}
   end
 
   defp delete_activity(%{data: %{"type" => "Create"}} = activity) do
-    Object.normalize(activity) |> ActivityPub.delete()
+    activity
+    |> Object.normalize()
+    |> ActivityPub.delete()
+  end
+
+  defp delete_activity(%{data: %{"type" => "Like"}} = activity) do
+    user = get_cached_by_ap_id(activity.actor)
+    object = Object.normalize(activity)
+
+    ActivityPub.unlike(user, object)
+  end
+
+  defp delete_activity(%{data: %{"type" => "Announce"}} = activity) do
+    user = get_cached_by_ap_id(activity.actor)
+    object = Object.normalize(activity)
+
+    ActivityPub.unannounce(user, object)
   end
 
   defp delete_activity(_activity), do: "Doing nothing"
@@ -1160,9 +1116,7 @@ defmodule Pleroma.User do
     Pleroma.HTML.Scrubber.TwitterText
   end
 
-  @default_scrubbers Pleroma.Config.get([:markup, :scrub_policy])
-
-  def html_filter_policy(_), do: @default_scrubbers
+  def html_filter_policy(_), do: Pleroma.Config.get([:markup, :scrub_policy])
 
   def fetch_by_ap_id(ap_id) do
     ap_try = ActivityPub.make_user_from_ap_id(ap_id)
@@ -1449,4 +1403,14 @@ defmodule Pleroma.User do
     )
     |> Repo.all()
   end
+
+  defdelegate search(query, opts \\ []), to: User.Search
+
+  defp put_password_hash(
+         %Ecto.Changeset{valid?: true, changes: %{password: password}} = changeset
+       ) do
+    change(changeset, password_hash: Pbkdf2.hashpwsalt(password))
+  end
+
+  defp put_password_hash(changeset), do: changeset
 end

lib/pleroma/user/query.ex

@@ -7,7 +7,7 @@ defmodule Pleroma.User.Query do
   User query builder module. Builds query from new query or another user query.
 
     ## Example:
-        query = Pleroma.User.Query(%{nickname: "nickname"})
+        query = Pleroma.User.Query.build(%{nickname: "nickname"})
         another_query = Pleroma.User.Query.build(query, %{email: "email@example.com"})
         Pleroma.Repo.all(query)
         Pleroma.Repo.all(another_query)
@@ -47,7 +47,10 @@ defmodule Pleroma.User.Query do
             friends: User.t(),
             recipients_from_activity: [String.t()],
             nickname: [String.t()],
-            ap_id: [String.t()]
+            ap_id: [String.t()],
+            order_by: term(),
+            select: term(),
+            limit: pos_integer()
           }
           | %{}
 
@@ -141,6 +144,18 @@ defmodule Pleroma.User.Query do
     where(query, [u], u.ap_id in ^to or fragment("? && ?", u.following, ^to))
   end
 
+  defp compose_query({:order_by, key}, query) do
+    order_by(query, [u], field(u, ^key))
+  end
+
+  defp compose_query({:select, keys}, query) do
+    select(query, [u], ^keys)
+  end
+
+  defp compose_query({:limit, limit}, query) do
+    limit(query, ^limit)
+  end
+
   defp compose_query(_unsupported_param, query), do: query
 
   defp prepare_tag_criteria(tag, query) do

lib/pleroma/user/search.ex

@@ -0,0 +1,227 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.User.Search do
+  alias Pleroma.Repo
+  alias Pleroma.User
+  import Ecto.Query
+
+  @similarity_threshold 0.25
+  @limit 20
+
+  def search(query_string, opts \\ []) do
+    resolve = Keyword.get(opts, :resolve, false)
+    following = Keyword.get(opts, :following, false)
+    result_limit = Keyword.get(opts, :limit, @limit)
+    offset = Keyword.get(opts, :offset, 0)
+
+    for_user = Keyword.get(opts, :for_user)
+
+    query_string = format_query(query_string)
+
+    maybe_resolve(resolve, for_user, query_string)
+
+    {:ok, results} =
+      Repo.transaction(fn ->
+        Ecto.Adapters.SQL.query(
+          Repo,
+          "select set_limit(#{@similarity_threshold})",
+          []
+        )
+
+        query_string
+        |> search_query(for_user, following)
+        |> paginate(result_limit, offset)
+        |> Repo.all()
+      end)
+
+    results
+  end
+
+  defp format_query(query_string) do
+    # Strip the beginning @ off if there is a query
+    query_string = String.trim_leading(query_string, "@")
+
+    with [name, domain] <- String.split(query_string, "@"),
+         formatted_domain <- String.replace(domain, ~r/[!-\-|@|[-`|{-~|\/|:]+/, "") do
+      name <> "@" <> to_string(:idna.encode(formatted_domain))
+    else
+      _ -> query_string
+    end
+  end
+
+  defp search_query(query_string, for_user, following) do
+    for_user
+    |> base_query(following)
+    |> filter_blocked_user(for_user)
+    |> filter_blocked_domains(for_user)
+    |> search_subqueries(query_string)
+    |> union_subqueries
+    |> distinct_query()
+    |> boost_search_rank_query(for_user)
+    |> subquery()
+    |> order_by(desc: :search_rank)
+    |> maybe_restrict_local(for_user)
+  end
+
+  defp base_query(_user, false), do: User
+  defp base_query(user, true), do: User.get_followers_query(user)
+
+  defp filter_blocked_user(query, %User{info: %{blocks: blocks}})
+       when length(blocks) > 0 do
+    from(q in query, where: not (q.ap_id in ^blocks))
+  end
+
+  defp filter_blocked_user(query, _), do: query
+
+  defp filter_blocked_domains(query, %User{info: %{domain_blocks: domain_blocks}})
+       when length(domain_blocks) > 0 do
+    domains = Enum.join(domain_blocks, ",")
+
+    from(
+      q in query,
+      where: fragment("substring(ap_id from '.*://([^/]*)') NOT IN (?)", ^domains)
+    )
+  end
+
+  defp filter_blocked_domains(query, _), do: query
+
+  defp paginate(query, limit, offset) do
+    from(q in query, limit: ^limit, offset: ^offset)
+  end
+
+  defp union_subqueries({fts_subquery, trigram_subquery}) do
+    from(s in trigram_subquery, union_all: ^fts_subquery)
+  end
+
+  defp search_subqueries(base_query, query_string) do
+    {
+      fts_search_subquery(base_query, query_string),
+      trigram_search_subquery(base_query, query_string)
+    }
+  end
+
+  defp distinct_query(q) do
+    from(s in subquery(q), order_by: s.search_type, distinct: s.id)
+  end
+
+  defp maybe_resolve(true, user, query) do
+    case {limit(), user} do
+      {:all, _} -> :noop
+      {:unauthenticated, %User{}} -> User.get_or_fetch(query)
+      {:unauthenticated, _} -> :noop
+      {false, _} -> User.get_or_fetch(query)
+    end
+  end
+
+  defp maybe_resolve(_, _, _), do: :noop
+
+  defp maybe_restrict_local(q, user) do
+    case {limit(), user} do
+      {:all, _} -> restrict_local(q)
+      {:unauthenticated, %User{}} -> q
+      {:unauthenticated, _} -> restrict_local(q)
+      {false, _} -> q
+    end
+  end
+
+  defp limit, do: Pleroma.Config.get([:instance, :limit_to_local_content], :unauthenticated)
+
+  defp restrict_local(q), do: where(q, [u], u.local == true)
+
+  defp boost_search_rank_query(query, nil), do: query
+
+  defp boost_search_rank_query(query, for_user) do
+    friends_ids = User.get_friends_ids(for_user)
+    followers_ids = User.get_followers_ids(for_user)
+
+    from(u in subquery(query),
+      select_merge: %{
+        search_rank:
+          fragment(
+            """
+             CASE WHEN (?) THEN 0.5 + (?) * 1.3
+             WHEN (?) THEN 0.5 + (?) * 1.2
+             WHEN (?) THEN (?) * 1.1
+             ELSE (?) END
+            """,
+            u.id in ^friends_ids and u.id in ^followers_ids,
+            u.search_rank,
+            u.id in ^friends_ids,
+            u.search_rank,
+            u.id in ^followers_ids,
+            u.search_rank,
+            u.search_rank
+          )
+      }
+    )
+  end
+
+  @spec fts_search_subquery(User.t() | Ecto.Query.t(), String.t()) :: Ecto.Query.t()
+  defp fts_search_subquery(query, term) do
+    processed_query =
+      String.trim_trailing(term, "@" <> local_domain())
+      |> String.replace(~r/[!-\/|@|[-`|{-~|:-?]+/, " ")
+      |> String.trim()
+      |> String.split()
+      |> Enum.map(&(&1 <> ":*"))
+      |> Enum.join(" | ")
+
+    from(
+      u in query,
+      select_merge: %{
+        search_type: ^0,
+        search_rank:
+          fragment(
+            """
+            ts_rank_cd(
+              setweight(to_tsvector('simple', regexp_replace(?, '\\W', ' ', 'g')), 'A') ||
+              setweight(to_tsvector('simple', regexp_replace(coalesce(?, ''), '\\W', ' ', 'g')), 'B'),
+              to_tsquery('simple', ?),
+              32
+            )
+            """,
+            u.nickname,
+            u.name,
+            ^processed_query
+          )
+      },
+      where:
+        fragment(
+          """
+            (setweight(to_tsvector('simple', regexp_replace(?, '\\W', ' ', 'g')), 'A') ||
+            setweight(to_tsvector('simple', regexp_replace(coalesce(?, ''), '\\W', ' ', 'g')), 'B')) @@ to_tsquery('simple', ?)
+          """,
+          u.nickname,
+          u.name,
+          ^processed_query
+        )
+    )
+    |> User.restrict_deactivated()
+  end
+
+  @spec trigram_search_subquery(User.t() | Ecto.Query.t(), String.t()) :: Ecto.Query.t()
+  defp trigram_search_subquery(query, term) do
+    term = String.trim_trailing(term, "@" <> local_domain())
+
+    from(
+      u in query,
+      select_merge: %{
+        # ^1 gives 'Postgrex expected a binary, got 1' for some weird reason
+        search_type: fragment("?", 1),
+        search_rank:
+          fragment(
+            "similarity(?, trim(? || ' ' || coalesce(?, '')))",
+            ^term,
+            u.nickname,
+            u.name
+          )
+      },
+      where: fragment("trim(? || ' ' || coalesce(?, '')) % ?", u.nickname, u.name, ^term)
+    )
+    |> User.restrict_deactivated()
+  end
+
+  defp local_domain, do: Pleroma.Config.get([Pleroma.Web.Endpoint, :url, :host])
+end

lib/pleroma/user/synchronization.ex

@@ -0,0 +1,60 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.User.Synchronization do
+  alias Pleroma.HTTP
+  alias Pleroma.User
+
+  @spec call([User.t()], map(), keyword()) :: {User.t(), map()}
+  def call(users, errors, opts \\ []) do
+    do_call(users, errors, opts)
+  end
+
+  defp do_call([user | []], errors, opts) do
+    updated = fetch_counters(user, errors, opts)
+    {user, updated}
+  end
+
+  defp do_call([user | others], errors, opts) do
+    updated = fetch_counters(user, errors, opts)
+    do_call(others, updated, opts)
+  end
+
+  defp fetch_counters(user, errors, opts) do
+    %{host: host} = URI.parse(user.ap_id)
+
+    info = %{}
+    {following, errors} = fetch_counter(user.ap_id <> "/following", host, errors, opts)
+    info = if following, do: Map.put(info, :following_count, following), else: info
+
+    {followers, errors} = fetch_counter(user.ap_id <> "/followers", host, errors, opts)
+    info = if followers, do: Map.put(info, :follower_count, followers), else: info
+
+    User.set_info_cache(user, info)
+    errors
+  end
+
+  defp available_domain?(domain, errors, opts) do
+    max_retries = Keyword.get(opts, :max_retries, 3)
+    not (Map.has_key?(errors, domain) && errors[domain] >= max_retries)
+  end
+
+  defp fetch_counter(url, host, errors, opts) do
+    with true <- available_domain?(host, errors, opts),
+         {:ok, %{body: body, status: code}} when code in 200..299 <-
+           HTTP.get(
+             url,
+             [{:Accept, "application/activity+json"}]
+           ),
+         {:ok, data} <- Jason.decode(body) do
+      {data["totalItems"], errors}
+    else
+      false ->
+        {nil, errors}
+
+      _ ->
+        {nil, Map.update(errors, host, 1, &(&1 + 1))}
+    end
+  end
+end

lib/pleroma/user/synchronization_worker.ex

@@ -0,0 +1,32 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-onl
+
+defmodule Pleroma.User.SynchronizationWorker do
+  use GenServer
+
+  def start_link do
+    config = Pleroma.Config.get([:instance, :external_user_synchronization])
+
+    if config[:enabled] do
+      GenServer.start_link(__MODULE__, interval: config[:interval])
+    else
+      :ignore
+    end
+  end
+
+  def init(opts) do
+    schedule_next(opts)
+    {:ok, opts}
+  end
+
+  def handle_info(:sync_follow_counters, opts) do
+    Pleroma.User.sync_follow_counter()
+    schedule_next(opts)
+    {:noreply, opts}
+  end
+
+  defp schedule_next(opts) do
+    Process.send_after(self(), :sync_follow_counters, opts[:interval])
+  end
+end

lib/pleroma/user/welcome_message.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.User.WelcomeMessage do
   alias Pleroma.User
   alias Pleroma.Web.CommonAPI

lib/pleroma/web/activity_pub/activity_pub.ex

@@ -188,6 +188,22 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     end)
   end
 
+  def stream_out_participations(%Object{data: %{"context" => context}}, user) do
+    with %Conversation{} = conversation <- Conversation.get_for_ap_id(context),
+         conversation = Repo.preload(conversation, :participations),
+         last_activity_id =
+           fetch_latest_activity_id_for_context(conversation.ap_id, %{
+             "user" => user,
+             "blocking_user" => user
+           }) do
+      if last_activity_id do
+        stream_out_participations(conversation.participations)
+      end
+    end
+  end
+
+  def stream_out_participations(_, _), do: :noop
+
   def stream_out(activity) do
     public = "https://www.w3.org/ns/activitystreams#Public"
 
@@ -388,6 +404,19 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     end
   end
 
+  def delete(%User{ap_id: ap_id, follower_address: follower_address} = user) do
+    with data <- %{
+           "to" => [follower_address],
+           "type" => "Delete",
+           "actor" => ap_id,
+           "object" => %{"type" => "Person", "id" => ap_id}
+         },
+         {:ok, activity} <- insert(data, true, true),
+         :ok <- maybe_federate(activity) do
+      {:ok, user}
+    end
+  end
+
   def delete(%Object{data: %{"id" => id, "actor" => actor}} = object, local \\ true) do
     user = User.get_cached_by_ap_id(actor)
     to = (object.data["to"] || []) ++ (object.data["cc"] || [])
@@ -400,7 +429,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
            "to" => to,
            "deleted_activity_id" => activity && activity.id
          },
-         {:ok, activity} <- insert(data, local),
+         {:ok, activity} <- insert(data, local, false),
+         stream_out_participations(object, user),
          _ <- decrease_replies_count_if_reply(object),
          # Changing note count prior to enqueuing federation task in order to avoid
          # race conditions on updating user.info

lib/pleroma/web/activity_pub/activity_pub_controller.ex

@@ -31,9 +31,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
       conn
     else
       conn
-      |> put_status(404)
-      |> json(%{error: "not found"})
-      |> halt
+      |> render_error(:not_found, "not found")
+      |> halt()
     end
   end
 
@@ -190,7 +189,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
       Logger.info(inspect(conn.req_headers))
     end
 
-    json(conn, "error")
+    json(conn, dgettext("errors", "error"))
   end
 
   def relay(conn, _params) do
@@ -218,9 +217,15 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
       |> put_resp_header("content-type", "application/activity+json")
       |> json(UserView.render("inbox.json", %{user: user, max_id: params["max_id"]}))
     else
+      err =
+        dgettext("errors", "can't read inbox of %{nickname} as %{as_nickname}",
+          nickname: nickname,
+          as_nickname: user.nickname
+        )
+
       conn
       |> put_status(:forbidden)
-      |> json("can't read inbox of #{nickname} as #{user.nickname}")
+      |> json(err)
     end
   end
 
@@ -246,7 +251,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
          {:ok, delete} <- ActivityPub.delete(object) do
       {:ok, delete}
     else
-      _ -> {:error, "Can't delete object"}
+      _ -> {:error, dgettext("errors", "Can't delete object")}
     end
   end
 
@@ -255,12 +260,12 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
          {:ok, activity, _object} <- ActivityPub.like(user, object) do
       {:ok, activity}
     else
-      _ -> {:error, "Can't like object"}
+      _ -> {:error, dgettext("errors", "Can't like object")}
     end
   end
 
   def handle_user_activity(_, _) do
-    {:error, "Unhandled activity type"}
+    {:error, dgettext("errors", "Unhandled activity type")}
   end
 
   def update_outbox(
@@ -288,22 +293,28 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
           |> json(message)
       end
     else
+      err =
+        dgettext("errors", "can't update outbox of %{nickname} as %{as_nickname}",
+          nickname: nickname,
+          as_nickname: user.nickname
+        )
+
       conn
       |> put_status(:forbidden)
-      |> json("can't update outbox of #{nickname} as #{user.nickname}")
+      |> json(err)
     end
   end
 
   def errors(conn, {:error, :not_found}) do
     conn
-    |> put_status(404)
-    |> json("Not found")
+    |> put_status(:not_found)
+    |> json(dgettext("errors", "Not found"))
   end
 
   def errors(conn, _e) do
     conn
-    |> put_status(500)
-    |> json("error")
+    |> put_status(:internal_server_error)
+    |> json(dgettext("errors", "error"))
   end
 
   defp set_requester_reachable(%Plug.Conn{} = conn, _) do

lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex

@@ -0,0 +1,48 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.MRF.AntiLinkSpamPolicy do
+  alias Pleroma.User
+
+  require Logger
+
+  # has the user successfully posted before?
+  defp old_user?(%User{} = u) do
+    u.info.note_count > 0 || u.info.follower_count > 0
+  end
+
+  # does the post contain links?
+  defp contains_links?(%{"content" => content} = _object) do
+    content
+    |> Floki.filter_out("a.mention,a.hashtag,a[rel~=\"tag\"],a.zrl")
+    |> Floki.attribute("a", "href")
+    |> length() > 0
+  end
+
+  defp contains_links?(_), do: false
+
+  def filter(%{"type" => "Create", "actor" => actor, "object" => object} = message) do
+    with {:ok, %User{} = u} <- User.get_or_fetch_by_ap_id(actor),
+         {:contains_links, true} <- {:contains_links, contains_links?(object)},
+         {:old_user, true} <- {:old_user, old_user?(u)} do
+      {:ok, message}
+    else
+      {:contains_links, false} ->
+        {:ok, message}
+
+      {:old_user, false} ->
+        {:reject, nil}
+
+      {:error, _} ->
+        {:reject, nil}
+
+      e ->
+        Logger.warn("[MRF anti-link-spam] WTF: unhandled error #{inspect(e)}")
+        {:reject, nil}
+    end
+  end
+
+  # in all other cases, pass through
+  def filter(message), do: {:ok, message}
+end

lib/pleroma/web/activity_pub/mrf/ensure_re_prepended.ex

@@ -9,8 +9,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.EnsureRePrepended do
   @behaviour Pleroma.Web.ActivityPub.MRF
 
   @reply_prefix Regex.compile!("^re:[[:space:]]*", [:caseless])
+
   def filter_by_summary(
-        %{"summary" => parent_summary} = _parent,
+        %{data: %{"summary" => parent_summary}} = _in_reply_to,
         %{"summary" => child_summary} = child
       )
       when not is_nil(child_summary) and byte_size(child_summary) > 0 and
@@ -24,17 +25,13 @@ defmodule Pleroma.Web.ActivityPub.MRF.EnsureRePrepended do
     end
   end
 
-  def filter_by_summary(_parent, child), do: child
-
-  def filter(%{"type" => activity_type} = object) when activity_type == "Create" do
-    child = object["object"]
-    in_reply_to = Object.normalize(child["inReplyTo"])
+  def filter_by_summary(_in_reply_to, child), do: child
 
+  def filter(%{"type" => "Create", "object" => child_object} = object) do
     child =
-      if(in_reply_to,
-        do: filter_by_summary(in_reply_to.data, child),
-        else: child
-      )
+      child_object["inReplyTo"]
+      |> Object.normalize(child_object["inReplyTo"])
+      |> filter_by_summary(child_object)
 
     object = Map.put(object, "object", child)
 

lib/pleroma/web/activity_pub/mrf/mediaproxy_warming_policy.ex

@@ -0,0 +1,56 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do
+  @moduledoc "Preloads any attachments in the MediaProxy cache by prefetching them"
+  @behaviour Pleroma.Web.ActivityPub.MRF
+
+  alias Pleroma.HTTP
+  alias Pleroma.Web.MediaProxy
+
+  require Logger
+
+  @hackney_options [
+    pool: :media,
+    recv_timeout: 10_000
+  ]
+
+  def perform(:prefetch, url) do
+    Logger.info("Prefetching #{inspect(url)}")
+
+    url
+    |> MediaProxy.url()
+    |> HTTP.get([], adapter: @hackney_options)
+  end
+
+  def perform(:preload, %{"object" => %{"attachment" => attachments}} = _message) do
+    Enum.each(attachments, fn
+      %{"url" => url} when is_list(url) ->
+        url
+        |> Enum.each(fn
+          %{"href" => href} ->
+            PleromaJobQueue.enqueue(:background, __MODULE__, [:prefetch, href])
+
+          x ->
+            Logger.debug("Unhandled attachment URL object #{inspect(x)}")
+        end)
+
+      x ->
+        Logger.debug("Unhandled attachment #{inspect(x)}")
+    end)
+  end
+
+  @impl true
+  def filter(
+        %{"type" => "Create", "object" => %{"attachment" => attachments} = _object} = message
+      )
+      when is_list(attachments) and length(attachments) > 0 do
+    PleromaJobQueue.enqueue(:background, __MODULE__, [:preload, message])
+
+    {:ok, message}
+  end
+
+  @impl true
+  def filter(message), do: {:ok, message}
+end

lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex

@@ -10,19 +10,11 @@ defmodule Pleroma.Web.ActivityPub.MRF.NoPlaceholderTextPolicy do
   def filter(
         %{
           "type" => "Create",
-          "object" => %{"content" => content, "attachment" => _attachment} = child_object
+          "object" => %{"content" => content, "attachment" => _} = _child_object
         } = object
       )
       when content in [".", "<p>.</p>"] do
-    child_object =
-      child_object
-      |> Map.put("content", "")
-
-    object =
-      object
-      |> Map.put("object", child_object)
-
-    {:ok, object}
+    {:ok, put_in(object, ["object", "content"], "")}
   end
 
   @impl true

lib/pleroma/web/activity_pub/mrf/normalize_markup.ex

@@ -8,18 +8,14 @@ defmodule Pleroma.Web.ActivityPub.MRF.NormalizeMarkup do
 
   @behaviour Pleroma.Web.ActivityPub.MRF
 
-  def filter(%{"type" => activity_type} = object) when activity_type == "Create" do
+  def filter(%{"type" => "Create", "object" => child_object} = object) do
     scrub_policy = Pleroma.Config.get([:mrf_normalize_markup, :scrub_policy])
 
-    child = object["object"]
-
     content =
-      child["content"]
+      child_object["content"]
       |> HTML.filter_tags(scrub_policy)
 
-    child = Map.put(child, "content", content)
-
-    object = Map.put(object, "object", child)
+    object = put_in(object, ["object", "content"], content)
 
     {:ok, object}
   end

lib/pleroma/web/activity_pub/mrf/reject_non_public.ex

@@ -3,46 +3,42 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.ActivityPub.MRF.RejectNonPublic do
-  alias Pleroma.User
   @moduledoc "Rejects non-public (followers-only, direct) activities"
+
+  alias Pleroma.Config
+  alias Pleroma.User
+
   @behaviour Pleroma.Web.ActivityPub.MRF
 
+  @public "https://www.w3.org/ns/activitystreams#Public"
+
   @impl true
   def filter(%{"type" => "Create"} = object) do
     user = User.get_cached_by_ap_id(object["actor"])
-    public = "https://www.w3.org/ns/activitystreams#Public"
 
     # Determine visibility
     visibility =
       cond do
-        public in object["to"] -> "public"
-        public in object["cc"] -> "unlisted"
+        @public in object["to"] -> "public"
+        @public in object["cc"] -> "unlisted"
         user.follower_address in object["to"] -> "followers"
         true -> "direct"
       end
 
-    policy = Pleroma.Config.get(:mrf_rejectnonpublic)
+    policy = Config.get(:mrf_rejectnonpublic)
 
-    case visibility do
-      "public" ->
+    cond do
+      visibility in ["public", "unlisted"] ->
         {:ok, object}
 
-      "unlisted" ->
+      visibility == "followers" and Keyword.get(policy, :allow_followersonly) ->
         {:ok, object}
 
-      "followers" ->
-        with true <- Keyword.get(policy, :allow_followersonly) do
-          {:ok, object}
-        else
-          _e -> {:reject, nil}
-        end
+      visibility == "direct" and Keyword.get(policy, :allow_direct) ->
+        {:ok, object}
 
-      "direct" ->
-        with true <- Keyword.get(policy, :allow_direct) do
-          {:ok, object}
-        else
-          _e -> {:reject, nil}
-        end
+      true ->
+        {:reject, nil}
     end
   end
 

lib/pleroma/web/activity_pub/mrf/tag_policy.ex

@@ -19,12 +19,17 @@ defmodule Pleroma.Web.ActivityPub.MRF.TagPolicy do
      - `mrf_tag:disable-any-subscription`: Reject any follow requests
   """
 
+  @public "https://www.w3.org/ns/activitystreams#Public"
+
   defp get_tags(%User{tags: tags}) when is_list(tags), do: tags
   defp get_tags(_), do: []
 
   defp process_tag(
          "mrf_tag:media-force-nsfw",
-         %{"type" => "Create", "object" => %{"attachment" => child_attachment} = object} = message
+         %{
+           "type" => "Create",
+           "object" => %{"attachment" => child_attachment} = object
+         } = message
        )
        when length(child_attachment) > 0 do
     tags = (object["tag"] || []) ++ ["nsfw"]
@@ -41,7 +46,10 @@ defmodule Pleroma.Web.ActivityPub.MRF.TagPolicy do
 
   defp process_tag(
          "mrf_tag:media-strip",
-         %{"type" => "Create", "object" => %{"attachment" => child_attachment} = object} = message
+         %{
+           "type" => "Create",
+           "object" => %{"attachment" => child_attachment} = object
+         } = message
        )
        when length(child_attachment) > 0 do
     object = Map.delete(object, "attachment")
@@ -52,19 +60,22 @@ defmodule Pleroma.Web.ActivityPub.MRF.TagPolicy do
 
   defp process_tag(
          "mrf_tag:force-unlisted",
-         %{"type" => "Create", "to" => to, "cc" => cc, "actor" => actor} = message
+         %{
+           "type" => "Create",
+           "to" => to,
+           "cc" => cc,
+           "actor" => actor,
+           "object" => object
+         } = message
        ) do
     user = User.get_cached_by_ap_id(actor)
 
-    if Enum.member?(to, "https://www.w3.org/ns/activitystreams#Public") do
-      to =
-        List.delete(to, "https://www.w3.org/ns/activitystreams#Public") ++ [user.follower_address]
-
-      cc =
-        List.delete(cc, user.follower_address) ++ ["https://www.w3.org/ns/activitystreams#Public"]
+    if Enum.member?(to, @public) do
+      to = List.delete(to, @public) ++ [user.follower_address]
+      cc = List.delete(cc, user.follower_address) ++ [@public]
 
       object =
-        message["object"]
+        object
         |> Map.put("to", to)
         |> Map.put("cc", cc)
 
@@ -82,19 +93,22 @@ defmodule Pleroma.Web.ActivityPub.MRF.TagPolicy do
 
   defp process_tag(
          "mrf_tag:sandbox",
-         %{"type" => "Create", "to" => to, "cc" => cc, "actor" => actor} = message
+         %{
+           "type" => "Create",
+           "to" => to,
+           "cc" => cc,
+           "actor" => actor,
+           "object" => object
+         } = message
        ) do
     user = User.get_cached_by_ap_id(actor)
 
-    if Enum.member?(to, "https://www.w3.org/ns/activitystreams#Public") or
-         Enum.member?(cc, "https://www.w3.org/ns/activitystreams#Public") do
-      to =
-        List.delete(to, "https://www.w3.org/ns/activitystreams#Public") ++ [user.follower_address]
-
-      cc = List.delete(cc, "https://www.w3.org/ns/activitystreams#Public")
+    if Enum.member?(to, @public) or Enum.member?(cc, @public) do
+      to = List.delete(to, @public) ++ [user.follower_address]
+      cc = List.delete(cc, @public)
 
       object =
-        message["object"]
+        object
         |> Map.put("to", to)
         |> Map.put("cc", cc)
 
@@ -123,7 +137,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.TagPolicy do
     end
   end
 
-  defp process_tag("mrf_tag:disable-any-subscription", %{"type" => "Follow"}), do: {:reject, nil}
+  defp process_tag("mrf_tag:disable-any-subscription", %{"type" => "Follow"}),
+    do: {:reject, nil}
 
   defp process_tag(_, message), do: {:ok, message}
 

lib/pleroma/web/activity_pub/mrf/user_allowlist_policy.ex

@@ -21,7 +21,12 @@ defmodule Pleroma.Web.ActivityPub.MRF.UserAllowListPolicy do
   @impl true
   def filter(%{"actor" => actor} = object) do
     actor_info = URI.parse(actor)
-    allow_list = Config.get([:mrf_user_allowlist, String.to_atom(actor_info.host)], [])
+
+    allow_list =
+      Config.get(
+        [:mrf_user_allowlist, String.to_atom(actor_info.host)],
+        []
+      )
 
     filter_by_list(object, allow_list)
   end

lib/pleroma/web/activity_pub/publisher.ex

@@ -88,7 +88,7 @@ defmodule Pleroma.Web.ActivityPub.Publisher do
       true
     else
       inbox_info = URI.parse(inbox)
-      !Enum.member?(Pleroma.Config.get([:instance, :quarantined_instances], []), inbox_info.host)
+      !Enum.member?(Config.get([:instance, :quarantined_instances], []), inbox_info.host)
     end
   end
 

lib/pleroma/web/activity_pub/transmogrifier.ex

@@ -14,6 +14,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.ActivityPub.Utils
   alias Pleroma.Web.ActivityPub.Visibility
+  alias Pleroma.Web.Federator
 
   import Ecto.Query
 
@@ -22,20 +23,20 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   @doc """
   Modifies an incoming AP object (mastodon format) to our internal format.
   """
-  def fix_object(object) do
+  def fix_object(object, options \\ []) do
     object
     |> fix_actor
     |> fix_url
     |> fix_attachments
     |> fix_context
-    |> fix_in_reply_to
+    |> fix_in_reply_to(options)
     |> fix_emoji
     |> fix_tag
     |> fix_content_map
     |> fix_likes
     |> fix_addressing
     |> fix_summary
-    |> fix_type
+    |> fix_type(options)
   end
 
   def fix_summary(%{"summary" => nil} = object) do
@@ -164,7 +165,9 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
     object
   end
 
-  def fix_in_reply_to(%{"inReplyTo" => in_reply_to} = object)
+  def fix_in_reply_to(object, options \\ [])
+
+  def fix_in_reply_to(%{"inReplyTo" => in_reply_to} = object, options)
       when not is_nil(in_reply_to) do
     in_reply_to_id =
       cond do
@@ -182,28 +185,34 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
           ""
       end
 
-    case get_obj_helper(in_reply_to_id) do
-      {:ok, replied_object} ->
-        with %Activity{} = _activity <-
-               Activity.get_create_by_object_ap_id(replied_object.data["id"]) do
-          object
-          |> Map.put("inReplyTo", replied_object.data["id"])
-          |> Map.put("inReplyToAtomUri", object["inReplyToAtomUri"] || in_reply_to_id)
-          |> Map.put("conversation", replied_object.data["context"] || object["conversation"])
-          |> Map.put("context", replied_object.data["context"] || object["conversation"])
-        else
-          e ->
-            Logger.error("Couldn't fetch \"#{inspect(in_reply_to_id)}\", error: #{inspect(e)}")
-            object
-        end
+    object = Map.put(object, "inReplyToAtomUri", in_reply_to_id)
 
-      e ->
-        Logger.error("Couldn't fetch \"#{inspect(in_reply_to_id)}\", error: #{inspect(e)}")
-        object
+    if Federator.allowed_incoming_reply_depth?(options[:depth]) do
+      case get_obj_helper(in_reply_to_id, options) do
+        {:ok, replied_object} ->
+          with %Activity{} = _activity <-
+                 Activity.get_create_by_object_ap_id(replied_object.data["id"]) do
+            object
+            |> Map.put("inReplyTo", replied_object.data["id"])
+            |> Map.put("inReplyToAtomUri", object["inReplyToAtomUri"] || in_reply_to_id)
+            |> Map.put("conversation", replied_object.data["context"] || object["conversation"])
+            |> Map.put("context", replied_object.data["context"] || object["conversation"])
+          else
+            e ->
+              Logger.error("Couldn't fetch \"#{inspect(in_reply_to_id)}\", error: #{inspect(e)}")
+              object
+          end
+
+        e ->
+          Logger.error("Couldn't fetch \"#{inspect(in_reply_to_id)}\", error: #{inspect(e)}")
+          object
+      end
+    else
+      object
     end
   end
 
-  def fix_in_reply_to(object), do: object
+  def fix_in_reply_to(object, _options), do: object
 
   def fix_context(object) do
     context = object["context"] || object["conversation"] || Utils.generate_context_id()
@@ -336,17 +345,22 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
 
   def fix_content_map(object), do: object
 
-  def fix_type(%{"inReplyTo" => reply_id} = object) when is_binary(reply_id) do
-    reply = Object.normalize(reply_id)
+  def fix_type(object, options \\ [])
 
-    if reply.data["type"] == "Question" and object["name"] do
+  def fix_type(%{"inReplyTo" => reply_id} = object, options) when is_binary(reply_id) do
+    reply =
+      if Federator.allowed_incoming_reply_depth?(options[:depth]) do
+        Object.normalize(reply_id, true)
+      end
+
+    if reply && (reply.data["type"] == "Question" and object["name"]) do
       Map.put(object, "type", "Answer")
     else
       object
     end
   end
 
-  def fix_type(object), do: object
+  def fix_type(object, _), do: object
 
   defp mastodon_follow_hack(%{"id" => id, "actor" => follower_id}, followed) do
     with true <- id =~ "follows",
@@ -374,9 +388,11 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
     end
   end
 
+  def handle_incoming(data, options \\ [])
+
   # Flag objects are placed ahead of the ID check because Mastodon 2.8 and earlier send them
   # with nil ID.
-  def handle_incoming(%{"type" => "Flag", "object" => objects, "actor" => actor} = data) do
+  def handle_incoming(%{"type" => "Flag", "object" => objects, "actor" => actor} = data, _options) do
     with context <- data["context"] || Utils.generate_context_id(),
          content <- data["content"] || "",
          %User{} = actor <- User.get_cached_by_ap_id(actor),
@@ -409,15 +425,19 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   end
 
   # disallow objects with bogus IDs
-  def handle_incoming(%{"id" => nil}), do: :error
-  def handle_incoming(%{"id" => ""}), do: :error
+  def handle_incoming(%{"id" => nil}, _options), do: :error
+  def handle_incoming(%{"id" => ""}, _options), do: :error
   # length of https:// = 8, should validate better, but good enough for now.
-  def handle_incoming(%{"id" => id}) when not (is_binary(id) and length(id) > 8), do: :error
+  def handle_incoming(%{"id" => id}, _options) when not (is_binary(id) and length(id) > 8),
+    do: :error
 
   # TODO: validate those with a Ecto scheme
   # - tags
   # - emoji
-  def handle_incoming(%{"type" => "Create", "object" => %{"type" => objtype} = object} = data)
+  def handle_incoming(
+        %{"type" => "Create", "object" => %{"type" => objtype} = object} = data,
+        options
+      )
       when objtype in ["Article", "Note", "Video", "Page", "Question", "Answer"] do
     actor = Containment.get_actor(data)
 
@@ -427,7 +447,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
 
     with nil <- Activity.get_create_by_object_ap_id(object["id"]),
          {:ok, %User{} = user} <- User.get_or_fetch_by_ap_id(data["actor"]) do
-      object = fix_object(data["object"])
+      options = Keyword.put(options, :depth, (options[:depth] || 0) + 1)
+      object = fix_object(data["object"], options)
 
       params = %{
         to: data["to"],
@@ -452,16 +473,19 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   end
 
   def handle_incoming(
-        %{"type" => "Follow", "object" => followed, "actor" => follower, "id" => id} = data
+        %{"type" => "Follow", "object" => followed, "actor" => follower, "id" => id} = data,
+        _options
       ) do
     with %User{local: true} = followed <- User.get_cached_by_ap_id(followed),
          {:ok, %User{} = follower} <- User.get_or_fetch_by_ap_id(follower),
          {:ok, activity} <- ActivityPub.follow(follower, followed, id, false) do
       with deny_follow_blocked <- Pleroma.Config.get([:user, :deny_follow_blocked]),
-           {:user_blocked, false} <-
+           {_, false} <-
              {:user_blocked, User.blocks?(followed, follower) && deny_follow_blocked},
-           {:user_locked, false} <- {:user_locked, User.locked?(followed)},
-           {:follow, {:ok, follower}} <- {:follow, User.follow(follower, followed)} do
+           {_, false} <- {:user_locked, User.locked?(followed)},
+           {_, {:ok, follower}} <- {:follow, User.follow(follower, followed)},
+           {_, {:ok, _}} <-
+             {:follow_state_update, Utils.update_follow_state_for_all(activity, "accept")} do
         ActivityPub.accept(%{
           to: [follower.ap_id],
           actor: followed,
@@ -470,7 +494,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
         })
       else
         {:user_blocked, true} ->
-          {:ok, _} = Utils.update_follow_state(activity, "reject")
+          {:ok, _} = Utils.update_follow_state_for_all(activity, "reject")
 
           ActivityPub.reject(%{
             to: [follower.ap_id],
@@ -480,7 +504,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
           })
 
         {:follow, {:error, _}} ->
-          {:ok, _} = Utils.update_follow_state(activity, "reject")
+          {:ok, _} = Utils.update_follow_state_for_all(activity, "reject")
 
           ActivityPub.reject(%{
             to: [follower.ap_id],
@@ -501,38 +525,35 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   end
 
   def handle_incoming(
-        %{"type" => "Accept", "object" => follow_object, "actor" => _actor, "id" => _id} = data
+        %{"type" => "Accept", "object" => follow_object, "actor" => _actor, "id" => _id} = data,
+        _options
       ) do
     with actor <- Containment.get_actor(data),
          {:ok, %User{} = followed} <- User.get_or_fetch_by_ap_id(actor),
          {:ok, follow_activity} <- get_follow_activity(follow_object, followed),
-         {:ok, follow_activity} <- Utils.update_follow_state(follow_activity, "accept"),
+         {:ok, follow_activity} <- Utils.update_follow_state_for_all(follow_activity, "accept"),
          %User{local: true} = follower <- User.get_cached_by_ap_id(follow_activity.data["actor"]),
-         {:ok, activity} <-
-           ActivityPub.accept(%{
-             to: follow_activity.data["to"],
-             type: "Accept",
-             actor: followed,
-             object: follow_activity.data["id"],
-             local: false
-           }) do
-      if not User.following?(follower, followed) do
-        {:ok, _follower} = User.follow(follower, followed)
-      end
-
-      {:ok, activity}
+         {:ok, _follower} = User.follow(follower, followed) do
+      ActivityPub.accept(%{
+        to: follow_activity.data["to"],
+        type: "Accept",
+        actor: followed,
+        object: follow_activity.data["id"],
+        local: false
+      })
     else
       _e -> :error
     end
   end
 
   def handle_incoming(
-        %{"type" => "Reject", "object" => follow_object, "actor" => _actor, "id" => _id} = data
+        %{"type" => "Reject", "object" => follow_object, "actor" => _actor, "id" => _id} = data,
+        _options
       ) do
     with actor <- Containment.get_actor(data),
          {:ok, %User{} = followed} <- User.get_or_fetch_by_ap_id(actor),
          {:ok, follow_activity} <- get_follow_activity(follow_object, followed),
-         {:ok, follow_activity} <- Utils.update_follow_state(follow_activity, "reject"),
+         {:ok, follow_activity} <- Utils.update_follow_state_for_all(follow_activity, "reject"),
          %User{local: true} = follower <- User.get_cached_by_ap_id(follow_activity.data["actor"]),
          {:ok, activity} <-
            ActivityPub.reject(%{
@@ -551,7 +572,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   end
 
   def handle_incoming(
-        %{"type" => "Like", "object" => object_id, "actor" => _actor, "id" => id} = data
+        %{"type" => "Like", "object" => object_id, "actor" => _actor, "id" => id} = data,
+        _options
       ) do
     with actor <- Containment.get_actor(data),
          {:ok, %User{} = actor} <- User.get_or_fetch_by_ap_id(actor),
@@ -564,7 +586,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   end
 
   def handle_incoming(
-        %{"type" => "Announce", "object" => object_id, "actor" => _actor, "id" => id} = data
+        %{"type" => "Announce", "object" => object_id, "actor" => _actor, "id" => id} = data,
+        _options
       ) do
     with actor <- Containment.get_actor(data),
          {:ok, %User{} = actor} <- User.get_or_fetch_by_ap_id(actor),
@@ -579,7 +602,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
 
   def handle_incoming(
         %{"type" => "Update", "object" => %{"type" => object_type} = object, "actor" => actor_id} =
-          data
+          data,
+        _options
       )
       when object_type in ["Person", "Application", "Service", "Organization"] do
     with %User{ap_id: ^actor_id} = actor <- User.get_cached_by_ap_id(object["id"]) do
@@ -617,7 +641,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   # an error or a tombstone.  This would allow us to verify that a deletion actually took
   # place.
   def handle_incoming(
-        %{"type" => "Delete", "object" => object_id, "actor" => _actor, "id" => _id} = data
+        %{"type" => "Delete", "object" => object_id, "actor" => actor, "id" => _id} = data,
+        _options
       ) do
     object_id = Utils.get_ap_id(object_id)
 
@@ -628,7 +653,30 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
          {:ok, activity} <- ActivityPub.delete(object, false) do
       {:ok, activity}
     else
-      _e -> :error
+      nil ->
+        case User.get_cached_by_ap_id(object_id) do
+          %User{ap_id: ^actor} = user ->
+            {:ok, followers} = User.get_followers(user)
+
+            Enum.each(followers, fn follower ->
+              User.unfollow(follower, user)
+            end)
+
+            {:ok, friends} = User.get_friends(user)
+
+            Enum.each(friends, fn followed ->
+              User.unfollow(user, followed)
+            end)
+
+            User.invalidate_cache(user)
+            Repo.delete(user)
+
+          nil ->
+            :error
+        end
+
+      _e ->
+        :error
     end
   end
 
@@ -638,7 +686,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
           "object" => %{"type" => "Announce", "object" => object_id},
           "actor" => _actor,
           "id" => id
-        } = data
+        } = data,
+        _options
       ) do
     with actor <- Containment.get_actor(data),
          {:ok, %User{} = actor} <- User.get_or_fetch_by_ap_id(actor),
@@ -656,7 +705,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
           "object" => %{"type" => "Follow", "object" => followed},
           "actor" => follower,
           "id" => id
-        } = _data
+        } = _data,
+        _options
       ) do
     with %User{local: true} = followed <- User.get_cached_by_ap_id(followed),
          {:ok, %User{} = follower} <- User.get_or_fetch_by_ap_id(follower),
@@ -674,7 +724,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
           "object" => %{"type" => "Block", "object" => blocked},
           "actor" => blocker,
           "id" => id
-        } = _data
+        } = _data,
+        _options
       ) do
     with true <- Pleroma.Config.get([:activitypub, :accept_blocks]),
          %User{local: true} = blocked <- User.get_cached_by_ap_id(blocked),
@@ -688,7 +739,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   end
 
   def handle_incoming(
-        %{"type" => "Block", "object" => blocked, "actor" => blocker, "id" => id} = _data
+        %{"type" => "Block", "object" => blocked, "actor" => blocker, "id" => id} = _data,
+        _options
       ) do
     with true <- Pleroma.Config.get([:activitypub, :accept_blocks]),
          %User{local: true} = blocked = User.get_cached_by_ap_id(blocked),
@@ -708,7 +760,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
           "object" => %{"type" => "Like", "object" => object_id},
           "actor" => _actor,
           "id" => id
-        } = data
+        } = data,
+        _options
       ) do
     with actor <- Containment.get_actor(data),
          {:ok, %User{} = actor} <- User.get_or_fetch_by_ap_id(actor),
@@ -720,10 +773,10 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
     end
   end
 
-  def handle_incoming(_), do: :error
+  def handle_incoming(_, _), do: :error
 
-  def get_obj_helper(id) do
-    if object = Object.normalize(id), do: {:ok, object}, else: nil
+  def get_obj_helper(id, options \\ []) do
+    if object = Object.normalize(id, true, options), do: {:ok, object}, else: nil
   end
 
   def set_reply_to_uri(%{"inReplyTo" => in_reply_to} = object) when is_binary(in_reply_to) do

lib/pleroma/web/activity_pub/utils.ex

@@ -151,16 +151,18 @@ defmodule Pleroma.Web.ActivityPub.Utils do
 
   def create_context(context) do
     context = context || generate_id("contexts")
-    changeset = Object.context_mapping(context)
 
-    case Repo.insert(changeset) do
-      {:ok, object} ->
+    # Ecto has problems accessing the constraint inside the jsonb,
+    # so we explicitly check for the existed object before insert
+    object = Object.get_cached_by_ap_id(context)
+
+    with true <- is_nil(object),
+         changeset <- Object.context_mapping(context),
+         {:ok, inserted_object} <- Repo.insert(changeset) do
+      inserted_object
+    else
+      _ ->
         object
-
-      # This should be solved by an upsert, but it seems ecto
-      # has problems accessing the constraint inside the jsonb.
-      {:error, _} ->
-        Object.get_cached_by_ap_id(context)
     end
   end
 
@@ -168,14 +170,17 @@ defmodule Pleroma.Web.ActivityPub.Utils do
   Enqueues an activity for federation if it's local
   """
   def maybe_federate(%Activity{local: true} = activity) do
-    priority =
-      case activity.data["type"] do
-        "Delete" -> 10
-        "Create" -> 1
-        _ -> 5
-      end
+    if Pleroma.Config.get!([:instance, :federating]) do
+      priority =
+        case activity.data["type"] do
+          "Delete" -> 10
+          "Create" -> 1
+          _ -> 5
+        end
+
+      Pleroma.Web.Federator.publish(activity, priority)
+    end
 
-    Pleroma.Web.Federator.publish(activity, priority)
     :ok
   end
 
@@ -376,8 +381,8 @@ defmodule Pleroma.Web.ActivityPub.Utils do
   @doc """
   Updates a follow activity's state (for locked accounts).
   """
-  def update_follow_state(
-        %Activity{data: %{"actor" => actor, "object" => object, "state" => "pending"}} = activity,
+  def update_follow_state_for_all(
+        %Activity{data: %{"actor" => actor, "object" => object}} = activity,
         state
       ) do
     try do

lib/pleroma/web/activity_pub/visibility.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Web.ActivityPub.Visibility do
   alias Pleroma.Activity
   alias Pleroma.Object

lib/pleroma/web/admin_api/admin_api_controller.ex

@@ -10,6 +10,8 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.ActivityPub.Relay
   alias Pleroma.Web.AdminAPI.AccountView
+  alias Pleroma.Web.AdminAPI.Config
+  alias Pleroma.Web.AdminAPI.ConfigView
   alias Pleroma.Web.AdminAPI.ReportView
   alias Pleroma.Web.AdminAPI.Search
   alias Pleroma.Web.CommonAPI
@@ -72,7 +74,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
   end
 
   def user_show(conn, %{"nickname" => nickname}) do
-    with %User{} = user <- User.get_cached_by_nickname(nickname) do
+    with %User{} = user <- User.get_cached_by_nickname_or_id(nickname) do
       conn
       |> json(AccountView.render("show.json", %{user: user}))
     else
@@ -158,9 +160,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
   end
 
   def right_add(conn, _) do
-    conn
-    |> put_status(404)
-    |> json(%{error: "No such permission_group"})
+    render_error(conn, :not_found, "No such permission_group")
   end
 
   def right_get(conn, %{"nickname" => nickname}) do
@@ -182,9 +182,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
       )
       when permission_group in ["moderator", "admin"] do
     if admin_nickname == nickname do
-      conn
-      |> put_status(403)
-      |> json(%{error: "You can't revoke your own admin status."})
+      render_error(conn, :forbidden, "You can't revoke your own admin status.")
     else
       user = User.get_cached_by_nickname(nickname)
 
@@ -205,9 +203,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
   end
 
   def right_delete(conn, _) do
-    conn
-    |> put_status(404)
-    |> json(%{error: "No such permission_group"})
+    render_error(conn, :not_found, "No such permission_group")
   end
 
   def set_activation_status(conn, %{"nickname" => nickname, "status" => status}) do
@@ -362,28 +358,63 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
     end
   end
 
+  def config_show(conn, _params) do
+    configs = Pleroma.Repo.all(Config)
+
+    conn
+    |> put_view(ConfigView)
+    |> render("index.json", %{configs: configs})
+  end
+
+  def config_update(conn, %{"configs" => configs}) do
+    updated =
+      if Pleroma.Config.get([:instance, :dynamic_configuration]) do
+        updated =
+          Enum.map(configs, fn
+            %{"group" => group, "key" => key, "value" => value} ->
+              {:ok, config} = Config.update_or_create(%{group: group, key: key, value: value})
+              config
+
+            %{"group" => group, "key" => key, "delete" => "true"} ->
+              {:ok, _} = Config.delete(%{group: group, key: key})
+              nil
+          end)
+          |> Enum.reject(&is_nil(&1))
+
+        Pleroma.Config.TransferTask.load_and_update_env()
+        Mix.Tasks.Pleroma.Config.run(["migrate_from_db", Pleroma.Config.get(:env), "false"])
+        updated
+      else
+        []
+      end
+
+    conn
+    |> put_view(ConfigView)
+    |> render("index.json", %{configs: updated})
+  end
+
   def errors(conn, {:error, :not_found}) do
     conn
-    |> put_status(404)
-    |> json("Not found")
+    |> put_status(:not_found)
+    |> json(dgettext("errors", "Not found"))
   end
 
   def errors(conn, {:error, reason}) do
     conn
-    |> put_status(400)
+    |> put_status(:bad_request)
     |> json(reason)
   end
 
   def errors(conn, {:param_cast, _}) do
     conn
-    |> put_status(400)
-    |> json("Invalid parameters")
+    |> put_status(:bad_request)
+    |> json(dgettext("errors", "Invalid parameters"))
   end
 
   def errors(conn, _) do
     conn
-    |> put_status(500)
-    |> json("Something went wrong")
+    |> put_status(:internal_server_error)
+    |> json(dgettext("errors", "Something went wrong"))
   end
 
   defp page_params(params) do

lib/pleroma/web/admin_api/config.ex

@@ -0,0 +1,165 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.AdminAPI.Config do
+  use Ecto.Schema
+  import Ecto.Changeset
+  import Pleroma.Web.Gettext
+  alias __MODULE__
+  alias Pleroma.Repo
+
+  @type t :: %__MODULE__{}
+
+  schema "config" do
+    field(:key, :string)
+    field(:group, :string)
+    field(:value, :binary)
+
+    timestamps()
+  end
+
+  @spec get_by_params(map()) :: Config.t() | nil
+  def get_by_params(params), do: Repo.get_by(Config, params)
+
+  @spec changeset(Config.t(), map()) :: Changeset.t()
+  def changeset(config, params \\ %{}) do
+    config
+    |> cast(params, [:key, :group, :value])
+    |> validate_required([:key, :group, :value])
+    |> unique_constraint(:key, name: :config_group_key_index)
+  end
+
+  @spec create(map()) :: {:ok, Config.t()} | {:error, Changeset.t()}
+  def create(params) do
+    %Config{}
+    |> changeset(Map.put(params, :value, transform(params[:value])))
+    |> Repo.insert()
+  end
+
+  @spec update(Config.t(), map()) :: {:ok, Config} | {:error, Changeset.t()}
+  def update(%Config{} = config, %{value: value}) do
+    config
+    |> change(value: transform(value))
+    |> Repo.update()
+  end
+
+  @spec update_or_create(map()) :: {:ok, Config.t()} | {:error, Changeset.t()}
+  def update_or_create(params) do
+    with %Config{} = config <- Config.get_by_params(Map.take(params, [:group, :key])) do
+      Config.update(config, params)
+    else
+      nil -> Config.create(params)
+    end
+  end
+
+  @spec delete(map()) :: {:ok, Config.t()} | {:error, Changeset.t()}
+  def delete(params) do
+    with %Config{} = config <- Config.get_by_params(params) do
+      Repo.delete(config)
+    else
+      nil ->
+        err =
+          dgettext("errors", "Config with params %{params} not found", params: inspect(params))
+
+        {:error, err}
+    end
+  end
+
+  @spec from_binary(binary()) :: term()
+  def from_binary(value), do: :erlang.binary_to_term(value)
+
+  @spec from_binary_to_map(binary()) :: any()
+  def from_binary_to_map(binary) do
+    from_binary(binary)
+    |> do_convert()
+  end
+
+  defp do_convert([{k, v}] = value) when is_list(value) and length(value) == 1,
+    do: %{k => do_convert(v)}
+
+  defp do_convert(values) when is_list(values), do: for(val <- values, do: do_convert(val))
+
+  defp do_convert({k, v} = value) when is_tuple(value),
+    do: %{k => do_convert(v)}
+
+  defp do_convert(value) when is_tuple(value), do: %{"tuple" => do_convert(Tuple.to_list(value))}
+
+  defp do_convert(value) when is_binary(value) or is_map(value) or is_number(value), do: value
+
+  defp do_convert(value) when is_atom(value) do
+    string = to_string(value)
+
+    if String.starts_with?(string, "Elixir."),
+      do: String.trim_leading(string, "Elixir."),
+      else: value
+  end
+
+  @spec transform(any()) :: binary()
+  def transform(%{"tuple" => _} = entity), do: :erlang.term_to_binary(do_transform(entity))
+
+  def transform(entity) when is_map(entity) do
+    tuples =
+      for {k, v} <- entity,
+          into: [],
+          do: {if(is_atom(k), do: k, else: String.to_atom(k)), do_transform(v)}
+
+    Enum.reject(tuples, fn {_k, v} -> is_nil(v) end)
+    |> Enum.sort()
+    |> :erlang.term_to_binary()
+  end
+
+  def transform(entity) when is_list(entity) do
+    list = Enum.map(entity, &do_transform(&1))
+    :erlang.term_to_binary(list)
+  end
+
+  def transform(entity), do: :erlang.term_to_binary(entity)
+
+  defp do_transform(%Regex{} = value) when is_map(value), do: value
+
+  defp do_transform(%{"tuple" => [k, values] = entity}) when length(entity) == 2 do
+    {do_transform(k), do_transform(values)}
+  end
+
+  defp do_transform(%{"tuple" => values}) do
+    Enum.reduce(values, {}, fn val, acc -> Tuple.append(acc, do_transform(val)) end)
+  end
+
+  defp do_transform(value) when is_map(value) do
+    values = for {key, val} <- value, into: [], do: {String.to_atom(key), do_transform(val)}
+
+    Enum.sort(values)
+  end
+
+  defp do_transform(value) when is_list(value) do
+    Enum.map(value, &do_transform(&1))
+  end
+
+  defp do_transform(entity) when is_list(entity) and length(entity) == 1, do: hd(entity)
+
+  defp do_transform(value) when is_binary(value) do
+    String.trim(value)
+    |> do_transform_string()
+  end
+
+  defp do_transform(value), do: value
+
+  defp do_transform_string(value) when byte_size(value) == 0, do: nil
+
+  defp do_transform_string(value) do
+    cond do
+      String.starts_with?(value, "Pleroma") or String.starts_with?(value, "Phoenix") ->
+        String.to_existing_atom("Elixir." <> value)
+
+      String.starts_with?(value, ":") ->
+        String.replace(value, ":", "") |> String.to_existing_atom()
+
+      String.starts_with?(value, "i:") ->
+        String.replace(value, "i:", "") |> String.to_integer()
+
+      true ->
+        value
+    end
+  end
+end

lib/pleroma/web/admin_api/views/account_view.ex

@@ -5,8 +5,11 @@
 defmodule Pleroma.Web.AdminAPI.AccountView do
   use Pleroma.Web, :view
 
+  alias Pleroma.HTML
+  alias Pleroma.User
   alias Pleroma.User.Info
   alias Pleroma.Web.AdminAPI.AccountView
+  alias Pleroma.Web.MediaProxy
 
   def render("index.json", %{users: users, count: count, page_size: page_size}) do
     %{
@@ -17,9 +20,14 @@ defmodule Pleroma.Web.AdminAPI.AccountView do
   end
 
   def render("show.json", %{user: user}) do
+    avatar = User.avatar_url(user) |> MediaProxy.url()
+    display_name = HTML.strip_tags(user.name || user.nickname)
+
     %{
       "id" => user.id,
+      "avatar" => avatar,
       "nickname" => user.nickname,
+      "display_name" => display_name,
       "deactivated" => user.info.deactivated,
       "local" => user.local,
       "roles" => Info.roles(user.info),

lib/pleroma/web/admin_api/views/config_view.ex

@@ -0,0 +1,21 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.AdminAPI.ConfigView do
+  use Pleroma.Web, :view
+
+  def render("index.json", %{configs: configs}) do
+    %{
+      configs: render_many(configs, __MODULE__, "show.json", as: :config)
+    }
+  end
+
+  def render("show.json", %{config: config}) do
+    %{
+      key: config.key,
+      group: config.group,
+      value: Pleroma.Web.AdminAPI.Config.from_binary_to_map(config.value)
+    }
+  end
+end

lib/pleroma/web/admin_api/views/report_view.ex

@@ -5,9 +5,9 @@
 defmodule Pleroma.Web.AdminAPI.ReportView do
   use Pleroma.Web, :view
   alias Pleroma.Activity
+  alias Pleroma.HTML
   alias Pleroma.User
   alias Pleroma.Web.CommonAPI.Utils
-  alias Pleroma.Web.MastodonAPI.AccountView
   alias Pleroma.Web.MastodonAPI.StatusView
 
   def render("index.json", %{reports: reports}) do
@@ -23,6 +23,13 @@ defmodule Pleroma.Web.AdminAPI.ReportView do
     [account_ap_id | status_ap_ids] = report.data["object"]
     account = User.get_cached_by_ap_id(account_ap_id)
 
+    content =
+      unless is_nil(report.data["content"]) do
+        HTML.filter_tags(report.data["content"])
+      else
+        nil
+      end
+
     statuses =
       Enum.map(status_ap_ids, fn ap_id ->
         Activity.get_by_ap_id_with_object(ap_id)
@@ -30,12 +37,19 @@ defmodule Pleroma.Web.AdminAPI.ReportView do
 
     %{
       id: report.id,
-      account: AccountView.render("account.json", %{user: account}),
-      actor: AccountView.render("account.json", %{user: user}),
-      content: report.data["content"],
+      account: merge_account_views(account),
+      actor: merge_account_views(user),
+      content: content,
       created_at: created_at,
       statuses: StatusView.render("index.json", %{activities: statuses, as: :activity}),
       state: report.data["state"]
     }
   end
+
+  defp merge_account_views(%User{} = user) do
+    Pleroma.Web.MastodonAPI.AccountView.render("account.json", %{user: user})
+    |> Map.merge(Pleroma.Web.AdminAPI.AccountView.render("show.json", %{user: user}))
+  end
+
+  defp merge_account_views(_), do: %{}
 end

lib/pleroma/web/auth/pleroma_authenticator.ex

@@ -24,6 +24,14 @@ defmodule Pleroma.Web.Auth.PleromaAuthenticator do
     end
   end
 
+  @doc """
+  Gets or creates Pleroma.Registration record from Ueberauth assigns.
+  Note: some strategies (like `keycloak`) might need extra configuration to fill `uid` from callback response —
+    see [`docs/config.md`](docs/config.md).
+  """
+  def get_registration(%Plug.Conn{assigns: %{ueberauth_auth: %{uid: nil}}}),
+    do: {:error, :missing_uid}
+
   def get_registration(%Plug.Conn{
         assigns: %{ueberauth_auth: %{provider: provider, uid: uid} = auth}
       }) do
@@ -51,9 +59,10 @@ defmodule Pleroma.Web.Auth.PleromaAuthenticator do
 
   def get_registration(%Plug.Conn{} = _conn), do: {:error, :missing_credentials}
 
+  @doc "Creates Pleroma.User record basing on params and Pleroma.Registration record."
   def create_from_registration(
         %Plug.Conn{params: %{"authorization" => registration_attrs}},
-        registration
+        %Registration{} = registration
       ) do
     nickname = value([registration_attrs["nickname"], Registration.nickname(registration)])
     email = value([registration_attrs["email"], Registration.email(registration)])

lib/pleroma/web/common_api/common_api.ex

@@ -11,7 +11,9 @@ defmodule Pleroma.Web.CommonAPI do
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.ActivityPub.Utils
+  alias Pleroma.Web.ActivityPub.Visibility
 
+  import Pleroma.Web.Gettext
   import Pleroma.Web.CommonAPI.Utils
 
   def follow(follower, followed) do
@@ -35,9 +37,9 @@ defmodule Pleroma.Web.CommonAPI do
   end
 
   def accept_follow_request(follower, followed) do
-    with {:ok, follower} <- User.maybe_follow(follower, followed),
+    with {:ok, follower} <- User.follow(follower, followed),
          %Activity{} = follow_activity <- Utils.fetch_latest_follow(follower, followed),
-         {:ok, follow_activity} <- Utils.update_follow_state(follow_activity, "accept"),
+         {:ok, follow_activity} <- Utils.update_follow_state_for_all(follow_activity, "accept"),
          {:ok, _activity} <-
            ActivityPub.accept(%{
              to: [follower.ap_id],
@@ -51,7 +53,7 @@ defmodule Pleroma.Web.CommonAPI do
 
   def reject_follow_request(follower, followed) do
     with %Activity{} = follow_activity <- Utils.fetch_latest_follow(follower, followed),
-         {:ok, follow_activity} <- Utils.update_follow_state(follow_activity, "reject"),
+         {:ok, follow_activity} <- Utils.update_follow_state_for_all(follow_activity, "reject"),
          {:ok, _activity} <-
            ActivityPub.reject(%{
              to: [follower.ap_id],
@@ -73,7 +75,7 @@ defmodule Pleroma.Web.CommonAPI do
       {:ok, delete}
     else
       _ ->
-        {:error, "Could not delete"}
+        {:error, dgettext("errors", "Could not delete")}
     end
   end
 
@@ -84,7 +86,7 @@ defmodule Pleroma.Web.CommonAPI do
       ActivityPub.announce(user, object)
     else
       _ ->
-        {:error, "Could not repeat"}
+        {:error, dgettext("errors", "Could not repeat")}
     end
   end
 
@@ -94,7 +96,7 @@ defmodule Pleroma.Web.CommonAPI do
       ActivityPub.unannounce(user, object)
     else
       _ ->
-        {:error, "Could not unrepeat"}
+        {:error, dgettext("errors", "Could not unrepeat")}
     end
   end
 
@@ -105,7 +107,7 @@ defmodule Pleroma.Web.CommonAPI do
       ActivityPub.like(user, object)
     else
       _ ->
-        {:error, "Could not favorite"}
+        {:error, dgettext("errors", "Could not favorite")}
     end
   end
 
@@ -115,7 +117,7 @@ defmodule Pleroma.Web.CommonAPI do
       ActivityPub.unlike(user, object)
     else
       _ ->
-        {:error, "Could not unfavorite"}
+        {:error, dgettext("errors", "Could not unfavorite")}
     end
   end
 
@@ -147,10 +149,10 @@ defmodule Pleroma.Web.CommonAPI do
       object = Object.get_cached_by_ap_id(object.data["id"])
       {:ok, answer_activities, object}
     else
-      {:author, _} -> {:error, "Poll's author can't vote"}
-      {:existing_votes, _} -> {:error, "Already voted"}
-      {:choice_check, {_, false}} -> {:error, "Invalid indices"}
-      {:count_check, false} -> {:error, "Too many choices"}
+      {:author, _} -> {:error, dgettext("errors", "Poll's author can't vote")}
+      {:existing_votes, _} -> {:error, dgettext("errors", "Already voted")}
+      {:choice_check, {_, false}} -> {:error, dgettext("errors", "Invalid indices")}
+      {:count_check, false} -> {:error, dgettext("errors", "Too many choices")}
     end
   end
 
@@ -218,7 +220,7 @@ defmodule Pleroma.Web.CommonAPI do
          cw <- data["spoiler_text"] || "",
          sensitive <- data["sensitive"] || Enum.member?(tags, {"#nsfw", "nsfw"}),
          full_payload <- String.trim(status <> cw),
-         length when length in 1..limit <- String.length(full_payload),
+         :ok <- validate_character_limit(full_payload, attachments, limit),
          object <-
            make_note_data(
              user.ap_id,
@@ -250,7 +252,14 @@ defmodule Pleroma.Web.CommonAPI do
         Pleroma.Web.ControllerHelper.truthy_param?(data["preview"]) || false
       )
     else
-      e -> {:error, e}
+      {:private_to_public, true} ->
+        {:error, dgettext("errors", "The message visibility must be direct")}
+
+      {:error, _} = e ->
+        e
+
+      e ->
+        {:error, e}
     end
   end
 
@@ -285,12 +294,11 @@ defmodule Pleroma.Web.CommonAPI do
            },
            object: %Object{
              data: %{
-               "to" => object_to,
                "type" => "Note"
              }
            }
          } = activity <- get_by_id_or_ap_id(id_or_ap_id),
-         true <- Enum.member?(object_to, "https://www.w3.org/ns/activitystreams#Public"),
+         true <- Visibility.is_public?(activity),
          %{valid?: true} = info_changeset <-
            User.Info.add_pinnned_activity(user.info, activity),
          changeset <-
@@ -302,7 +310,7 @@ defmodule Pleroma.Web.CommonAPI do
         {:error, err}
 
       _ ->
-        {:error, "Could not pin"}
+        {:error, dgettext("errors", "Could not pin")}
     end
   end
 
@@ -319,7 +327,7 @@ defmodule Pleroma.Web.CommonAPI do
         {:error, err}
 
       _ ->
-        {:error, "Could not unpin"}
+        {:error, dgettext("errors", "Could not unpin")}
     end
   end
 
@@ -327,7 +335,7 @@ defmodule Pleroma.Web.CommonAPI do
     with {:ok, _} <- ThreadMute.add_mute(user.id, activity.data["context"]) do
       {:ok, activity}
     else
-      {:error, _} -> {:error, "conversation is already muted"}
+      {:error, _} -> {:error, dgettext("errors", "conversation is already muted")}
     end
   end
 
@@ -372,8 +380,8 @@ defmodule Pleroma.Web.CommonAPI do
       {:ok, activity}
     else
       {:error, err} -> {:error, err}
-      {:account_id, %{}} -> {:error, "Valid `account_id` required"}
-      {:account, nil} -> {:error, "Account not found"}
+      {:account_id, %{}} -> {:error, dgettext("errors", "Valid `account_id` required")}
+      {:account, nil} -> {:error, dgettext("errors", "Account not found")}
     end
   end
 
@@ -382,14 +390,9 @@ defmodule Pleroma.Web.CommonAPI do
          {:ok, activity} <- Utils.update_report_state(activity, state) do
       {:ok, activity}
     else
-      nil ->
-        {:error, :not_found}
-
-      {:error, reason} ->
-        {:error, reason}
-
-      _ ->
-        {:error, "Could not update state"}
+      nil -> {:error, :not_found}
+      {:error, reason} -> {:error, reason}
+      _ -> {:error, dgettext("errors", "Could not update state")}
     end
   end
 
@@ -399,11 +402,8 @@ defmodule Pleroma.Web.CommonAPI do
          {:ok, activity} <- set_visibility(activity, opts) do
       {:ok, activity}
     else
-      nil ->
-        {:error, :not_found}
-
-      {:error, reason} ->
-        {:error, reason}
+      nil -> {:error, :not_found}
+      {:error, reason} -> {:error, reason}
     end
   end
 

lib/pleroma/web/common_api/utils.ex

@@ -3,6 +3,8 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.CommonAPI.Utils do
+  import Pleroma.Web.Gettext
+
   alias Calendar.Strftime
   alias Comeonin.Pbkdf2
   alias Pleroma.Activity
@@ -381,7 +383,7 @@ defmodule Pleroma.Web.CommonAPI.Utils do
          true <- Pbkdf2.checkpw(password, db_user.password_hash) do
       {:ok, db_user}
     else
-      _ -> {:error, "Invalid password."}
+      _ -> {:error, dgettext("errors", "Invalid password.")}
     end
   end
 
@@ -464,7 +466,8 @@ defmodule Pleroma.Web.CommonAPI.Utils do
     if String.length(comment) <= max_size do
       {:ok, format_input(comment, "text/plain")}
     else
-      {:error, "Comment must be up to #{max_size} characters"}
+      {:error,
+       dgettext("errors", "Comment must be up to %{max_size} characters", max_size: max_size)}
     end
   end
 
@@ -499,7 +502,7 @@ defmodule Pleroma.Web.CommonAPI.Utils do
       context
     else
       _e ->
-        {:error, "No such conversation"}
+        {:error, dgettext("errors", "No such conversation")}
     end
   end
 
@@ -513,4 +516,18 @@ defmodule Pleroma.Web.CommonAPI.Utils do
       "inReplyTo" => object.data["id"]
     }
   end
+
+  def validate_character_limit(full_payload, attachments, limit) do
+    length = String.length(full_payload)
+
+    if length < limit do
+      if length > 0 or Enum.count(attachments) > 0 do
+        :ok
+      else
+        {:error, dgettext("errors", "Cannot post an empty status without attachments")}
+      end
+    else
+      {:error, dgettext("errors", "The status is over the character limit")}
+    end
+  end
 end

lib/pleroma/web/controller_helper.ex

@@ -15,4 +15,22 @@ defmodule Pleroma.Web.ControllerHelper do
     |> put_status(status)
     |> json(json)
   end
+
+  @spec fetch_integer_param(map(), String.t(), integer() | nil) :: integer() | nil
+  def fetch_integer_param(params, name, default \\ nil) do
+    params
+    |> Map.get(name, default)
+    |> param_to_integer(default)
+  end
+
+  defp param_to_integer(val, _) when is_integer(val), do: val
+
+  defp param_to_integer(val, default) when is_binary(val) do
+    case Integer.parse(val) do
+      {res, _} -> res
+      _ -> default
+    end
+  end
+
+  defp param_to_integer(_, default), do: default
 end

lib/pleroma/web/endpoint.ex

@@ -7,13 +7,9 @@ defmodule Pleroma.Web.Endpoint do
 
   socket("/socket", Pleroma.Web.UserSocket)
 
-  # Serve at "/" the static files from "priv/static" directory.
-  #
-  # You should set gzip to true if you are running phoenix.digest
-  # when deploying your static files in production.
+  plug(Pleroma.Plugs.SetLocalePlug)
   plug(CORSPlug)
   plug(Pleroma.Plugs.HTTPSecurityPlug)
-
   plug(Pleroma.Plugs.UploadedMedia)
 
   @static_cache_control "public, no-cache"
@@ -30,6 +26,10 @@ defmodule Pleroma.Web.Endpoint do
     }
   )
 
+  # Serve at "/" the static files from "priv/static" directory.
+  #
+  # You should set gzip to true if you are running phoenix.digest
+  # when deploying your static files in production.
   plug(
     Plug.Static,
     at: "/",
@@ -91,7 +91,7 @@ defmodule Pleroma.Web.Endpoint do
     Plug.Session,
     store: :cookie,
     key: cookie_name,
-    signing_salt: {Pleroma.Config, :get, [[__MODULE__, :signing_salt], "CqaoopA2"]},
+    signing_salt: Pleroma.Config.get([__MODULE__, :signing_salt], "CqaoopA2"),
     http_only: true,
     secure: secure_cookies,
     extra: extra

lib/pleroma/web/federator/federator.ex

@@ -22,6 +22,18 @@ defmodule Pleroma.Web.Federator do
     refresh_subscriptions()
   end
 
+  @doc "Addresses [memory leaks on recursive replies fetching](https://git.pleroma.social/pleroma/pleroma/issues/161)"
+  # credo:disable-for-previous-line Credo.Check.Readability.MaxLineLength
+  def allowed_incoming_reply_depth?(depth) do
+    max_replies_depth = Pleroma.Config.get([:instance, :federation_incoming_replies_max_depth])
+
+    if max_replies_depth do
+      (depth || 1) <= max_replies_depth
+    else
+      true
+    end
+  end
+
   # Client API
 
   def incoming_doc(doc) do

lib/pleroma/web/federator/retry_queue.ex

@@ -15,7 +15,9 @@ defmodule Pleroma.Web.Federator.RetryQueue do
 
   def start_link do
     enabled =
-      if Mix.env() == :test, do: true, else: Pleroma.Config.get([__MODULE__, :enabled], false)
+      if Pleroma.Config.get(:env) == :test,
+        do: true,
+        else: Pleroma.Config.get([__MODULE__, :enabled], false)
 
     if enabled do
       Logger.info("Starting retry queue")
@@ -219,7 +221,7 @@ defmodule Pleroma.Web.Federator.RetryQueue do
     {:noreply, state}
   end
 
-  if Mix.env() == :test do
+  if Pleroma.Config.get(:env) == :test do
     defp growth_function(_retries) do
       _shutit = Pleroma.Config.get([__MODULE__, :initial_timeout])
       DateTime.to_unix(DateTime.utc_now()) - 1

lib/pleroma/web/mastodon_api/mastodon_api.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Web.MastodonAPI.MastodonAPI do
   import Ecto.Query
   import Ecto.Changeset

lib/pleroma/web/mastodon_api/mastodon_api_controller.ex

@@ -14,7 +14,6 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
   alias Pleroma.HTTP
   alias Pleroma.Notification
   alias Pleroma.Object
-  alias Pleroma.Object.Fetcher
   alias Pleroma.Pagination
   alias Pleroma.Repo
   alias Pleroma.ScheduledActivity
@@ -47,14 +46,8 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
 
   require Logger
 
-  plug(
-    Pleroma.Plugs.RateLimitPlug,
-    %{
-      max_requests: Config.get([:app_account_creation, :max_requests]),
-      interval: Config.get([:app_account_creation, :interval])
-    }
-    when action in [:account_register]
-  )
+  plug(Pleroma.Plugs.RateLimiter, :app_account_creation when action == :account_register)
+  plug(Pleroma.Plugs.RateLimiter, :search when action in [:search, :search2, :account_search])
 
   @local_mastodon_name "Mastodon-Local"
 
@@ -143,6 +136,14 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
           _ -> :error
         end
       end)
+      |> add_if_present(params, "pleroma_background_image", :background, fn value ->
+        with %Plug.Upload{} <- value,
+             {:ok, object} <- ActivityPub.upload(value, type: :background) do
+          {:ok, object.data}
+        else
+          _ -> :error
+        end
+      end)
       |> Map.put(:emoji, user_info_emojis)
 
     info_cng = User.Info.profile_update(user.info, info_params)
@@ -159,16 +160,83 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
         AccountView.render("account.json", %{user: user, for: user, with_pleroma_settings: true})
       )
     else
-      _e ->
-        conn
-        |> put_status(403)
-        |> json(%{error: "Invalid request"})
+      _e -> render_error(conn, :forbidden, "Invalid request")
+    end
+  end
+
+  def update_avatar(%{assigns: %{user: user}} = conn, %{"img" => ""}) do
+    change = Changeset.change(user, %{avatar: nil})
+    {:ok, user} = User.update_and_set_cache(change)
+    CommonAPI.update(user)
+
+    json(conn, %{url: nil})
+  end
+
+  def update_avatar(%{assigns: %{user: user}} = conn, params) do
+    {:ok, object} = ActivityPub.upload(params, type: :avatar)
+    change = Changeset.change(user, %{avatar: object.data})
+    {:ok, user} = User.update_and_set_cache(change)
+    CommonAPI.update(user)
+    %{"url" => [%{"href" => href} | _]} = object.data
+
+    json(conn, %{url: href})
+  end
+
+  def update_banner(%{assigns: %{user: user}} = conn, %{"banner" => ""}) do
+    with new_info <- %{"banner" => %{}},
+         info_cng <- User.Info.profile_update(user.info, new_info),
+         changeset <- Ecto.Changeset.change(user) |> Ecto.Changeset.put_embed(:info, info_cng),
+         {:ok, user} <- User.update_and_set_cache(changeset) do
+      CommonAPI.update(user)
+
+      json(conn, %{url: nil})
+    end
+  end
+
+  def update_banner(%{assigns: %{user: user}} = conn, params) do
+    with {:ok, object} <- ActivityPub.upload(%{"img" => params["banner"]}, type: :banner),
+         new_info <- %{"banner" => object.data},
+         info_cng <- User.Info.profile_update(user.info, new_info),
+         changeset <- Ecto.Changeset.change(user) |> Ecto.Changeset.put_embed(:info, info_cng),
+         {:ok, user} <- User.update_and_set_cache(changeset) do
+      CommonAPI.update(user)
+      %{"url" => [%{"href" => href} | _]} = object.data
+
+      json(conn, %{url: href})
+    end
+  end
+
+  def update_background(%{assigns: %{user: user}} = conn, %{"img" => ""}) do
+    with new_info <- %{"background" => %{}},
+         info_cng <- User.Info.profile_update(user.info, new_info),
+         changeset <- Ecto.Changeset.change(user) |> Ecto.Changeset.put_embed(:info, info_cng),
+         {:ok, _user} <- User.update_and_set_cache(changeset) do
+      json(conn, %{url: nil})
+    end
+  end
+
+  def update_background(%{assigns: %{user: user}} = conn, params) do
+    with {:ok, object} <- ActivityPub.upload(params, type: :background),
+         new_info <- %{"background" => object.data},
+         info_cng <- User.Info.profile_update(user.info, new_info),
+         changeset <- Ecto.Changeset.change(user) |> Ecto.Changeset.put_embed(:info, info_cng),
+         {:ok, _user} <- User.update_and_set_cache(changeset) do
+      %{"url" => [%{"href" => href} | _]} = object.data
+
+      json(conn, %{url: href})
     end
   end
 
   def verify_credentials(%{assigns: %{user: user}} = conn, _) do
+    chat_token = Phoenix.Token.sign(conn, "user socket", user.id)
+
     account =
-      AccountView.render("account.json", %{user: user, for: user, with_pleroma_settings: true})
+      AccountView.render("account.json", %{
+        user: user,
+        for: user,
+        with_pleroma_settings: true,
+        with_chat_token: chat_token
+      })
 
     json(conn, account)
   end
@@ -187,10 +255,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
       account = AccountView.render("account.json", %{user: user, for: for_user})
       json(conn, account)
     else
-      _e ->
-        conn
-        |> put_status(404)
-        |> json(%{error: "Can't find user"})
+      _e -> render_error(conn, :not_found, "Can't find user")
     end
   end
 
@@ -234,7 +299,9 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
         "static_url" => url,
         "visible_in_picker" => true,
         "url" => url,
-        "tags" => tags
+        "tags" => tags,
+        # Assuming that a comma is authorized in the category name
+        "category" => (tags -- ["Custom"]) |> Enum.join(",")
       }
     end)
   end
@@ -348,6 +415,10 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
 
   def user_statuses(%{assigns: %{user: reading_user}} = conn, params) do
     with %User{} = user <- User.get_cached_by_id(params["id"]) do
+      params =
+        params
+        |> Map.put("tag", params["tagged"])
+
       activities = ActivityPub.fetch_user_activities(user, reading_user, params)
 
       conn
@@ -434,41 +505,44 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
       |> put_view(StatusView)
       |> try_render("poll.json", %{object: object, for: user})
     else
-      nil ->
-        conn
-        |> put_status(404)
-        |> json(%{error: "Record not found"})
-
-      false ->
-        conn
-        |> put_status(404)
-        |> json(%{error: "Record not found"})
+      nil -> render_error(conn, :not_found, "Record not found")
+      false -> render_error(conn, :not_found, "Record not found")
     end
   end
 
+  defp get_cached_vote_or_vote(user, object, choices) do
+    idempotency_key = "polls:#{user.id}:#{object.data["id"]}"
+
+    {_, res} =
+      Cachex.fetch(:idempotency_cache, idempotency_key, fn _ ->
+        case CommonAPI.vote(user, object, choices) do
+          {:error, _message} = res -> {:ignore, res}
+          res -> {:commit, res}
+        end
+      end)
+
+    res
+  end
+
   def poll_vote(%{assigns: %{user: user}} = conn, %{"id" => id, "choices" => choices}) do
     with %Object{} = object <- Object.get_by_id(id),
          true <- object.data["type"] == "Question",
          %Activity{} = activity <- Activity.get_create_by_object_ap_id(object.data["id"]),
          true <- Visibility.visible_for_user?(activity, user),
-         {:ok, _activities, object} <- CommonAPI.vote(user, object, choices) do
+         {:ok, _activities, object} <- get_cached_vote_or_vote(user, object, choices) do
       conn
       |> put_view(StatusView)
       |> try_render("poll.json", %{object: object, for: user})
     else
       nil ->
-        conn
-        |> put_status(404)
-        |> json(%{error: "Record not found"})
+        render_error(conn, :not_found, "Record not found")
 
       false ->
-        conn
-        |> put_status(404)
-        |> json(%{error: "Record not found"})
+        render_error(conn, :not_found, "Record not found")
 
       {:error, message} ->
         conn
-        |> put_status(422)
+        |> put_status(:unprocessable_entity)
         |> json(%{error: message})
     end
   end
@@ -522,15 +596,6 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
     end
   end
 
-  def post_status(conn, %{"status" => "", "media_ids" => media_ids} = params)
-      when length(media_ids) > 0 do
-    params =
-      params
-      |> Map.put("status", ".")
-
-    post_status(conn, params)
-  end
-
   def post_status(%{assigns: %{user: user}} = conn, %{"status" => _} = params) do
     params =
       params
@@ -548,18 +613,13 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
     else
       params = Map.drop(params, ["scheduled_at"])
 
-      case get_cached_status_or_post(conn, params) do
-        {:ignore, message} ->
-          conn
-          |> put_status(422)
-          |> json(%{error: message})
-
+      case CommonAPI.post(user, params) do
         {:error, message} ->
           conn
-          |> put_status(422)
+          |> put_status(:unprocessable_entity)
           |> json(%{error: message})
 
-        {_, activity} ->
+        {:ok, activity} ->
           conn
           |> put_view(StatusView)
           |> try_render("status.json", %{activity: activity, for: user, as: :activity})
@@ -567,29 +627,11 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
     end
   end
 
-  defp get_cached_status_or_post(%{assigns: %{user: user}} = conn, params) do
-    idempotency_key =
-      case get_req_header(conn, "idempotency-key") do
-        [key] -> key
-        _ -> Ecto.UUID.generate()
-      end
-
-    Cachex.fetch(:idempotency_cache, idempotency_key, fn _ ->
-      case CommonAPI.post(user, params) do
-        {:ok, activity} -> activity
-        {:error, message} -> {:ignore, message}
-      end
-    end)
-  end
-
   def delete_status(%{assigns: %{user: user}} = conn, %{"id" => id}) do
     with {:ok, %Activity{}} <- CommonAPI.delete(id, user) do
       json(conn, %{})
     else
-      _e ->
-        conn
-        |> put_status(403)
-        |> json(%{error: "Can't delete this post"})
+      _e -> render_error(conn, :forbidden, "Can't delete this post")
     end
   end
 
@@ -637,8 +679,8 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
     else
       {:error, reason} ->
         conn
-        |> put_resp_content_type("application/json")
-        |> send_resp(:bad_request, Jason.encode!(%{"error" => reason}))
+        |> put_status(:bad_request)
+        |> json(%{"error" => reason})
     end
   end
 
@@ -714,8 +756,8 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
     else
       {:error, reason} ->
         conn
-        |> put_resp_content_type("application/json")
-        |> send_resp(403, Jason.encode!(%{"error" => reason}))
+        |> put_status(:forbidden)
+        |> json(%{"error" => reason})
     end
   end
 
@@ -730,8 +772,8 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
     else
       {:error, reason} ->
         conn
-        |> put_resp_content_type("application/json")
-        |> send_resp(403, Jason.encode!(%{"error" => reason}))
+        |> put_status(:forbidden)
+        |> json(%{"error" => reason})
     end
   end
 
@@ -809,9 +851,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
         conn
         |> json(rendered)
       else
-        conn
-        |> put_resp_content_type("application/json")
-        |> send_resp(415, Jason.encode!(%{"error" => "mascots can only be images"}))
+        render_error(conn, :unsupported_media_type, "mascots can only be images")
       end
     end
   end
@@ -831,7 +871,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
 
       conn
       |> put_view(AccountView)
-      |> render(AccountView, "accounts.json", %{for: user, users: users, as: :user})
+      |> render("accounts.json", %{for: user, users: users, as: :user})
     else
       _ -> json(conn, [])
     end
@@ -940,8 +980,8 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
     else
       {:error, message} ->
         conn
-        |> put_resp_content_type("application/json")
-        |> send_resp(403, Jason.encode!(%{"error" => message}))
+        |> put_status(:forbidden)
+        |> json(%{error: message})
     end
   end
 
@@ -954,8 +994,8 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
     else
       {:error, message} ->
         conn
-        |> put_resp_content_type("application/json")
-        |> send_resp(403, Jason.encode!(%{"error" => message}))
+        |> put_status(:forbidden)
+        |> json(%{error: message})
     end
   end
 
@@ -972,8 +1012,8 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
 
       {:error, message} ->
         conn
-        |> put_resp_content_type("application/json")
-        |> send_resp(403, Jason.encode!(%{"error" => message}))
+        |> put_status(:forbidden)
+        |> json(%{error: message})
     end
   end
 
@@ -990,8 +1030,8 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
 
       {:error, message} ->
         conn
-        |> put_resp_content_type("application/json")
-        |> send_resp(403, Jason.encode!(%{"error" => message}))
+        |> put_status(:forbidden)
+        |> json(%{error: message})
     end
   end
 
@@ -1020,8 +1060,8 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
     else
       {:error, message} ->
         conn
-        |> put_resp_content_type("application/json")
-        |> send_resp(403, Jason.encode!(%{"error" => message}))
+        |> put_status(:forbidden)
+        |> json(%{error: message})
     end
   end
 
@@ -1034,8 +1074,8 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
     else
       {:error, message} ->
         conn
-        |> put_resp_content_type("application/json")
-        |> send_resp(403, Jason.encode!(%{"error" => message}))
+        |> put_status(:forbidden)
+        |> json(%{error: message})
     end
   end
 
@@ -1056,8 +1096,8 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
     else
       {:error, message} ->
         conn
-        |> put_resp_content_type("application/json")
-        |> send_resp(403, Jason.encode!(%{"error" => message}))
+        |> put_status(:forbidden)
+        |> json(%{error: message})
     end
   end
 
@@ -1071,8 +1111,8 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
     else
       {:error, message} ->
         conn
-        |> put_resp_content_type("application/json")
-        |> send_resp(403, Jason.encode!(%{"error" => message}))
+        |> put_status(:forbidden)
+        |> json(%{error: message})
     end
   end
 
@@ -1106,8 +1146,8 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
     else
       {:error, message} ->
         conn
-        |> put_resp_content_type("application/json")
-        |> send_resp(403, Jason.encode!(%{"error" => message}))
+        |> put_status(:forbidden)
+        |> json(%{error: message})
     end
   end
 
@@ -1120,119 +1160,11 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
     else
       {:error, message} ->
         conn
-        |> put_resp_content_type("application/json")
-        |> send_resp(403, Jason.encode!(%{"error" => message}))
+        |> put_status(:forbidden)
+        |> json(%{error: message})
     end
   end
 
-  def status_search_query_with_gin(q, query) do
-    from([a, o] in q,
-      where:
-        fragment(
-          "to_tsvector('english', ?->>'content') @@ plainto_tsquery('english', ?)",
-          o.data,
-          ^query
-        ),
-      order_by: [desc: :id]
-    )
-  end
-
-  def status_search_query_with_rum(q, query) do
-    from([a, o] in q,
-      where:
-        fragment(
-          "? @@ plainto_tsquery('english', ?)",
-          o.fts_content,
-          ^query
-        ),
-      order_by: [fragment("? <=> now()::date", o.inserted_at)]
-    )
-  end
-
-  def status_search(user, query) do
-    fetched =
-      if Regex.match?(~r/https?:/, query) do
-        with {:ok, object} <- Fetcher.fetch_object_from_id(query),
-             %Activity{} = activity <- Activity.get_create_by_object_ap_id(object.data["id"]),
-             true <- Visibility.visible_for_user?(activity, user) do
-          [activity]
-        else
-          _e -> []
-        end
-      end || []
-
-    q =
-      from([a, o] in Activity.with_preloaded_object(Activity),
-        where: fragment("?->>'type' = 'Create'", a.data),
-        where: "https://www.w3.org/ns/activitystreams#Public" in a.recipients,
-        limit: 40
-      )
-
-    q =
-      if Pleroma.Config.get([:database, :rum_enabled]) do
-        status_search_query_with_rum(q, query)
-      else
-        status_search_query_with_gin(q, query)
-      end
-
-    Repo.all(q) ++ fetched
-  end
-
-  def search2(%{assigns: %{user: user}} = conn, %{"q" => query} = params) do
-    accounts = User.search(query, resolve: params["resolve"] == "true", for_user: user)
-
-    statuses = status_search(user, query)
-
-    tags_path = Web.base_url() <> "/tag/"
-
-    tags =
-      query
-      |> String.split()
-      |> Enum.uniq()
-      |> Enum.filter(fn tag -> String.starts_with?(tag, "#") end)
-      |> Enum.map(fn tag -> String.slice(tag, 1..-1) end)
-      |> Enum.map(fn tag -> %{name: tag, url: tags_path <> tag} end)
-
-    res = %{
-      "accounts" => AccountView.render("accounts.json", users: accounts, for: user, as: :user),
-      "statuses" =>
-        StatusView.render("index.json", activities: statuses, for: user, as: :activity),
-      "hashtags" => tags
-    }
-
-    json(conn, res)
-  end
-
-  def search(%{assigns: %{user: user}} = conn, %{"q" => query} = params) do
-    accounts = User.search(query, resolve: params["resolve"] == "true", for_user: user)
-
-    statuses = status_search(user, query)
-
-    tags =
-      query
-      |> String.split()
-      |> Enum.uniq()
-      |> Enum.filter(fn tag -> String.starts_with?(tag, "#") end)
-      |> Enum.map(fn tag -> String.slice(tag, 1..-1) end)
-
-    res = %{
-      "accounts" => AccountView.render("accounts.json", users: accounts, for: user, as: :user),
-      "statuses" =>
-        StatusView.render("index.json", activities: statuses, for: user, as: :activity),
-      "hashtags" => tags
-    }
-
-    json(conn, res)
-  end
-
-  def account_search(%{assigns: %{user: user}} = conn, %{"q" => query} = params) do
-    accounts = User.search(query, resolve: params["resolve"] == "true", for_user: user)
-
-    res = AccountView.render("accounts.json", users: accounts, for: user, as: :user)
-
-    json(conn, res)
-  end
-
   def favourites(%{assigns: %{user: user}} = conn, params) do
     params =
       params
@@ -1277,13 +1209,8 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
       |> put_view(StatusView)
       |> render("index.json", %{activities: activities, for: for_user, as: :activity})
     else
-      nil ->
-        {:error, :not_found}
-
-      true ->
-        conn
-        |> put_status(403)
-        |> json(%{error: "Can't get favorites"})
+      nil -> {:error, :not_found}
+      true -> render_error(conn, :forbidden, "Can't get favorites")
     end
   end
 
@@ -1315,10 +1242,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
       res = ListView.render("list.json", list: list)
       json(conn, res)
     else
-      _e ->
-        conn
-        |> put_status(404)
-        |> json(%{error: "Record not found"})
+      _e -> render_error(conn, :not_found, "Record not found")
     end
   end
 
@@ -1334,7 +1258,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
       json(conn, %{})
     else
       _e ->
-        json(conn, "error")
+        json(conn, dgettext("errors", "error"))
     end
   end
 
@@ -1385,7 +1309,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
       json(conn, res)
     else
       _e ->
-        json(conn, "error")
+        json(conn, dgettext("errors", "error"))
     end
   end
 
@@ -1409,10 +1333,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
       |> put_view(StatusView)
       |> render("index.json", %{activities: activities, for: user, as: :activity})
     else
-      _e ->
-        conn
-        |> put_status(403)
-        |> json(%{error: "Error."})
+      _e -> render_error(conn, :forbidden, "Error.")
     end
   end
 
@@ -1531,8 +1452,8 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
     else
       e ->
         conn
-        |> put_resp_content_type("application/json")
-        |> send_resp(500, Jason.encode!(%{"error" => inspect(e)}))
+        |> put_status(:internal_server_error)
+        |> json(%{error: inspect(e)})
     end
   end
 
@@ -1700,20 +1621,18 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
       |> Enum.map_join(", ", fn {_k, v} -> v end)
 
     conn
-    |> put_status(422)
+    |> put_status(:unprocessable_entity)
     |> json(%{error: error_message})
   end
 
   def errors(conn, {:error, :not_found}) do
-    conn
-    |> put_status(404)
-    |> json(%{error: "Record not found"})
+    render_error(conn, :not_found, "Record not found")
   end
 
   def errors(conn, _) do
     conn
-    |> put_status(500)
-    |> json("Something went wrong")
+    |> put_status(:internal_server_error)
+    |> json(dgettext("errors", "Something went wrong"))
   end
 
   def suggestions(%{assigns: %{user: user}} = conn, _) do
@@ -1833,21 +1752,17 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
     else
       {:error, errors} ->
         conn
-        |> put_status(400)
-        |> json(Jason.encode!(errors))
+        |> put_status(:bad_request)
+        |> json(errors)
     end
   end
 
   def account_register(%{assigns: %{app: _app}} = conn, _params) do
-    conn
-    |> put_status(400)
-    |> json(%{error: "Missing parameters"})
+    render_error(conn, :bad_request, "Missing parameters")
   end
 
   def account_register(conn, _) do
-    conn
-    |> put_status(403)
-    |> json(%{error: "Invalid credentials"})
+    render_error(conn, :forbidden, "Invalid credentials")
   end
 
   def conversations(%{assigns: %{user: user}} = conn, params) do
@@ -1877,21 +1792,14 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
 
   def try_render(conn, target, params)
       when is_binary(target) do
-    res = render(conn, target, params)
-
-    if res == nil do
-      conn
-      |> put_status(501)
-      |> json(%{error: "Can't display this activity"})
-    else
-      res
+    case render(conn, target, params) do
+      nil -> render_error(conn, :not_implemented, "Can't display this activity")
+      res -> res
     end
   end
 
   def try_render(conn, _, _) do
-    conn
-    |> put_status(501)
-    |> json(%{error: "Can't display this activity"})
+    render_error(conn, :not_implemented, "Can't display this activity")
   end
 
   defp present?(nil), do: false

lib/pleroma/web/mastodon_api/search_controller.ex

@@ -0,0 +1,90 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.MastodonAPI.SearchController do
+  use Pleroma.Web, :controller
+
+  alias Pleroma.Activity
+  alias Pleroma.Plugs.RateLimiter
+  alias Pleroma.User
+  alias Pleroma.Web
+  alias Pleroma.Web.ControllerHelper
+  alias Pleroma.Web.MastodonAPI.AccountView
+  alias Pleroma.Web.MastodonAPI.StatusView
+
+  require Logger
+  plug(RateLimiter, :search when action in [:search, :search2, :account_search])
+
+  def search2(%{assigns: %{user: user}} = conn, %{"q" => query} = params) do
+    accounts = with_fallback(fn -> User.search(query, search_options(params, user)) end, [])
+    statuses = with_fallback(fn -> Activity.search(user, query) end, [])
+
+    tags_path = Web.base_url() <> "/tag/"
+
+    tags =
+      query
+      |> prepare_tags
+      |> Enum.map(fn tag -> %{name: tag, url: tags_path <> tag} end)
+
+    res = %{
+      "accounts" => AccountView.render("accounts.json", users: accounts, for: user, as: :user),
+      "statuses" =>
+        StatusView.render("index.json", activities: statuses, for: user, as: :activity),
+      "hashtags" => tags
+    }
+
+    json(conn, res)
+  end
+
+  def search(%{assigns: %{user: user}} = conn, %{"q" => query} = params) do
+    accounts = with_fallback(fn -> User.search(query, search_options(params, user)) end)
+    statuses = with_fallback(fn -> Activity.search(user, query) end)
+
+    tags = prepare_tags(query)
+
+    res = %{
+      "accounts" => AccountView.render("accounts.json", users: accounts, for: user, as: :user),
+      "statuses" =>
+        StatusView.render("index.json", activities: statuses, for: user, as: :activity),
+      "hashtags" => tags
+    }
+
+    json(conn, res)
+  end
+
+  def account_search(%{assigns: %{user: user}} = conn, %{"q" => query} = params) do
+    accounts = User.search(query, search_options(params, user))
+    res = AccountView.render("accounts.json", users: accounts, for: user, as: :user)
+
+    json(conn, res)
+  end
+
+  defp prepare_tags(query) do
+    query
+    |> String.split()
+    |> Enum.uniq()
+    |> Enum.filter(fn tag -> String.starts_with?(tag, "#") end)
+    |> Enum.map(fn tag -> String.slice(tag, 1..-1) end)
+  end
+
+  defp search_options(params, user) do
+    [
+      resolve: params["resolve"] == "true",
+      following: params["following"] == "true",
+      limit: ControllerHelper.fetch_integer_param(params, "limit"),
+      offset: ControllerHelper.fetch_integer_param(params, "offset"),
+      for_user: user
+    ]
+  end
+
+  defp with_fallback(f, fallback \\ []) do
+    try do
+      f.()
+    rescue
+      error ->
+        Logger.error("#{__MODULE__} search error: #{inspect(error)}")
+        fallback
+    end
+  end
+end

lib/pleroma/web/mastodon_api/subscription_controller.ex

@@ -59,13 +59,13 @@ defmodule Pleroma.Web.MastodonAPI.SubscriptionController do
   #
   def errors(conn, {:error, :not_found}) do
     conn
-    |> put_status(404)
-    |> json("Not found")
+    |> put_status(:not_found)
+    |> json(dgettext("errors", "Not found"))
   end
 
   def errors(conn, _) do
     conn
-    |> put_status(500)
-    |> json("Something went wrong")
+    |> put_status(:internal_server_error)
+    |> json(dgettext("errors", "Something went wrong"))
   end
 end

lib/pleroma/web/mastodon_api/views/account_view.ex

@@ -66,6 +66,8 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do
   end
 
   defp do_render("account.json", %{user: user} = opts) do
+    display_name = HTML.strip_tags(user.name || user.nickname)
+
     image = User.avatar_url(user) |> MediaProxy.url()
     header = User.banner_url(user) |> MediaProxy.url()
     user_info = User.get_cached_user_info(user)
@@ -96,7 +98,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do
       id: to_string(user.id),
       username: username_from_nickname(user.nickname),
       acct: user.nickname,
-      display_name: user.name || user.nickname,
+      display_name: display_name,
       locked: user_info.locked,
       created_at: Utils.to_masto_date(user.inserted_at),
       followers_count: user_info.follower_count,
@@ -125,13 +127,15 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do
         hide_follows: user.info.hide_follows,
         hide_favorites: user.info.hide_favorites,
         relationship: relationship,
-        skip_thread_containment: user.info.skip_thread_containment
+        skip_thread_containment: user.info.skip_thread_containment,
+        background_image: image_url(user.info.background) |> MediaProxy.url()
       }
     }
     |> maybe_put_role(user, opts[:for])
     |> maybe_put_settings(user, opts[:for], user_info)
     |> maybe_put_notification_settings(user, opts[:for])
     |> maybe_put_settings_store(user, opts[:for], opts)
+    |> maybe_put_chat_token(user, opts[:for], opts)
   end
 
   defp username_from_nickname(string) when is_binary(string) do
@@ -163,6 +167,15 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do
 
   defp maybe_put_settings_store(data, _, _, _), do: data
 
+  defp maybe_put_chat_token(data, %User{id: id}, %User{id: id}, %{
+         with_chat_token: token
+       }) do
+    data
+    |> Kernel.put_in([:pleroma, :chat_token], token)
+  end
+
+  defp maybe_put_chat_token(data, _, _, _), do: data
+
   defp maybe_put_role(data, %User{info: %{show_role: true}} = user, _) do
     data
     |> Kernel.put_in([:pleroma, :is_admin], user.info.is_admin)
@@ -182,4 +195,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do
   end
 
   defp maybe_put_notification_settings(data, _, _), do: data
+
+  defp image_url(%{"url" => [%{"href" => href} | _]}), do: href
+  defp image_url(_), do: nil
 end

lib/pleroma/web/mastodon_api/views/conversation_view.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Web.MastodonAPI.ConversationView do
   use Pleroma.Web, :view